Index: /branches/eam_branches/ipp-20130307/DataStore/lib/DataStore/Utils.pm =================================================================== --- /branches/eam_branches/ipp-20130307/DataStore/lib/DataStore/Utils.pm (revision 35412) +++ /branches/eam_branches/ipp-20130307/DataStore/lib/DataStore/Utils.pm (revision 35413) @@ -42,5 +42,5 @@ $BYTE_FIELD = qr/^\d+$/; $MD5_FIELD = qr/^[0-9a-f]{32}$/; -%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS notset ); +%KNOWN_FILE_TYPES = map { $_ => 1 } qw( chip psrequest psresults pstamp chipproc warp stack diff ipp-mops table text xml tgz fits IPP-MOPS IPP-PSPS ipp-psps notset ); %KNOWN_FILESET_TYPES = map { $_ => 1 } qw( OBJECT BIAS DARK SKYFLAT DOMEFLAT OOF SHACKHARTMANN PSREQUEST PSRESULTS IPP-MOPS XRAY FOCUS MOPS_DETECTABILITY_QUERY MOPS_DETECTABILITY_RESPONSE MOPS_TRANSIENT_DETECTIONS LED notset IPP_PSPS IPP-DIST table); %KNOWN_PRODUCT_TYPES = map { $_ => 1 } qw( image dump psrequest psresults table ipp-dist ipp-misc dqresults); Index: /branches/eam_branches/ipp-20130307/DataStoreServer/scripts/dsreg =================================================================== --- /branches/eam_branches/ipp-20130307/DataStoreServer/scripts/dsreg (revision 35412) +++ /branches/eam_branches/ipp-20130307/DataStoreServer/scripts/dsreg (revision 35413) @@ -236,7 +236,7 @@ $dbh->disconnect(); - if ($remove) { + if ($remove and -e $fileset_dir) { my $rc; - if (($rc = system "rm -r $fileset_dir")) { + if (($rc = system "rm -rf $fileset_dir")) { &my_die("failed to remove $fileset_dir error code: $rc", $PS_EXIT_UNKNOWN_ERROR); } Index: /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35412) +++ /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35413) @@ -15,4 +15,5 @@ our @EXPORT_OK = qw( locate_images + locate_images_for_row resolve_project getCamRunByCamID @@ -46,5 +47,5 @@ my $last_project = ""; -my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool); +my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool, $releasetool); my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump); @@ -56,5 +57,5 @@ my $rowList = shift; # required my $req_type = shift; # required - my $img_type = shift; # required + my $stage = shift; # required my $id = shift; # required unless req_type eq bycoord or byskycell my $tess_id = shift; @@ -78,5 +79,7 @@ ($req_type ne "byskycell"); - $default_data_groups = load_data_groups($verbose); + if (!$data_group and !$default_data_groups) { + $default_data_groups = load_data_groups($verbose); + } my $dateobs_begin; @@ -104,14 +107,14 @@ my $x = $row->{CENTER_X}; my $y = $row->{CENTER_Y}; - my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); return $results; } - if (($req_type eq "byid") and ($img_type eq "diff")) { + if (($req_type eq "byid") and ($stage eq "diff")) { # lookups of all of the information for diff images requires a two level lookup to # get the exposure information. Switching the req_type allows us to keep that code # in one place $req_type = "bydiff"; - my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose); + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose); return $results; } @@ -120,9 +123,9 @@ # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or # the image from the diffRun - my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose); + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose); if (!$results) { return undef; } - if ($img_type eq "diff") { + if ($stage eq "diff") { # lookup_diff has done all of the work return $results; @@ -130,15 +133,15 @@ my $image = $results->[0]; - if ($img_type eq "raw") { + if ($stage eq "raw") { $req_type = "byid"; $id = $image->{exp_id}; return undef if !$id; # fall through and lookup byid - } elsif ($img_type eq "chip") { + } elsif ($stage eq "chip") { $req_type = "byid"; $id = $image->{chip_id}; return undef if !$id; # fall through and lookup byid - } elsif ($img_type eq "warp") { + } elsif ($stage eq "warp") { $req_type = "byid"; $id = $image->{warp_id}; @@ -146,5 +149,5 @@ return undef if !$id; # fall through and lookup by warp_id - } elsif ($img_type eq "stack") { + } elsif ($stage eq "stack") { $req_type = "byid"; $id = $image->{stack_id}; @@ -153,9 +156,9 @@ } else { # This is checked this elsewhere? - print STDERR "Error: $img_type is an unknown image type\n"; + print STDERR "Error: $stage is an unknown image type\n"; return undef; } } elsif ($req_type eq "byskycell") { - if (($img_type eq "raw") or ($img_type eq "chip")) { + if (($stage eq "raw") or ($stage eq "chip")) { print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n"; return undef; @@ -167,6 +170,140 @@ } - my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component, - $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component, + $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + undef, undef, undef, $verbose); + + return $results; +} + +sub locate_images_for_row { + my $ipprc = shift; + my $imagedb = shift; + my $camera = shift; + my $row = shift; + my $verbose = shift; + + my $req_type = $row->{REQ_TYPE}; + + # we die in response to bad data in request files + # The caller is responsible for updating the database + # pstampparse.pl error checks now so this shouldn't happen + &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) + if ($req_type ne "byid") and + ($req_type ne "byexp") and + ($req_type ne "bycoord") and + ($req_type ne "bydiff") and + ($req_type ne "byskycell"); + + my $stage = $row->{IMG_TYPE}; + my $id = $row->{ID}; + my $component = $row->{COMPONENT}; + my $tess_id = $row->{TESS_ID}; + + my $filter = $row->{REQFILT}; + my $mjd_min = $row->{MJD_MIN}; + my $mjd_max = $row->{MJD_MAX}; + my $data_group = $row->{DATA_GROUP}; + if (isnull($data_group) and !$default_data_groups) { + $default_data_groups = load_data_groups($verbose); + } + + my $rownum = $row->{ROWNUM}; + my $job_type = $row->{JOB_TYPE}; + my $option_mask= $row->{OPTION_MASK}; + + my $dateobs_begin; + my $dateobs_end; + if (!iszero($mjd_min)) { + $dateobs_begin = mjd_to_dateobs($mjd_min); + } + if (!iszero($mjd_max)) { + $dateobs_end = mjd_to_dateobs($mjd_max); + } + if (isnull($tess_id)) { + $tess_id = undef; + } + if (isnull($data_group)) { + $data_group = undef; + } + if (isnull($filter)) { + $filter = undef; + } + + if ($req_type eq "bycoord") { + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $results = lookup_bycoord($ipprc, $row, $imagedb, $stage, $tess_id, $component, 0, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + return $results; + } + + # for compatability with the lower level functions that haven't been converted to take a single row + my $rowList = [$row]; + + if (($req_type eq "byid") and ($stage eq "diff")) { + # lookups of all of the information for diff images requires a two level lookup to + # get the exposure information. Switching the req_type allows us to keep that code + # in one place + $req_type = "bydiff"; + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $stage, $verbose); + return $results; + } + + my ($release_name, $survey) = get_release_info($row); + + if ($req_type eq "bydiff") { + # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or + # the image from the diffRun + my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $stage, $verbose); + if (!$results) { + return undef; + } + if ($stage eq "diff") { + # lookup_diff has done all of the work + return $results; + } + + my $image = $results->[0]; + if ($stage eq "raw") { + $req_type = "byid"; + $id = $image->{exp_id}; + return undef if !$id; + # fall through and lookup byid + } elsif ($stage eq "chip") { + $req_type = "byid"; + $id = $image->{chip_id}; + return undef if !$id; + # fall through and lookup byid + } elsif ($stage eq "warp") { + $req_type = "byid"; + $id = $image->{warp_id}; + $component = $image->{skycell_id}; + return undef if !$id; + # fall through and lookup by warp_id + } elsif ($stage eq "stack") { + $req_type = "byid"; + $id = $image->{stack_id}; + return undef if !$id; + # fall though and lookup by stack_id + } else { + # This is checked this elsewhere? + print STDERR "Error: $stage is an unknown image type\n"; + return undef; + } + } elsif ($req_type eq "byskycell") { + if (($stage eq "raw") or ($stage eq "chip")) { + print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n"; + return undef; + } + if (!$tess_id or !$component) { + print STDERR "component and tess_id are required for REQ_TYPE byskycell\n"; + return undef; + } + } + + my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $stage, $id, $tess_id, $component, + 0, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + undef, $release_name, $survey, + $verbose); return $results; @@ -193,4 +330,7 @@ my $mjd_min = shift; my $mjd_max = shift; + my $selectedAstrom = shift; + my $release_name = shift; + my $survey = shift; my $verbose = shift; @@ -217,6 +357,18 @@ my $magic_arg = $need_magic ? " -destreaked" : ""; + my $row = $rowList->[0]; + # all rows have the same center type - my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + my $skycenter = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + + my $use_releasetool = 0; + my $release_args; + if (($req_type eq 'byexp' and $stage ne 'raw') or ($req_type eq 'byskycell' and $stage ne 'diff')) { + unless (isnull($release_name) and isnull($survey)) { + $use_releasetool = 1; + $release_args = " -release_name $release_name" if $release_name; + $release_args .= " -surveyName $survey" if $survey; + } + } my $component_args; @@ -249,15 +401,21 @@ # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then # choose the chips containing the center pixel by calling selectComponets() below - if ($component or $use_imfile_id or !$skycenter) { - $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb"; - if ($component and $component ne 'all') { - $class_id = $component; - $component_args = " -class_id $class_id"; - } - } else { - $command = "$chiptool -listrun -pstamp_order -dbname $imagedb"; + if ($use_releasetool) { + $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args"; $choose_components = 1; - } - $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id"; + } else { + $command = "$chiptool -pstamp_order -dbname $imagedb"; + if ($component or $use_imfile_id or !$skycenter) { + $command .= " -processedimfile"; + if ($component and $component ne 'all') { + $class_id = $component; + $component_args = " -class_id $class_id"; + } + } else { + $command .= " -listrun"; + $choose_components = 1; + } + $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id"; + } # XXX: perhaps we should stop resolving images to this level here and do it at job run time. # With the mdc file it has all of the information that it needs. @@ -273,16 +431,22 @@ $want_astrom = 1; $set_class_id = 1; - } elsif ($stage eq "warp") { - if ($component or $use_imfile_id or !$skycenter) { - $command = "$warptool -warped -pstamp_order -dbname $imagedb"; - if ($component and $component ne 'all') { - $skycell_id = $component; - $component_args = " -skycell_id $skycell_id"; - } - } else { - $command = "$warptool -listrun -pstamp_order -dbname $imagedb"; + } elsif ($stage eq 'warp') { + if ($use_releasetool) { + $command = "$releasetool -dbname $imagedb -priority_order -listrelexp $release_args"; $choose_components = 1; - } - $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id"; + } else { + $command = "$warptool -pstamp_order -dbname $imagedb"; + if ($component or $use_imfile_id or !$skycenter) { + $command .= " -warped"; + if ($component and $component ne 'all') { + $skycell_id = $component; + $component_args = " -skycell_id $skycell_id"; + } + } else { + $command = " -listrun"; + $choose_components = 1; + } + $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id"; + } $image_name = "PSWARP.OUTPUT"; $mask_name = "PSWARP.OUTPUT.MASK"; @@ -292,4 +456,5 @@ $base_name = "path_base"; # name of the field for the warptool output } elsif ($stage eq "diff") { + # XXX: is this path ever used for diff images anymore? if ($component or $use_imfile_id or !$skycenter) { $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb"; @@ -312,8 +477,14 @@ } elsif ($stage eq "stack") { $skycell_id = $component; - $command = "$stacktool -sumskyfile -dbname $imagedb"; - # only consider stacks in full state. - $command .= " -state full"; - $id_opt = "-stack_id"; + if ($use_releasetool) { + $command = "$releasetool -listrelstack -dbname $imagedb $release_args"; + $base_name = 'stack_path_base'; + } else { + $command = "$stacktool -sumskyfile -dbname $imagedb"; + # only consider stacks in full state. + $command .= " -state full"; + $id_opt = "-stack_id"; + $base_name = 'path_base'; + } $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne 'all'; @@ -329,10 +500,9 @@ # this is wrong but gets the right answer. Need to figure out how to find the # rule properly - #$cmf_name = "PSPHOT.OUTPUT"; # this puts .fpa. in the name $cmf_name = "PSWARP.OUTPUT.SOURCES"; + # $cmf_name = "PSPHOT.OUTPUT"; # this puts .fpa. in the name $psf_name = "PPSTACK.TARGET.PSF"; - $base_name = "path_base"; } else { - die "Unknown IMG_TYPe supplied: $stage"; + die "Unknown IMG_TYPE supplied: $stage"; } @@ -377,5 +547,6 @@ if (!isnull($data_group)) { $command .= " -data_group $data_group"; - } elsif ($req_type eq 'byskycell') { + } elsif (!$use_releasetool and $req_type eq 'byskycell') { + # XXX: Why am I using default data_groups only for byskycell requests? $command .= $default_data_groups; } @@ -392,11 +563,16 @@ $images = filterRuns($stage, $need_magic, $images, $inverse, $verbose); } + if ($choose_components) { # the list of "images" is actually a list of "Runs" # match the coords in the rows to the components in the runs # returns an actual list of images - $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose); + if ($skycenter) { + $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose); + } else { + $images = selectComponentsByName($ipprc, $imagedb, $req_type, $stage, $component, $rowList, $images, $verbose); + } } else { - # put index for each of the rows into all of the the images + # put index for each of the rows into all of the the selected images setRowRefs($rowList, $images); } @@ -462,5 +638,12 @@ # dquery wants these set $out->{magicked} = 0; - $out->{data_state} = $out->{state}; + if ($use_releasetool) { + if ($image->{skycal_path_base}) { + $out->{astrom} = $image->{skycal_path_base} . ".cmf"; + } + $out->{data_state} = $out->{state} = $out->{stack_state}; + } else { + $out->{data_state} = $out->{state}; + } } $out->{stage_id} = $stage_id; @@ -470,5 +653,8 @@ my $mask_base; if ($want_astrom) { - if (! defined $out->{astrom}) { + if ($selectedAstrom) { + $out->{astrom} = $selectedAstrom; + $out->{cam_path_base} = basename($selectedAstrom); + } elsif (! defined $out->{astrom}) { if (! find_astrometry($ipprc, $imagedb, $out, $verbose)) { print STDERR "failed to find astrometry for $stage $stage_id\n"; @@ -505,5 +691,5 @@ my $byid = shift; my $option_mask = shift; - my $img_type = shift; + my $stage = shift; my $verbose = shift; @@ -573,6 +759,6 @@ # we have a stack - stack diff (well it might be stack - warp....) # but at any rate we only handle image type diff and stack - if (($img_type ne "diff") and ($img_type ne "stack")) { - print STDERR "lookup_diff: cannot lookup IMG_TYPE $img_type bydiff from a stack stack diff run\n"; + if (($stage ne "diff") and ($stage ne "stack")) { + print STDERR "lookup_diff: cannot lookup IMG_TYPE $stage bydiff from a stack stack diff run\n"; setErrorCodes($rowList, $PSTAMP_INVALID_REQUEST); # all images will be the same so we can stop @@ -625,5 +811,5 @@ } - if ($img_type eq "diff") { + if ($stage eq "diff") { my @imageList = ($image); @@ -656,7 +842,7 @@ sub lookup_bycoord { my $ipprc = shift; - my $rowList = shift; + my $row = shift; my $imagedb = shift; - my $img_type = shift; + my $stage = shift; my $tess_id = shift; my $component = shift; @@ -671,11 +857,16 @@ my $mjd_min = shift; my $mjd_max = shift; + my $astrom = shift; my $verbose = shift; + my ($release_name, $survey) = get_release_info($row); + + my $rowList = [$row]; + my $results = (); - if (($img_type eq "raw") or ($img_type eq "chip")) { - - my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $img_type, - $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose); + if (($stage eq "raw") or ($stage eq "chip")) { + + my $chips = lookup_by_cam_id_and_coords($ipprc, $imagedb, $stage, + $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose, $release_name, $survey, $row->{FWHM_MIN}, $row->{FWHM_MAX}); if (!$chips or scalar @$chips == 0) { @@ -684,7 +875,9 @@ foreach my $chip (@$chips) { next if $component and ($chip->{component} ne $component); - my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $chip->{id}, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $stage, $chip->{id}, $tess_id, $chip->{component}, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + $chip->{astrom}, undef, undef, + $verbose); next if !$these_results; @@ -695,6 +888,6 @@ } else { # this should have been checked elsewhere - die "unexpected image type $img_type" if ($img_type ne "warp") - and ($img_type ne "stack") and ($img_type ne "diff"); + die "unexpected image type $stage" if ($stage ne "warp") + and ($stage ne "stack") and ($stage ne "diff"); my $skycells = lookup_skycell_by_coords($ipprc, $tess_id, $component, $ra, $dec, $verbose); @@ -704,7 +897,9 @@ } else { foreach my $skycell (@$skycells) { - my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef, + my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $stage, undef, $skycell->{tess_id}, $skycell->{component}, $need_magic, - $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, $verbose); + $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $mjd_min, $mjd_max, + undef, $release_name, $survey, + $verbose); next if !$these_results; @@ -722,10 +917,10 @@ sub lookup_by_cam_id_and_coords { - my $ipprc = shift; - my $imagedb = shift; - my $img_type = shift; - my $ra = shift; - my $dec = shift; - my $need_magic = shift; + my $ipprc = shift; + my $imagedb = shift; + my $stage = shift; + my $ra = shift; + my $dec = shift; + my $need_magic = shift; my $dateobs_begin = shift; my $dateobs_end = shift; @@ -733,15 +928,32 @@ my $data_group = shift; my $verbose = shift; - - my $camruns; - - { - my $search_radius = 1.6; # XXX: this should be camera specific - + my $release_name = shift; + my $survey = shift; + my $fwhm_min = shift; + my $fwhm_max = shift; + + my $search_radius = 1.6; # XXX: this should be camera specific + + my $command; + my $using_camtool = 0; + if ($release_name or $survey) { + $command = "$releasetool -dbname $imagedb -listrelexp"; + if (!isnull($survey)) { + $command .= " -survey_name $survey"; + } + if (!isnull($release_name)) { + $command .= " -release_name $release_name"; + } else { + $command .= " -priority_order"; + } + if ($fwhm_min != 0) { + $command .= " -fwhm_min $fwhm_min"; + } + if ($fwhm_max != 0) { + $command .= " -fwhm_max $fwhm_max"; + } + } else { + $using_camtool = 1; my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order"; - $command .= " -ra $ra -decl $dec -radius $search_radius"; - $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; - $command .= " -dateobs_end $dateobs_end" if $dateobs_end ; - $command .= " -filter $filter" if $filter; # NOTE: we are applying the data_group to the camera run. # If we're looking for chip stage images there is no guarentee that @@ -755,28 +967,37 @@ } $command .= " -destreaked" if $need_magic; - - # run the tool and parse the output - $camruns = runToolAndParse($command, $verbose); - } - if (!$camruns) { + } + $command .= " -ra $ra -decl $dec -radius $search_radius"; + $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; + $command .= " -dateobs_end $dateobs_end" if $dateobs_end ; + $command .= " -filter $filter" if $filter; + + # run the tool and parse the output + my $results = runToolAndParse($command, $verbose); + if (!$results) { return undef; } my $components; my $last_exp_id = 0; - foreach my $camRun (@$camruns) { - my $cam_id = $camRun->{cam_id}; - my $exp_id = $camRun->{exp_id}; - + foreach my $result (@$results) { + my $cam_id = $result->{cam_id}; + my $exp_id = $result->{exp_id}; + + # go on if we already have a result for this exposure next if $exp_id eq $last_exp_id; - next if $camRun->{quality}; - next if $camRun->{fault}; - - updateCamRunCache($camRun); + next if $result->{quality}; + + if ($using_camtool) { + next if $result->{fault}; + } else { + $result->{path_base} = $result->{cam_path_base}; + } + updateCamRunCache($result); $last_exp_id = $exp_id; # XXX Use file rule - my $astrom = $camRun->{path_base} . ".smf"; + my $astrom = $result->{path_base} . ".smf"; my $astrom_resolved = $ipprc->file_resolve($astrom); next if !$astrom_resolved; @@ -832,5 +1053,5 @@ } - print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n"; + print "cam_id $cam_id exp_id $result->{exp_id} chip_id $result->{chip_id} $class_id\n"; # build the hash to return @@ -840,19 +1061,20 @@ # problem my $comp = { - exp_id => $camRun->{exp_id}, - exp_name => $camRun->{exp_name}, - chip_id => $camRun->{chip_id}, - cam_id => $camRun->{cam_id}, + exp_id => $result->{exp_id}, + exp_name => $result->{exp_name}, + chip_id => $result->{chip_id}, + cam_id => $result->{cam_id}, class_id => $class_id, - component => $class_id + component => $class_id, + astrom => $astrom }; - if ($img_type eq "chip") { - $comp->{id} = $camRun->{chip_id}; - $comp->{state} = $camRun->{chip_state}; - $comp->{magicked} = $camRun->{chip_magicked}; - } else { - $comp->{id} = $camRun->{exp_id}; + if ($stage eq "chip") { + $comp->{id} = $result->{chip_id}; + $comp->{state} = $result->{chip_state}; + $comp->{magicked} = 0; + } else { + $comp->{id} = $result->{exp_id}; $comp->{state} = 'full'; - $comp->{magicked} = $camRun->{raw_magicked}; + $comp->{magicked} = $0; } push @$components, $comp; @@ -1281,4 +1503,67 @@ } +sub selectComponentsByName { + my $ipprc = shift; + my $imagedb = shift; + my $req_type = shift; + my $stage = shift; + my $component = shift; + my $rowList = shift; + my $runList = shift; + my $verbose = shift; + my $results = []; + + $component = "" if $component and lc($component) eq 'all'; + + # all runs match all rows so they can share a common row_index + # XXX: I'm not sure we ever actually get here with more than one + # row + my $numRows = scalar @$rowList; + my $row_index = []; + for (my $i=0; $i < $numRows; $i++) { + push @$row_index, $i; + } + + if (($req_type eq "byid") or ($req_type eq "byexp")) { + my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", ""); + foreach my $run (@$runList) { + # now find the images for this run + #foreach my $c (keys %components) { + my $command; + my $stage_id; + if ($stage eq 'raw') { + $stage_id = $run->{exp_id}; + $command = "$regtool -processedimfile -exp_id $stage_id"; + $command .= " -class_id $component" if $component; + } elsif ($stage eq 'chip') { + $stage_id = $run->{chip_id}; + $command = "$chiptool -processedimfile -chip_id $stage_id"; + $command .= " -class_id $component" if $component; + } elsif ($stage eq 'warp') { + $stage_id = $run->{warp_id}; + $command = "$warptool -warped -warp_id $stage_id"; + $command .= " -skycell_id $component" if $component; + } + $command .= " -dbname $imagedb"; + my $images = runToolAndParse($command, $verbose); + if (!defined $images) { + print "No $component found for $stage ${stage}_id $stage_id\n"; + next; + } + foreach my $image (@$images) { + $image->{row_index} = $row_index; + if (($stage eq "raw") or ($stage eq 'chip')) { + $image->{cam_id} = $run->{cam_id}; + $image->{cam_path_base} = $run->{cam_path_base}; + } + push @$results, $image; + } + } + } else { + &my_die ("selectComponentsByName not supported for REQ_TYPE: $req_type\n", $PS_EXIT_PROG_ERROR); + } + return $results; +} + sub filterRuns { my $stage = shift; @@ -1407,4 +1692,6 @@ $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); + $releasetool = can_run('releasetool') or (warn "Can't find releasetool" + and $missing_tools = 1); $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1); $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); @@ -1462,4 +1749,20 @@ } +sub get_release_info { + my $row = shift; + + my $survey; + my $release_name = $row->{IPP_RELEASE}; + if (isnull($release_name)) { + $release_name = ""; + # no release check for survey + $survey = $row->{SURVEY_NAME}; + } else { + $survey = ""; + } + + return ($release_name, $survey); +} + sub my_die { Index: /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 35412) +++ /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 35413) @@ -52,4 +52,5 @@ $PSTAMP_NO_OVERLAP $PSTAMP_NOT_AUTHORIZED + $PSTAMP_UNKNOWN_PROJECT ); our %EXPORT_TAGS = (standard => [@EXPORT_OK]); @@ -101,4 +102,5 @@ our $PSTAMP_NO_OVERLAP = 28; our $PSTAMP_NOT_AUTHORIZED = 29; +our $PSTAMP_UNKNOWN_PROJECT = 22; @@ -167,5 +169,5 @@ my $fields_output; { - my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USERNAME EMAIL"; + my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -187,16 +189,12 @@ if ($extver > 1) { $header{ACTION} = $action; - $header{USER} = $username; $header{EMAIL} = $email; } else { $header{ACTION} = $action = "PROCESS"; - $header{USERNAME} = 'null'; $header{EMAIL} = 'null'; } - if ($action eq "LIST") { - return (\%header, undef); - } elsif ($action ne "PROCESS") { - die "unexpected request ACTION found: $action in $request_file_name"; + if ($action ne "PROCESS" and $action ne 'PREVIEW') { + die "\nunexpected request ACTION found: $action in $request_file_name"; } Index: /branches/eam_branches/ipp-20130307/ippMonitor/raw/czartool_getplot.php =================================================================== --- /branches/eam_branches/ipp-20130307/ippMonitor/raw/czartool_getplot.php (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippMonitor/raw/czartool_getplot.php (revision 35413) @@ -38,5 +38,8 @@ else if ($type=="hp_s") $filePath = "$path/heathers_plots/skycal.png"; - +else if ($type=="hp_cm") +$filePath = "$path/heathers_plots/cam_merged.png"; +else if ($type=="hp_sm") +$filePath = "$path/heathers_plots/skycal_merged.png"; $file = fopen ($filePath, "r"); Index: /branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php =================================================================== --- /branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php (revision 35413) @@ -10,5 +10,5 @@ menu('ipp.czar.dat', 'useful tools for the czar', 'ipp.css', $ID['link'], $ID['proj']); -echo "These images are generated every day at 7:00 HST (17:00 GMT)\n"; +echo "These images are generated every day at 7:00 HST (17:00 GMT) on ipp@ippc11\n"; // document body echo "\n"; @@ -23,4 +23,14 @@ echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + echo "
Camera MergedSkyCal Merged
\n"; Index: /branches/eam_branches/ipp-20130307/ippTasks/release.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/release.pro (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippTasks/release.pro (revision 35413) @@ -129,5 +129,5 @@ host anyhost - $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME + $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME --rebuild if ("$GROUP_TYPE" == "lap") $run = $run --lap_id $LAP_ID Index: /branches/eam_branches/ipp-20130307/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/survey.pro (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippTasks/survey.pro (revision 35413) @@ -19,4 +19,5 @@ book create SURVEY_SKYCAL book create SURVEY_LAPGROUP + book create SURVEY_RELEXP $haveSurveyBooks = TRUE end @@ -35,4 +36,5 @@ $SURVEY_SKYCAL_DB = 0 $SURVEY_LAPGROUP_DB = 0 +$SURVEY_RELEXP_DB = 0 $SURVEY_STATICSKYSINGLE_DB = 0 $SURVEY_EXEC = 120 @@ -83,4 +85,7 @@ active true end + task survey.relexp + active true + end end @@ -126,4 +131,7 @@ end task survey.lapgroup + active false + end + task survey.relexp active false end @@ -533,4 +541,31 @@ end book listbook SURVEY_LAPGROUP +end + +macro survey.add.relexp + if ($0 != 3) + echo "USAGE: survey.add.relexp (label) (releasename)" + break + end + book newpage SURVEY_RELEXP $1 + book setword SURVEY_RELEXP $1 LABEL $1 + book setword SURVEY_RELEXP $1 RELEASE_NAME $2 + book setword SURVEY_RELEXP $1 STATE PENDING +end + +macro survey.del.relexp + if ($0 != 2) + echo "USAGE: survey.del.relexp (label)" + break + end + book delpage SURVEY_RELEXP $1 +end + +macro survey.show.relexp + if ($0 != 1) + echo "USAGE: survey.show.relexp" + break + end + book listbook SURVEY_RELEXP end @@ -1626,6 +1661,6 @@ # Select different database - $SURVEY_LAPGROUP ++ - if ($SURVEY_LAPGROUP >= $DB:n) set SURVEY_LAPGROUP = 0 + $SURVEY_LAPGROUP_DB ++ + if ($SURVEY_LAPGROUP_DB >= $DB:n) set SURVEY_LAPGROUP_DB = 0 end @@ -1633,5 +1668,5 @@ book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID - # For now the list of filters + # For now the list of filters is hardcoded here $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000 @@ -1666,2 +1701,71 @@ end end + +task survey.relexp + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.relexp.log + stderr $LOGDIR/survey.relexp.log + + task.exec + book npages SURVEY_RELEXP -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_RELEXP 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_RELEXP $i -var label + book setword SURVEY_RELEXP $label STATE NEW + end + book getpage SURVEY_RELEXP 0 -var label -key STATE NEW + + # Select different database + $SURVEY_RELEXP_DB ++ + if ($SURVEY_RELEXP_DB >= $DB:n) set SURVEY_RELEXP_DB = 0 + end + + book setword SURVEY_RELEXP $label STATE DONE + book getword SURVEY_RELEXP $label RELEASE_NAME -var RELEASE_NAME + + $run = releasetool -definerelexp -label $label -release_name $RELEASE_NAME -set_state processed + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_RELEXP_DB + option $DB:$SURVEY_RELEXP_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/ipp-20130307/ippconfig/dvo.photcodes =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/dvo.photcodes (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/dvo.photcodes (revision 35413) @@ -165,4 +165,5 @@ 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 nan 0.000 2.000 0.600 0x0000 0x0000 0x0000 0x0000 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 nan 0.000 2.000 0.600 0x0000 0x0000 0x0000 0x0000 + 3006 SYNTH.w ref 0.000 0.000 0.000 - - 0.0000 0 9 nan 0.000 2.000 0.600 0x0000 0x0000 0x0000 0x0000 100 MEGACAM.u.00 dep 25.280 -0.350 0.000 - - 0.0000 0 - 0.000 0.000 1.000 0.000 0x0000 0x3888 0x0000 0x0000 @@ -836,64 +837,64 @@ 10475 GPC1.y.XY75 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 10476 GPC1.y.XY76 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10501 GPC1.w.XY01 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10502 GPC1.w.XY02 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10503 GPC1.w.XY03 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10504 GPC1.w.XY04 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10505 GPC1.w.XY05 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10506 GPC1.w.XY06 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10510 GPC1.w.XY10 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10511 GPC1.w.XY11 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10512 GPC1.w.XY12 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10513 GPC1.w.XY13 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10514 GPC1.w.XY14 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10515 GPC1.w.XY15 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10516 GPC1.w.XY16 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10517 GPC1.w.XY17 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10520 GPC1.w.XY20 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10521 GPC1.w.XY21 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10522 GPC1.w.XY22 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10523 GPC1.w.XY23 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10524 GPC1.w.XY24 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10525 GPC1.w.XY25 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10526 GPC1.w.XY26 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10527 GPC1.w.XY27 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10530 GPC1.w.XY30 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10531 GPC1.w.XY31 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10532 GPC1.w.XY32 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10533 GPC1.w.XY33 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10534 GPC1.w.XY34 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10535 GPC1.w.XY35 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10536 GPC1.w.XY36 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10537 GPC1.w.XY37 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10540 GPC1.w.XY40 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10541 GPC1.w.XY41 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10542 GPC1.w.XY42 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10543 GPC1.w.XY43 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10544 GPC1.w.XY44 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10545 GPC1.w.XY45 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10546 GPC1.w.XY46 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10547 GPC1.w.XY47 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10550 GPC1.w.XY50 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10551 GPC1.w.XY51 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10552 GPC1.w.XY52 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10553 GPC1.w.XY53 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10554 GPC1.w.XY54 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10555 GPC1.w.XY55 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10556 GPC1.w.XY56 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10557 GPC1.w.XY57 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10560 GPC1.w.XY60 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10561 GPC1.w.XY61 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10562 GPC1.w.XY62 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10563 GPC1.w.XY63 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10564 GPC1.w.XY64 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10565 GPC1.w.XY65 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10566 GPC1.w.XY66 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10567 GPC1.w.XY67 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10571 GPC1.w.XY71 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10572 GPC1.w.XY72 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10573 GPC1.w.XY73 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10574 GPC1.w.XY74 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10575 GPC1.w.XY75 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 - 10576 GPC1.w.XY76 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10501 GPC1.w.XY01 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10502 GPC1.w.XY02 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10503 GPC1.w.XY03 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10504 GPC1.w.XY04 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10505 GPC1.w.XY05 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10506 GPC1.w.XY06 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10510 GPC1.w.XY10 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10511 GPC1.w.XY11 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10512 GPC1.w.XY12 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10513 GPC1.w.XY13 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10514 GPC1.w.XY14 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10515 GPC1.w.XY15 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10516 GPC1.w.XY16 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10517 GPC1.w.XY17 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10520 GPC1.w.XY20 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10521 GPC1.w.XY21 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10522 GPC1.w.XY22 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10523 GPC1.w.XY23 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10524 GPC1.w.XY24 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10525 GPC1.w.XY25 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10526 GPC1.w.XY26 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10527 GPC1.w.XY27 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10530 GPC1.w.XY30 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10531 GPC1.w.XY31 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10532 GPC1.w.XY32 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10533 GPC1.w.XY33 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10534 GPC1.w.XY34 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10535 GPC1.w.XY35 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10536 GPC1.w.XY36 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10537 GPC1.w.XY37 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10540 GPC1.w.XY40 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10541 GPC1.w.XY41 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10542 GPC1.w.XY42 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10543 GPC1.w.XY43 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10544 GPC1.w.XY44 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10545 GPC1.w.XY45 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10546 GPC1.w.XY46 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10547 GPC1.w.XY47 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10550 GPC1.w.XY50 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10551 GPC1.w.XY51 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10552 GPC1.w.XY52 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10553 GPC1.w.XY53 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10554 GPC1.w.XY54 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10555 GPC1.w.XY55 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10556 GPC1.w.XY56 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10557 GPC1.w.XY57 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10560 GPC1.w.XY60 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10561 GPC1.w.XY61 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10562 GPC1.w.XY62 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10563 GPC1.w.XY63 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10564 GPC1.w.XY64 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10565 GPC1.w.XY65 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10566 GPC1.w.XY66 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10567 GPC1.w.XY67 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10571 GPC1.w.XY71 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10572 GPC1.w.XY72 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10573 GPC1.w.XY73 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10574 GPC1.w.XY74 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10575 GPC1.w.XY75 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10576 GPC1.w.XY76 dep 26.060 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 11000 GPC1.g.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 11100 GPC1.r.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 Index: /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config (revision 35413) @@ -1,3 +1,4 @@ PSF.MODEL STR PS_MODEL_PS1_V1 # Model for PSF generation +BACKGROUND.MODEL BOOL F ZP.AIRMASS METADATA # Airmass terms by filter @@ -24,4 +25,6 @@ PSF.INPUT.MAX F32 6.0 # never really want >6 pixels in refstack PSF.INPUT.CLIP.NSIGMA F32 1.0 # sample typically on rising side of distribution (set even smaller?) + PSF.INPUT.THRESH F32 0.0 + PSF.INPUT.ASYMMETRY F32 0.0 END @@ -30,6 +33,8 @@ OUTPUT.LOGFLUX BOOL FALSE OUTPUT.REPLICATE BOOL TRUE - PSF.INPUT.MAX F32 7.5 + PSF.INPUT.MAX F32 10.0 PSF.INPUT.CLIP.NSIGMA F32 1.5 + PSF.INPUT.THRESH F32 5.0 + PSF.INPUT.ASYMMETRY F32 0.2 END @@ -40,6 +45,8 @@ OUTPUT.NOCOMP BOOL TRUE OUTPUT.LOGFLUX BOOL FALSE - PSF.INPUT.MAX F32 7.5 # since not convolved limit, relax/tune? + PSF.INPUT.MAX F32 10.0 # since not convolved limit, relax/tune? PSF.INPUT.CLIP.NSIGMA F32 1.5 + PSF.INPUT.THRESH F32 5.0 + PSF.INPUT.ASYMMETRY F32 0.2 END @@ -52,6 +59,8 @@ OUTPUT.NOCOMP BOOL FALSE OUTPUT.LOGFLUX BOOL TRUE - PSF.INPUT.MAX F32 7.5 # input cut more restrictive now 12->7.5 pix, limit target PSF in convolved stacks + PSF.INPUT.MAX F32 10.0 # input cut more restrictive now 12->10 pix, limit target PSF in convolved stacks PSF.INPUT.CLIP.NSIGMA F32 1.5 # additional cut if majority of inputs much smaller than INPUT.MAX + PSF.INPUT.THRESH F32 5.0 + PSF.INPUT.ASYMMETRY F32 0.2 END Index: /branches/eam_branches/ipp-20130307/ippconfig/isp/dvo.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/isp/dvo.config (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/isp/dvo.config (revision 35413) @@ -1,5 +1,6 @@ # location of DVO database tables -CATDIR /data/alala/eugene/isp/catdir +#CATDIR /data/alala/eugene/isp/catdir +MOSAICNAME ISP-APOGEE # keywords used by DVO to interpret the headers Index: /branches/eam_branches/ipp-20130307/ippconfig/isp/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/isp/ppImage.config (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/isp/ppImage.config (revision 35413) @@ -623,2 +623,36 @@ END END + +PPIMAGE_BKGCONT METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels + MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels + BACKGROUND.CONTINUITY BOOL FALSE + PATTERN.CONTINUITY.WIDTH S32 2 +END + Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-mef.mdc (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-mef.mdc (revision 35413) @@ -118,7 +118,7 @@ PSWARP.VARIANCE INPUT @FILES CHIP VARIANCE PSWARP.MASK INPUT @FILES CHIP MASK -PSWARP.SKYCELL INPUT @FILES CHIP IMAGE -PSWARP.ASTROM INPUT @FILES CHIP CMF -PSWARP.BKGMODEL INPUT @FILES CHIP IMAGE +PSWARP.SKYCELL INPUT @FILES CHIP WCS +PSWARP.ASTROM INPUT @FILES CHIP WCS +PSWARP.BKGMODEL INPUT @FILES CHIP IMAGE ## files used by ppsub Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-simple.mdc (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-simple.mdc (revision 35413) @@ -88,6 +88,6 @@ PSWARP.VARIANCE INPUT @FILES FPA VARIANCE PSWARP.MASK INPUT @FILES FPA MASK -PSWARP.SKYCELL INPUT @FILES FPA IMAGE -PSWARP.ASTROM INPUT @FILES FPA CMF +PSWARP.SKYCELL INPUT @FILES FPA WCS +PSWARP.ASTROM INPUT @FILES FPA WCS PSWARP.BKGMODEL INPUT @FILES CHIP IMAGE Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-split.mdc (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/filerules-split.mdc (revision 35413) @@ -106,6 +106,8 @@ PSWARP.VARIANCE INPUT @FILES CHIP VARIANCE PSWARP.MASK INPUT @FILES CHIP MASK -PSWARP.SKYCELL INPUT @FILES FPA IMAGE -PSWARP.ASTROM INPUT @FILES CHIP CMF +PSWARP.SKYCELL INPUT @FILES CHIP WCS +PSWARP.ASTROM INPUT @FILES CHIP WCS +#PSWARP.SKYCELL INPUT @FILES CHIP CMF +#PSWARP.ASTROM INPUT @FILES CHIP CMF PSWARP.BKGMODEL INPUT @FILES CHIP IMAGE Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config (revision 35412) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config (revision 35413) @@ -92,4 +92,6 @@ PSF.INPUT.MAX F32 NAN PSF.INPUT.CLIP.NSIGMA F32 NAN +PSF.INPUT.THRESH F32 NAN +PSF.INPUT.ASYMMETRY F32 NAN TEMP.IMAGE STR conv.im.fits # Suffix for temporary convolved images Index: /branches/eam_branches/ipp-20130307/ppImage/src/ppImageParseCamera.c =================================================================== --- /branches/eam_branches/ipp-20130307/ppImage/src/ppImageParseCamera.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/ppImage/src/ppImageParseCamera.c (revision 35413) @@ -22,28 +22,4 @@ ppImageOptions *options = ppImageOptionsParse(config); - // parse the header to extract out whether this OTA has video cells. - // CZW: 2012-05-01 I'm somewhat ashamed of what needs to be done to get to the header. There has to be a simpler way, right? -/* pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest */ -/* pmChip *chip; // Chip from FPA */ -/* while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { */ -/* if (!chip->process || !chip->file_exists) { */ -/* continue; */ -/* } */ -/* /\* if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { *\/ */ -/* /\* continue; *\/ */ -/* /\* } *\/ */ -/* psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */ -/* if (psMetadataLookupStr(NULL,chip->hdu->header,"CELLMODE")) { */ -/* psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */ -/* char *Vptr = strchr(psMetadataLookupStr(NULL,chip->hdu->header,"CELLMODE"),'V'); */ -/* if (Vptr) { */ -/* options->hasVideo = true; */ -/* psWarning("VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); */ -/* } */ -/* } */ -/* } */ -/* pmFPAviewReset(view); */ -/* psFree(view); */ - // the following are defined from the argument list, if given, // otherwise they revert to the config information Index: /branches/eam_branches/ipp-20130307/ppStack/src/ppStackMatch.c =================================================================== --- /branches/eam_branches/ipp-20130307/ppStack/src/ppStackMatch.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/ppStack/src/ppStackMatch.c (revision 35413) @@ -8,4 +8,6 @@ // #define TESTING // Enable debugging output // #define TESTING_REUSE_CONV // Enable debugging for re-using convolved outputs from previous run + +// #define TESTING_CZW #ifdef TESTING_REUSE_CONV @@ -264,13 +266,13 @@ // Add the background into the target image psImage *bgImage = stackBackgroundModel(readout, config); // Image of background - psBinaryOp(fake->image, fake->image, "+", bgImage); + psBinaryOp(fake->image, fake->image, "+", bgImage); psFree(bgImage); #endif -#ifdef TESTING +#ifdef TESTING_CZW { pmHDU *hdu = pmHDUFromCell(readout->parent); psString name = NULL; - psStringAppend(&name, "fake_%03d.fits", index); + psStringAppend(&name, "/local/tmp/fake_%03d.fits", index); pmStackVisualPlotTestImage(fake->image, name); psFits *fits = psFitsOpen(name, "w"); @@ -282,9 +284,19 @@ pmHDU *hdu = pmHDUFromCell(readout->parent); psString name = NULL; - psStringAppend(&name, "real_%03d.fits", index); + psStringAppend(&name, "/local/tmp/real_%03d.fits", index); pmStackVisualPlotTestImage(readout->image, name); psFits *fits = psFitsOpen(name, "w"); psFree(name); psFitsWriteImage(fits, hdu->header, readout->image, 0, NULL); + psFitsClose(fits); + } + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/real_var_%03d.fits", index); + pmStackVisualPlotTestImage(readout->variance, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, readout->variance, 0, NULL); psFitsClose(fits); } @@ -355,9 +367,9 @@ -#ifdef TESTING +#ifdef TESTING_CZW { pmHDU *hdu = pmHDUFromCell(readout->parent); psString name = NULL; - psStringAppend(&name, "conv_%03d.fits", index); + psStringAppend(&name, "/local/tmp/conv_%03d.fits", index); pmStackVisualPlotTestImage(conv->image, name); psFits *fits = psFitsOpen(name, "w"); @@ -369,5 +381,15 @@ pmHDU *hdu = pmHDUFromCell(readout->parent); psString name = NULL; - psStringAppend(&name, "diff_%03d.fits", index); + psStringAppend(&name, "/local/tmp/conv_var_%03d.fits", index); + pmStackVisualPlotTestImage(conv->variance, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, conv->variance, 0, NULL); + psFitsClose(fits); + } + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/diff_%03d.fits", index); pmStackVisualPlotTestImage(fake->image, name); psFits *fits = psFitsOpen(name, "w"); @@ -510,10 +532,6 @@ } - if (!stackRenormaliseReadout(config, readout)) { - psFree(rng); - psFree(bg); - return false; - } - +#ifdef TESTING_CZW + // CZW begin check of weight before renormalize step // Measure the variance level for the weighting if (psMetadataLookupBool(NULL, recipe, "WEIGHTS")) { @@ -529,6 +547,31 @@ options->weightings->data.F32[index] = 1.0; } - psLogMsg("ppStack", PS_LOG_INFO, "Weighting for image %d is %f\n", - index, options->weightings->data.F32[index]); + psLogMsg("ppStack", PS_LOG_INFO, "Prenorm Weighting for image %d is %g covar %g median %g\n", + index, options->weightings->data.F32[index],psImageCovarianceFactor(readout->covariance), + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN)); + // CZW end +#endif + if (!stackRenormaliseReadout(config, readout)) { + psFree(rng); + psFree(bg); + return false; + } + + // Measure the variance level for the weighting + if (psMetadataLookupBool(NULL, recipe, "WEIGHTS")) { + if (!psImageBackground(bg, NULL, readout->variance, readout->mask, maskVal | maskBad, rng)) { + psError(PPSTACK_ERR_DATA, false, "Can't measure mean variance for image."); + psFree(rng); + psFree(bg); + return false; + } + options->weightings->data.F32[index] = 1.0 / (psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN) * + psImageCovarianceFactor(readout->covariance)); + } else { + options->weightings->data.F32[index] = 1.0; + } + psLogMsg("ppStack", PS_LOG_INFO, "Weighting for image %d is %g covar %g median %g\n", + index, options->weightings->data.F32[index],psImageCovarianceFactor(readout->covariance), + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN)); psFree(rng); Index: /branches/eam_branches/ipp-20130307/ppStack/src/ppStackPrepare.c =================================================================== --- /branches/eam_branches/ipp-20130307/ppStack/src/ppStackPrepare.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/ppStack/src/ppStackPrepare.c (revision 35413) @@ -1,3 +1,9 @@ # include "ppStack.h" + + +// Option to use the old FWHM rejection code. +# define NEW_REJECTION +// #define OLD_REJECTION + # define RE_PHOTOMETER 0 @@ -253,5 +259,6 @@ float maxFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.MAX"); // max allowed input fwhm float clipFWHMnSig = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.CLIP.NSIGMA"); // sigma clipping of inputs - + float threshFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.THRESH"); // FWHM size that we refuse to clip below + float asymmetryFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.ASYMMETRY"); // max bimodal asymmetry psString log = psStringCopy("Input seeing FWHMs:\n"); // Log message bool havePSFs = false; // Do we have any PSFs? @@ -286,5 +293,5 @@ options->inputSeeing->data.F32[i] = sumFWHM / (float)numFWHM; } - +#ifdef OLD_REJECTION // reject any input images which exceed the specified max FWHM if (isfinite(maxFWHM) && (options->inputSeeing->data.F32[i] > maxFWHM)) { @@ -292,5 +299,6 @@ psLogMsg("ppStack", PS_LOG_INFO, "PSF FWHM for image %d is too large (%f vs %f maxFWHM) --- rejected.", i, options->inputSeeing->data.F32[i], maxFWHM); } - + // End old rejection fixed limit. +#endif psStringAppend(&log, "Input %d: %f\n", i, options->inputSeeing->data.F32[i]); } @@ -300,4 +308,72 @@ psFree(log); +#ifdef NEW_REJECTION + // Do GMM test with two modes to decide where to put the break. + double Punimodal; + int m = 2; + psVector *modes = psVectorAlloc(num,PS_TYPE_F32); + psVector *means = psVectorAlloc(m,PS_TYPE_F32); + psVector *S = psVectorAlloc(m,PS_TYPE_F32); + psVector *pi = psVectorAlloc(m,PS_TYPE_F32); + psImage *P = psImageAlloc(m,num,PS_TYPE_F32); + float limit; + if (!psMM1DClass(options->inputSeeing, + options->inputSeeing->n, + modes,means, + S,pi,P, + 2, + &Punimodal)) { + // Handle error here + psFree(modes); + psFree(means); + psFree(S); + psFree(pi); + psFree(P); + return(false); + } + // fprintf(stderr,"means: %g %g\n",means->data.F32[0],means->data.F32[1]); + // fprintf(stderr,"sigma: %g %g \n",S->data.F32[0],S->data.F32[1]); + + // fprintf(stderr,"pi: %g %g\n",pi->data.F32[0],pi->data.F32[1]); + + // unwrittenGMMfunction(options->inputSeeing, options->inputMask, 0xff, + // &Punimodal, + // &m1,&s1,&pi1, + // &m2,&s2,&pi2); + + // Use Gaussian mixture model analysis of the FWHM distribution to decide an optional FWHM limit + if ((Punimodal > 0.5)|| // This distribution is best represented by a single mode + (num <= 4)) { // Or we have a small number of inputs, making this statistic poor. + limit = maxFWHM; + } + else { // This is a bimodal distribution + if ((fabs(pi->data.F32[0] - pi->data.F32[1]) < asymmetryFWHM)|| // However, both modes are equally populated + (pi->data.F32[1] > pi->data.F32[0])) { // Or the larger FWHM mode is more populated + limit = means->data.F32[1] + clipFWHMnSig * S->data.F32[1]; + } + else { // The smaller FWHM mode is more populated + limit = means->data.F32[0] + clipFWHMnSig * S->data.F32[0]; + } + } + if (limit > maxFWHM) { limit = maxFWHM; } // We should not be larger than our max + if (limit < threshFWHM) { limit = threshFWHM; } // Nor smaller than our min + + for (int i = 0; i < num; i++) { + if (options->inputSeeing->data.F32[i] > limit) { + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_PSF; + psLogMsg("ppStack", PS_LOG_INFO, + "PSF FWHM for image %d is too large (%f vs %f fwhmlimit) --- rejected", + i, options->inputSeeing->data.F32[i],limit); + } + } + psFree(means); + psFree(modes); + psFree(S); + psFree(pi); + psFree(P); + // End new rejection code +#endif + +#ifdef OLD_REJECTION // We should have the ability to filter the input list based on the seeing: // * reject above some max value and/or min value @@ -323,5 +399,6 @@ } } - + // End old rejection sigma clip +#endif // Generate target PSF if (options->convolve) { Index: /branches/eam_branches/ipp-20130307/ppStack/src/ppStackSources.c =================================================================== --- /branches/eam_branches/ipp-20130307/ppStack/src/ppStackSources.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/ppStack/src/ppStackSources.c (revision 35413) @@ -11,5 +11,7 @@ #endif -#ifdef TESTING +// #define TESTING_CZW + +#ifdef TESTING_CZW // Dump matches to a file static void dumpMatches(const char *filename, // File to which to dump @@ -28,7 +30,9 @@ psVectorInit(magErr, NAN); for (int j = 0; j < match->num; j++) { + if (match->mask) { if (match->mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) { - continue; - } + continue; + } + } int index = match->image->data.U32[j]; // Image index mag->data.F32[index] = match->mag->data.F32[j] - zp->data.F32[index]; @@ -39,5 +43,11 @@ } for (int j = 0; j < num; j++) { - fprintf(outMatches, "%f (%f) ", mag->data.F32[j], magErr->data.F32[j]); + fprintf(outMatches, "%f %f %f %f %f %f\t", + match->mag->data.F32[j], + zp->data.F32[j], + trans ? trans->data.F32[j] : NAN, + magErr->data.F32[j], + match->x->data.F32[j], + match->y->data.F32[j]); } fprintf(outMatches, "\n"); @@ -271,11 +281,14 @@ return false; } - -#ifdef TESTING + options->sources = psMemIncrRefCounter(matches); + psArray *matchedSources = psArrayAlloc(0); + +#ifdef TESTING_CZW dumpMatches("source_match.dat", num, matches, zp, NULL); #endif if (options->matchZPs) { - psVector *trans = pmSourceMatchRelphot(matches, zp, tol, iter1, starRej1, starSys1, + + psVector *trans = pmSourceMatchRelphot(matches, matchedSources, zp, tol, iter1, starRej1, starSys1, iter2, starRej2, starSys2, starLimit, transIter, transRej, transThresh); // Transparencies per image @@ -324,11 +337,13 @@ // m_inst_o - m_inst_i = zp[i] - zpTarget - c1 * airmassTarget - 2.5log(sumExpTime) - trans_i - +#ifdef TESTING_CZW + dumpMatches("source_match2.dat", num, matches, zp, trans); +#endif if (options->matchZPs) { options->norm = psVectorAlloc(num, PS_TYPE_F32); for (int i = 0; i < num; i++) { - if (inputMask->data.U8[i] || !isfinite(trans->data.F32[i])) { - continue; - } + if (!isfinite(trans->data.F32[i])) { + continue; + } psArray *sources = sourceLists->data[i]; // Sources of interest float magCorr = zp->data.F32[i] - trans->data.F32[i] - 2.5*log10(options->sumExposure) - airmassTerm * airmassTarget; @@ -341,18 +356,25 @@ options->norm->data.F32[i] = magCorr; psLogMsg("ppStack", PS_LOG_INFO, - "Applying scale correction to image %d: %f mag (%f)\n", - i, magCorr, trans->data.F32[i]); + "Applying scale correction to image %d: %f mag (%f) (with inputMask value %d)\n", + i, magCorr, trans->data.F32[i],inputMask->data.U8[i]); for (int j = 0; j < sources->n; j++) { pmSource *source = sources->data[j]; // Source of interest + source->psfMag += magCorr; + source->apMag += magCorr; + if (!source) { continue; } // XXX need to apply to apMag as well - source->psfMag += magCorr; - source->apMag += magCorr; + psTrace("ppStack",5,"Source corrections: %d %d %f %f %f %f %f %f %f %f\n",i,j, + source->peak->xf,source->peak->yf, + source->psfMag,source->psfMagErr, + airmassTerm,airmassTarget,2.5 * log10(options->sumExposure), + source->psfMag + magCorr); } } } + matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches if (zpExpNum == numGoodImages) { @@ -366,13 +388,11 @@ -#ifdef TESTING - dumpMatches("source_mags.dat", num, matches, zp, trans); -#endif psFree(trans); -#ifdef TESTING +#ifdef TESTING_CZW // Double check: all transparencies should be zero { psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches + psArray *matchedSources = psArrayAlloc(matches->n); if (!matches) { psError(PPSTACK_ERR_DATA, false, "Unable to match sources"); @@ -380,5 +400,5 @@ return false; } - psVector *trans = pmSourceMatchRelphot(matches, zp, tol, iter1, starRej1, starSys1, + psVector *trans = pmSourceMatchRelphot(matches, matchedSources, zp, tol, iter1, starRej1, starSys1, iter2, starRej2, starSys2, starLimit, transIter, transRej, transThresh); // Transparencies @@ -386,14 +406,17 @@ fprintf(stderr, "Transparency of image %d: %f\n", i, trans->data.F32[i]); } + psFree(trans); psFree(matches); - } -#endif - } - - psFree(zp); + psFree(matchedSources); + } +#endif + } + + psFree(zpExp); #ifdef ASTROMETRY + // CZW: This is off by default. // Position offsets { @@ -436,4 +459,5 @@ #if (defined TESTING && defined ASTROMETRY) + // CZW: This is off by default. // Double check: all offsets should be zero { @@ -458,40 +482,59 @@ if (options->photometry) { // Save best matches for future photometry - psArray *sourcesBest = options->sources = psArrayAllocEmpty(matches->n); - + psArray *sourcesBest = options->sources = psArrayAllocEmpty(matches->n); // XXX something of a hack: require at least 2 detections or the nominated fraction of the max possible //int minMatches = PS_MAX(2, fracMatch * num);// Minimum number of matches required //MEH - if many poor images, will hit oddity with num fraction. use numGoodImages instead - int minMatches = PS_MAX(2, fracMatch * numGoodImages);// Minimum number of matches required - - int good0 = 0; - for (int i = 0; i < num; i++) { - if (inputMask->data.U8[i]) { - continue; - } - good0 = i; - break; + int minMatches = PS_MAX(2, fracMatch * numGoodImages);// Minimum number of matches required + + int good0 = 0; + for (int i = 0; i < num; i++) { + if (inputMask->data.U8[i]) { + continue; } - - for (int i = 0; i < matches->n; i++) { - pmSourceMatch *match = matches->data[i]; // Match of interest - if (match->num < minMatches) { - continue; - } - - // We need to grab a single instance of this source: just take the first available - // MEH - if first available is a rejected image, then problem. so take first non-rejected instance above - //int image = match->image->data.S32[0]; // Index of image - //int index = match->index->data.S32[0]; // Index of source within image - int image = match->image->data.S32[good0]; // Index of image - int index = match->index->data.S32[good0]; // Index of source within image - psArray *sources = sourceLists->data[image]; // Sources for image - pmSource *source = sources->data[index]; // Source of interest - - psArrayAdd(sourcesBest, sourcesBest->n, source); - } - psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis", sourcesBest->n); - } - + good0 = i; + break; + } + + for (int i = 0; i < matches->n; i++) { + pmSourceMatch *match = matches->data[i]; // Match of interest + if (match->num < minMatches) { + continue; + } + + // We need to grab a single instance of this source: just take the first available + // MEH - if first available is a rejected image, then problem. so take first non-rejected instance above + // int image = match->image->data.S32[0]; // Index of image + // int index = match->index->data.S32[0]; // Index of source within image + int image = match->image->data.S32[good0]; // Index of image + int index = match->index->data.S32[good0]; // Index of source within image + psArray *sources = sourceLists->data[image]; // Sources for image + pmSource *source = sources->data[index]; // Source of interest + + // CZW: Accept the first source for all properties, except for the magnitudes. + // We have average magnitudes from relphot, so we should use those + pmSource *photsource = matchedSources->data[i]; + if ((photsource)&&(isfinite(photsource->psfMag))) { + // CZW: photsource is the average relphot mag, which is equal to + // (primed are averaged values for this discussion) + // Therefore, the correct magnitude for psfMag' is: + // psfMag' = (photsource - zpIN_j + magCorr) + // apMag - psfMag should be retained, so: + // apMag - psfMag = apMag' - psfMag' + // apMag' = apMag + psfMag' - psfMag + source->apMag = source->apMag + (photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image] - source->psfMag); + source->psfMag = photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image]; + + psTrace("ppStack", 5, "Best Source %d/%ld @ %f %f %f %f %f %d",i,sourcesBest->n,source->peak->xf,source->peak->yf, + source->psfMag, + photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image] , + source->psfMagErr,-1); + psArrayAdd(sourcesBest, sourcesBest->n, source); + } + } + psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis", sourcesBest->n); + + } + psFree(zp); psFree(matches); Index: /branches/eam_branches/ipp-20130307/ppStack/src/ppStackTarget.c =================================================================== --- /branches/eam_branches/ipp-20130307/ppStack/src/ppStackTarget.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/ppStack/src/ppStackTarget.c (revision 35413) @@ -60,5 +60,6 @@ } float xSource, ySource; // Coordinates of source - coordsFromSource(&xSource, &ySource, NULL, source); + float Smag; // magnitude of source + coordsFromSource(&xSource, &ySource, &Smag, source); coords->data.F64[0] = xSource; @@ -66,6 +67,6 @@ psVector *indices = psTreeAllWithin(tree, coords, exclusion); // Number within exclusion zone - psTrace("ppStack", 9, "Source at %.0lf,%.0lf has %ld sources in exclusion zone", - coords->data.F64[0], coords->data.F64[1], indices->n); + psTrace("ppStack", 5, "Filtering: Source at %.0lf %.0lf with mag %f has %ld sources in exclusion zone", + coords->data.F64[0], coords->data.F64[1], Smag, indices->n); if (indices->n == 1) { // Only itself inside the exclusion zone Index: /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceIO.c =================================================================== --- /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceIO.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceIO.c (revision 35413) @@ -980,4 +980,5 @@ return false; } + // if this is not TRUE, the output files only contain the psf measurements. bool XSRC_OUTPUT = psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_ANALYSIS"); @@ -1005,4 +1006,15 @@ hdu->header = psFitsReadHeader (NULL, file->fits); } + + // "WCS" is CMF without detected objects + if (file->type == PM_FPA_FILE_WCS) { + psTrace("psModules.objects", 6, "read CMF table from %s : %s : %s", file->filename, headname, dataname); + psFree (headname); + psFree (dataname); + psFree (deteffname); + break; + } + + // EXTDATA is the PSF data associated with this image header // we need to find the corresponding table EXTNAME. Index: /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.c =================================================================== --- /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.c (revision 35413) @@ -630,4 +630,5 @@ psVector *pmSourceMatchRelphot(const psArray *matches, // Array of matches + psArray *matchedSources, // Array of averaged sources const psVector *zp, // Zero points for each image (including airmass term) float tol, // Relative tolerance for convergence @@ -648,5 +649,6 @@ PS_ASSERT_VECTOR_TYPE(zp, PS_TYPE_F32, NULL); PS_ASSERT_FLOAT_LARGER_THAN(transClip, 0.0, NULL); - + PS_ASSERT_ARRAY_NON_NULL(matchedSources, NULL); + sys1 *= sys1; sys2 *= sys2; @@ -742,4 +744,10 @@ } + for (int i = 0; i < matches->n; i++) { + pmSource *source = pmSourceAlloc(); + source->psfMag = stars->data.F32[i]; + psArrayAdd(matchedSources,matchedSources->n,source); + } + psFree(photo); psFree(badImage); Index: /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.h =================================================================== --- /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.h (revision 35412) +++ /branches/eam_branches/ipp-20130307/psModules/src/objects/pmSourceMatch.h (revision 35413) @@ -74,4 +74,5 @@ /// Perform relative photometry to calibrate images psVector *pmSourceMatchRelphot(const psArray *matches, // Array of matches + psArray *matchedSources, // Array of average sources const psVector *zp, // Zero points for each image (including airmass term) float tol, // Relative tolerance for convergence Index: /branches/eam_branches/ipp-20130307/pswarp/doc/notes.20130406.txt =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/doc/notes.20130406.txt (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/doc/notes.20130406.txt (revision 35413) @@ -31,4 +31,47 @@ * sets file->fileLevel based on format, but slightly out of sync wrt pmFPAfileDefineFromArgs (NOTE: does not expect / examine any other file) - + +in pswarpLoop (major blocks): + + * find the needed recipe / filerules / etc + * pswarpIOChecksBefore() : generate the outputs? (isn't this done in pswarpDefine?) + * read input astrometry headers (also loads sources or not?) + * apply astrometry headers to input->fpa + * pswarpTransformReadout : transform pixels, etc + * pswarpPixelsLit : measure filling & masks stats + * modify the covariance + * modify the ZP & trimsec (seems like these should be encapsulated) + * update headers + * measure the PSF + * measure pixel stats on output + * add MD5 to header + + + +---- + +pswarp issues: + +1) operations need to loop over the output astrometry / targets for fpa/chip/cell levels + a) I replaced pswarpDefineSkycell with an update to pmFPAfileDefineXX which does the looping + b) is it identical in behavior to pswarpDefineSkycell? + * YES : I can drop pswarpDefineSkycell + +2) how do we load the SOURCES? + + * I need to load the headers for the astrometry, and I do this by + setting the type of the input astrometry to be WCS. is this the + same as CMF, but the latter reads the sources?? + + * by changing the filerules entry CMF <-> WCS pswarpDefine.c either does or does not read the sources + + (for testing, I want to use a skycell image or template to define the astrometry; this would not have sources to be read) + +3) does the I/O loop in pswarpLoop (pswarpIOChecksBefore) do anything useful? + ** I don't think so... this operation is done in pswarpDefine. + +4) how do I determine the overlaps correctly? (I'd like to do this before I load sources or pixels) + + ** pswarpMapSetLocalModel does not seem to depend on astrometry model style (GOOD!) + Index: /branches/eam_branches/ipp-20130307/pswarp/src/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/src/Makefile.am (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/src/Makefile.am (revision 35413) @@ -35,4 +35,5 @@ pswarpDefine.c \ pswarpDefineBackground.c \ + pswarpDefineSkycell.c \ pswarpErrorCodes.c \ pswarpMapGrid.c \ Index: /branches/eam_branches/ipp-20130307/pswarp/src/pswarp.c =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/src/pswarp.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/src/pswarp.c (revision 35413) @@ -39,5 +39,8 @@ } - // load the skycell layout information + // load the skycell layout information: + // read the output astrometry description + // generate the output pixels + // convert header astrometry to pmFPA astrometry structures if (!pswarpDefine(config)) { pswarpCleanup(config, statsFile); Index: /branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefine.c =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefine.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefine.c (revision 35413) @@ -38,51 +38,90 @@ } - // open the full skycell file; no need to defer different depths. only load the header data - pmFPAview *view = pmFPAviewAlloc (0); - pmFPAfileOpen (skycell, view, config); + // Read the output astrometry + // XXX this section loops over the input skycell to load the header info + { + pmFPAview *view = pmFPAviewAlloc(0); - // Read header and create target - { - if (!pmFPAReadHeaderSet(skycell->fpa, skycell->fits, config)) { - psError(psErrorCodeLast(), false, "Unable to read headers for skycell."); - psFree(view); - return false; - } - view->chip = 0; - view->cell = 0; - view->readout = 0; - pmCell *source = pmFPAfileThisCell(config->files, view, "PSWARP.SKYCELL"); ///< Source cell - pmHDU *hdu = pmHDUFromCell(source); ///< HDU for source - if (!hdu || !hdu->header) { - psError(PM_ERR_PROG, false, "Unable to find header for sky cell."); - psFree(view); - return false; - } - int numCols = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); ///< Number of columns - int numRows = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); ///< Number of rows - if ((numCols == 0) || (numRows == 0)) { - psError(PSWARP_ERR_DATA, false, "skycell has invalid dimensions %d x %d", numCols, numRows); - psFree(view); - return false; - } + pmChip *chip; + if (!pmFPAfileRead (skycell, view, config)) { + psError(PS_ERR_IO, false, "failed READ at FPA %s", skycell->name); + psFree(view); + return false; + } + while ((chip = pmFPAviewNextChip (view, skycell->fpa, 1)) != NULL) { + psTrace ("pswarp", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (!chip->process || !chip->file_exists) { continue; } + if (!pmFPAfileRead (skycell, view, config)) { + psError(psErrorCodeLast(), false, "failed READ at CHIP %s", skycell->name); + psFree(view); + return false; + } + pmCell *cell; + while ((cell = pmFPAviewNextCell (view, skycell->fpa, 1)) != NULL) { + psTrace ("pswarp", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!cell->process || !cell->file_exists) { continue; } + if (!pmFPAfileRead (skycell, view, config)) { + psError(psErrorCodeLast(), false, "failed READ at CELL %s", skycell->name); + psFree(view); + return false; + } + if (!pmFPAfileClose(skycell, view)) { + psError(psErrorCodeLast(), false, "failed CLOSE at CELL %s", skycell->name); + psFree (view); + return false; + } - pmCell *target = pmFPAviewThisCell(view, output->fpa); ///< Target cell - pmReadout *readout = pmReadoutAlloc(target); ///< Target readout - readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); - psImageInit(readout->image, NAN); - psFree(readout); // Drop reference - - bool status = false; - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.XBIN"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.YBIN"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.XSIZE"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.YSIZE"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.XPARITY"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.YPARITY"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.X0"); - psMetadataItemSupplement(&status, target->concepts, source->concepts, "CELL.Y0"); + // we've got the output astrom header + pmHDU *hdu = pmHDUFromCell(cell); ///< HDU for source + if (!hdu || !hdu->header) { + psError(PM_ERR_PROG, false, "Unable to find header for sky cell."); + psFree(view); + return false; + } + int numCols = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); ///< Number of columns + int numRows = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); ///< Number of rows + if ((numCols == 0) || (numRows == 0)) { + psError(PSWARP_ERR_DATA, false, "skycell has invalid dimensions %d x %d", numCols, numRows); + psFree(view); + return false; + } + + // XXX generate the output pixels + pmCell *target = pmFPAviewThisCell(view, output->fpa); ///< Target cell + pmReadout *readout = pmReadoutAlloc(target); ///< Target readout + readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(readout->image, NAN); + psFree(readout); // Drop reference + + bool status = false; + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.XBIN"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.YBIN"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.XSIZE"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.YSIZE"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.XPARITY"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.YPARITY"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.X0"); + psMetadataItemSupplement(&status, target->concepts, cell->concepts, "CELL.Y0"); + } + if (!pmFPAfileClose(skycell, view)) { + psError(psErrorCodeLast(), false, "failed CLOSE at CHIP %s", skycell->name); + psFree (view); + return false; + } + } + if (!pmFPAfileClose(skycell, view)) { + psError(psErrorCodeLast(), false, "failed CLOSE at FPA %s", skycell->name); + psFree (view); + return false; + } + psFree(view); } - // XXX this is not a sufficient test + // the section below converts the header astrometry info (in skycell) to the pmFPA + // astrometry structures, saving them on the output fpa structure. + + // XXX this block should be modified to loop over all chips + + pmFPAview *view = pmFPAviewAlloc (0); view->chip = 0; view->cell = 0; Index: /branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefineSkycell.c =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefineSkycell.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/src/pswarpDefineSkycell.c (revision 35413) @@ -73,15 +73,18 @@ psFitsClose(fits); - // We need to force the format for the skycell to be equivalent to SIMPLE. Determine - // the current format from the header; Determine camera if not specified already - // XXX EAM : this operation should be defined as a pmConfig function (pmConfigCopy?) - skyConfig = pmConfigAlloc(); - skyConfig->user = psMemIncrRefCounter(config->user); - skyConfig->system = psMemIncrRefCounter(config->system); + { // this bit is unique to pswarpDefineSkycell: - psFree (skyConfig->files); - skyConfig->files = psMemIncrRefCounter(config->files); - psFree (skyConfig->arguments); - skyConfig->arguments = psMemIncrRefCounter(config->arguments); + // We need to force the format for the skycell to be equivalent to SIMPLE. Determine + // the current format from the header; Determine camera if not specified already + // XXX EAM : this operation should be defined as a pmConfig function (pmConfigCopy?) + skyConfig = pmConfigAlloc(); + skyConfig->user = psMemIncrRefCounter(config->user); + skyConfig->system = psMemIncrRefCounter(config->system); + + psFree (skyConfig->files); + skyConfig->files = psMemIncrRefCounter(config->files); + psFree (skyConfig->arguments); + skyConfig->arguments = psMemIncrRefCounter(config->arguments); + } format = pmConfigCameraFormatFromHeader (NULL, NULL, NULL, skyConfig, phu, false); @@ -94,6 +97,6 @@ // Record the camera name of the skycell, so we can save its configuration - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "SKYCELL.CAMERA", 0, "Name of camera for skycell", - skyConfig->cameraName); + // XXX this can be put outside of the f + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "SKYCELL.CAMERA", 0, "Name of camera for skycell", skyConfig->cameraName); // build the template fpa, set up the basic view Index: /branches/eam_branches/ipp-20130307/pswarp/src/pswarpLoop.c =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/src/pswarpLoop.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/src/pswarpLoop.c (revision 35413) @@ -77,15 +77,18 @@ // Turn all skycell files on to generate them, and then turn them off for the loop over the input images // the input, which is in a different format. - { - pswarpFileActivation(config, detectorFiles, false); - pswarpFileActivation(config, photFiles, false); - pswarpFileActivation(config, independentFiles, false); - pswarpFileActivation(config, skycellFiles, true); - if (!pswarpIOChecksBefore(config)) { - psError(psErrorCodeLast(), false, "Unable to read files."); - goto DONE; - } - pswarpFileActivation(config, skycellFiles, false); - } + // XXX why is this being done here, doesn't it duplicate the work in pswarpDefine.c?? + pswarpFileActivation(config, detectorFiles, false); + pswarpFileActivation(config, photFiles, false); + pswarpFileActivation(config, independentFiles, false); + pswarpFileActivation(config, skycellFiles, false); + + // pswarpFileActivation(config, skycellFiles, true); + // if (0) { + // if (!pswarpIOChecksBefore(config)) { + // psError(psErrorCodeLast(), false, "Unable to read files."); + // goto DONE; + // } + // } + // Read the input astrometry @@ -164,4 +167,5 @@ } + // *** main transformation block pmChip *chip; while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { Index: /branches/eam_branches/ipp-20130307/pswarp/src/pswarpParseCamera.c =================================================================== --- /branches/eam_branches/ipp-20130307/pswarp/src/pswarpParseCamera.c (revision 35412) +++ /branches/eam_branches/ipp-20130307/pswarp/src/pswarpParseCamera.c (revision 35413) @@ -55,7 +55,4 @@ } - - - bool pswarpParseCamera(pmConfig *config) { @@ -70,7 +67,21 @@ } - pmFPAfile *astrom = defineInputFile(config, NULL, "PSWARP.ASTROM", "ASTROM", PM_FPA_FILE_CMF); + // XXX I should allow the astrometry input to be either WCS or CMF + // Define the input astrometry file(s) + bool status = false; + pmFPAfile *astrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM"); + if (!astrom) { + // look for the file on the RUN metadata + astrom = pmFPAfileDefineFromRun(&status, NULL, config, "PSWARP.ASTROM"); // File to return + if (!status) { + psLogMsg("pswarp", PS_LOG_INFO, "Failed to load file definition for %s", "PSWARP.ASTROM"); + } + } + if ((astrom->type != PM_FPA_FILE_CMF) && (astrom->type != PM_FPA_FILE_WCS)) { + psLogMsg("pswarp", PS_LOG_INFO, "%s is neither CMF nor WCS", "PSWARP.ASTROM"); + } psLogMsg("pswarp", PS_LOG_INFO, "Astrometry source: %s", astrom ? "supplied" : "header"); + // Define the input mask file(s) pmFPAfile *inMask = defineInputFile(config, input, "PSWARP.MASK", "MASK", PM_FPA_FILE_MASK); if (!inMask) { @@ -78,4 +89,5 @@ } + // Define the input variance file(s) pmFPAfile *inVariance = defineInputFile(config, input, "PSWARP.VARIANCE", "VARIANCE", PM_FPA_FILE_VARIANCE); @@ -91,7 +103,15 @@ // The input skycell is a required argument: it defines the output image - bool status = false; pmConfig *skyConfig = NULL; - pmFPAfile *skycell = pmFPAfileDefineNewConfig(&status, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL"); + pmFPAfile *skycell = NULL; + +# define USE_PSWARP_DEFINE_SKYCELL 0 +# if (USE_PSWARP_DEFINE_SKYCELL) + // XXX this function is no longer needed; drop + status = pswarpDefineSkycell(&skycell, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL"); +# else + skycell = pmFPAfileDefineNewConfig(&status, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL"); +# endif + if (!status || !skycell) { psError(psErrorCodeLast(), false, "Failed to build FPA from PSWARP.SKYCELL");