Index: /branches/eam_branches/eam_branch_20080618/ippScripts/.cvsignore =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/.cvsignore (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/.cvsignore (revision 22155) @@ -0,0 +1,6 @@ +Build +META.yml +Makefile +Makefile.PL +_build +blib Index: /branches/eam_branches/eam_branch_20080618/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/Build.PL (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/Build.PL (revision 22155) @@ -0,0 +1,78 @@ +use Module::Build; +# See perldoc Module::Build for details of how this works + +Module::Build->new( + module_name => 'ippscripts', + dist_version => '0.01', + author => 'Joshua Hoblitt ', + license => 'gpl', + create_makefile_pl => 'passthrough', + requires => { + 'Astro::FITS::CFITSIO' => 0, + 'Cache::File' => 0, + 'Carp' => 0, + 'Data::Dumper' => 0, + 'File::Basename' => 0, + 'File::Spec' => 0, + 'File::Temp' => 0, + 'Getopt::Long' => 0, + 'IPC::Cmd' => 0, + 'IPC::Run' => 0, + 'Math::Trig' => 0, + 'Pod::Usage' => 0, + 'PS::IPP::Config' => '1.01', + 'PS::IPP::Metadata::Config' => 0, + 'Statistics::Descriptive' => 0, + 'Storable' => 0, + 'Sys::Hostname' => 0, + }, + recommends => { + 'Test::Distribution' => '1.22', + }, + script_files => [qw( + scripts/ipp_datapath.pl + scripts/ipp_filename.pl + scripts/mdc2list.pl + scripts/register_imfile.pl + scripts/register_exp.pl + scripts/detrend_process_imfile.pl + scripts/detrend_process_exp.pl + scripts/detrend_stack.pl + scripts/detrend_norm_calc.pl + scripts/detrend_norm_apply.pl + scripts/detrend_norm_exp.pl + scripts/detrend_resid.pl + scripts/detrend_reject_imfile.pl + scripts/detrend_reject_exp.pl + scripts/chip_imfile.pl + scripts/camera_exp.pl + scripts/fake_imfile.pl + scripts/warp_overlap.pl + scripts/warp_skycell.pl + scripts/diff_skycell.pl + scripts/stack_skycell.pl + scripts/magic_tree.pl + scripts/ipp_inject_fileset.pl + scripts/ipp_inject_expname.pl + scripts/ipp_serial_inject.pl + scripts/ipp_serial_inject_split.pl + scripts/ipp_serial_inject_mosaic.pl + scripts/ipp_serial_register.pl + scripts/ipp_serial_detrend.pl + scripts/ipp_serial_chip.pl + scripts/ipp_serial_camera.pl + scripts/ipp_serial_warp.pl + scripts/ipp_serial_diff.pl + scripts/ipp_serial_stack.pl + scripts/ipp_serial_mops.pl + scripts/ipp_mops_translate.pl + scripts/ipp_simulation_data.pl + scripts/isp_trans.pl + scripts/ds9_cmf_regions.pl + scripts/ipp_detrend_combine.pl + scripts/summit_copy.pl + scripts/ipp_image_path.pl + )], + dist_abstract => 'Scripts for running the Pan-STARRS IPP', + dist_author => 'Paul Price ', +)->create_build_script; Index: /branches/eam_branches/eam_branch_20080618/ippScripts/Changes =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/Changes (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/Changes (revision 22155) @@ -0,0 +1,6 @@ +Revision history for Perl module ippScripts + +0.01 Thu Feb 23 13:55:57 2006 + - help scripts split out of the ippTools package + + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/LICENSE =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/LICENSE (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/LICENSE (revision 22155) @@ -0,0 +1,260 @@ +The General Public License (GPL) +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. 675 Mass Ave, +Cambridge, MA 02139, USA. Everyone is permitted to copy and distribute +verbatim copies of this license document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to most of +the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom +to distribute copies of free software (and charge for this service if you wish), that +you receive source code or can get it if you want it, that you can change the +software or use pieces of it in new free programs; and that you know you can do +these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for a +fee, you must give the recipients all the rights that you have. You must make +sure that they, too, receive or can get the source code. And you must show +them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) offer +you this license which gives you legal permission to copy, distribute and/or +modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced by +others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We wish +to avoid the danger that redistributors of a free program will individually obtain +patent licenses, in effect making the program proprietary. To prevent this, we +have made it clear that any patent must be licensed for everyone's free use or +not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +GNU GENERAL PUBLIC LICENSE +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND +MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or translated +into another language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is not +restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and appropriately +publish on each copy an appropriate copyright notice and disclaimer of warranty; +keep intact all the notices that refer to this License and to the absence of any +warranty; and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may at +your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such +modifications or work under the terms of Section 1 above, provided that you also +meet all of these conditions: + +a) You must cause the modified files to carry prominent notices stating that you +changed the files and the date of any change. + +b) You must cause any work that you distribute or publish, that in whole or in +part contains or is derived from the Program or any part thereof, to be licensed +as a whole at no charge to all third parties under the terms of this License. + +c) If the modified program normally reads commands interactively when run, you +must cause it, when started running for such interactive use in the most ordinary +way, to print or display an announcement including an appropriate copyright +notice and a notice that there is no warranty (or else, saying that you provide a +warranty) and that users may redistribute the program under these conditions, +and telling the user how to view a copy of this License. (Exception: if the +Program itself is interactive but does not normally print such an announcement, +your work based on the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, +and its terms, do not apply to those sections when you distribute them as +separate works. But when you distribute the same sections as part of a whole +which is a work based on the Program, the distribution of the whole must be on +the terms of this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your rights to +work written entirely by you; rather, the intent is to exercise the right to control +the distribution of derivative or collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and 2 +above provided that you also do one of the following: + +a) Accompany it with the complete corresponding machine-readable source +code, which must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange; or, + +b) Accompany it with a written offer, valid for at least three years, to give any +third party, for a charge no more than your cost of physically performing source +distribution, a complete machine-readable copy of the corresponding source +code, to be distributed under the terms of Sections 1 and 2 above on a medium +customarily used for software interchange; or, + +c) Accompany it with the information you received as to the offer to distribute +corresponding source code. (This alternative is allowed only for noncommercial +distribution and only if you received the program in object code or executable +form with such an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all the +source code for all modules it contains, plus any associated interface definition +files, plus the scripts used to control compilation and installation of the +executable. However, as a special exception, the source code distributed need +not include anything that is normally distributed (in either source or binary form) +with the major components (compiler, kernel, and so on) of the operating system +on which the executable runs, unless that component itself accompanies the +executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so long +as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not accept +this License. Therefore, by modifying or distributing the Program (or any work +based on the Program), you indicate your acceptance of this License to do so, +and all its terms and conditions for copying, distributing or modifying the +Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to copy, +distribute or modify the Program subject to these terms and conditions. You +may not impose any further restrictions on the recipients' exercise of the rights +granted herein. You are not responsible for enforcing compliance by third parties +to this License. + +7. If, as a consequence of a court judgment or allegation of patent infringement +or for any other reason (not limited to patent issues), conditions are imposed on +you (whether by court order, agreement or otherwise) that contradict the +conditions of this License, they do not excuse you from the conditions of this +License. If you cannot distribute so as to satisfy simultaneously your obligations +under this License and any other pertinent obligations, then as a consequence +you may not distribute the Program at all. For example, if a patent license would +not permit royalty-free redistribution of the Program by all those who receive +copies directly or indirectly through you, then the only way you could satisfy +both it and this License would be to refrain entirely from distribution of the +Program. + +If any portion of this section is held invalid or unenforceable under any particular +circumstance, the balance of the section is intended to apply and the section as +a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or other +property right claims or to contest validity of any such claims; this section has +the sole purpose of protecting the integrity of the free software distribution +system, which is implemented by public license practices. Many people have +made generous contributions to the wide range of software distributed through +that system in reliance on consistent application of that system; it is up to the +author/donor to decide if he or she is willing to distribute software through any +other system and a licensee cannot impose that choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain countries +either by patents or by copyrighted interfaces, the original copyright holder who +places the Program under this License may add an explicit geographical +distribution limitation excluding those countries, so that distribution is permitted +only in or among countries not thus excluded. In such case, this License +incorporates the limitation as if written in the body of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems or +concerns. + +Each version is given a distinguishing version number. If the Program specifies a +version number of this License which applies to it and "any later version", you +have the option of following the terms and conditions either of that version or of +any later version published by the Free Software Foundation. If the Program does +not specify a version number of this License, you may choose any version ever +published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of all +derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS +NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE +COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM +"AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR +IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE +ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED +TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY +WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM +(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY +OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS Index: /branches/eam_branches/eam_branch_20080618/ippScripts/MANIFEST =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/MANIFEST (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/MANIFEST (revision 22155) @@ -0,0 +1,38 @@ +Build.PL +Changes +LICENSE +MANIFEST +README +scripts/ipp_datapath.pl +scripts/ipp_filename.pl +scripts/mdc2list.pl +scripts/register_imfile.pl +scripts/register_exp.pl +scripts/detrend_process_imfile.pl +scripts/detrend_process_exp.pl +scripts/detrend_stack.pl +scripts/detrend_norm_calc.pl +scripts/detrend_norm_apply.pl +scripts/detrend_norm_exp.pl +scripts/detrend_resid.pl +scripts/detrend_reject_imfile.pl +scripts/detrend_reject_exp.pl +scripts/chip_imfile.pl +scripts/camera_exp.pl +scripts/warp_overlap.pl +scripts/warp_skycell.pl +scripts/diff_skycell.pl +scripts/stack_skycell.pl +scripts/ipp_serial_inject.pl +scripts/ipp_serial_inject_mosaic.pl +scripts/ipp_serial_register.pl +scripts/ipp_serial_detrend.pl +scripts/ipp_serial_chip.pl +scripts/ipp_serial_camera.pl +scripts/ipp_serial_warp.pl +scripts/ipp_serial_diff.pl +scripts/ipp_serial_stack.pl +scripts/ipp_simulation_data.pl +scripts/isp_trans.pl +scripts/ds9_cmf_regions.pl +t/00_distribution.t Index: /branches/eam_branches/eam_branch_20080618/ippScripts/README =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/README (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/README (revision 22155) @@ -0,0 +1,15 @@ +pod2text foo.pm > README + +If this is still here it means the programmer was too lazy to create the readme file. + +You can create it now by using the command shown above from this directory. + +At the very least you should be able to use this set of instructions +to install the module... + +perl Build.PL +./Build +./Build test +./Build install + +If you are on a windows box you should use 'nmake' rather than 'make'. Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/calibrate_dvo.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/calibrate_dvo.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/calibrate_dvo.pl (revision 22155) @@ -0,0 +1,196 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cal_id, $dvodb, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'cal_id|i=s' => \$cal_id, + 'dvodb|c=s' => \$dvodb, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --cal_id --dvodb --region", + -exitval => 3) unless + defined $cal_id and + defined $dvodb and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addrun"; +$command .= " -cal_id $cal_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -state 0"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cal_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $cal_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -cal_id $cal_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -state $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/camera_exp.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/camera_exp.pl (revision 22155) @@ -0,0 +1,339 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $exp_tag, $cam_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, + $no_op, $save_temps ); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'cam_id=s' => \$cam_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'outroot|w=s' => \$outroot, # output file base name + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --cam_id --camera --outroot", + -exitval => 3, + ) unless + defined $exp_tag and + defined $cam_id and + defined $outroot and + defined $camera; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# these should be coming from the psastro results +my $CHIPSTATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bias", type => "mean", flag => "-bias", dtype => "float" }, + { name => "bias_stdev", type => "rms", flag => "-bias_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, +# { name => "sigma_ra", type => "rms", flag => "-sigma_ra", dtype => "float" }, +# { name => "sigma_dec", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "ap_resid", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "ap_resid_stdev", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, + { name => "zp_mean", type => "mean", flag => "-zp_mean", dtype => "float" }, + { name => "zp_stdev", type => "rms", flag => "-zp_stdev", dtype => "float" }, + { name => "fwhm_major", type => "mean", flag => "-fwhm_major", dtype => "float" }, + { name => "fwhm_minor", type => "mean", flag => "-fwhm_minor", dtype => "float" }, + { name => "dtime_detrend", type => "sum", flag => "-dtime_detrend", dtype => "float" }, + { name => "dtime_photom", type => "sum", flag => "-dtime_photom", dtype => "float" }, + { name => "dtime_astrom", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "n_stars", type => "sum", flag => "-n_stars", dtype => "int" }, + { name => "n_extended", type => "sum", flag => "-n_extended", dtype => "int" }, + { name => "n_cr", type => "sum", flag => "-n_cr", dtype => "int" }, + ]; +my $chipStats = PS::IPP::Metadata::Stats->new($CHIPSTATS); # Stats parser + +# values to extract from camera-level output metadata and the stats to calculate +my $CAMSTATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, + { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "DT_ASTR", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, +# { name => "ZP??", type => "mean", flag => "-zp_mean", dtype => "float" }, +# { name => "ZP??", type => "rms", flag => "-zp_stdev", dtype => "float" }, + ]; +my $camStats = PS::IPP::Metadata::Stats->new($CAMSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +# test for addstar and psastro: +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$camtool -pendingimfile -cam_id $cam_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $cam_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + unless ($chipStats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $cam_id, $PS_EXIT_PROG_ERROR); + } +} + +### not needed to have such an extensive temp file name. +my ($list1File, $list1Name) = tempfile( "$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => !$save_temps ); # For binning 1 +my ($list2File, $list2Name) = tempfile( "$exp_tag.cm.$cam_id.b2.list.XXXX", UNLINK => !$save_temps ); # For binning 2 +my ($list3File, $list3Name) = tempfile( "$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => !$save_temps ); # For astrometry + +# XXX we perform astrometry iff photometry output exists +my $chipObjects; +my $chipObjectsExist = 0; +foreach my $file (@$files) { + # use the path_base as OUTPUT root and convert the filenames with ipprc->filename: + my $class_id = $file->{class_id}; + + # If there is only one chip, we use this name for the input to addstar + # we expect the chip analysis stage to produce psphot output (cmf file) and two binned images + $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id); + print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n"); + print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n"); + print $list3File ($ipprc->convert_filename_absolute($chipObjects) . "\n"); + + # if any of the output chip photometry files exist, we can run psastro / addstar below + if ($ipprc->file_exists($chipObjects)) { + $chipObjectsExist = 1; + } +} +close $list1File; +close $list2File; +close $list3File; + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $fpaStats = $ipprc->filename("PSASTRO.STATS", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} + +unless ($no_op) { + + ## build the output JPEG images first so we get them even if the astrometry fails + + # Make the jpeg for binning 1 + { + my $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg1); + } + + # Make the jpeg for binning 2 + { + my $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg2); + } + + # only run psastro / addstar if any of the output chip astrometry files exist (should we test for successful astrometry?) + if ($chipObjectsExist) { + # run psastro on the chipObjects, producing fpaObjects + # XXX add a ppStats call which will collect the astrometry stats + my $command; + $command = "$psastro -list $list3Name $outroot"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -stats $fpaStats -recipe PPSTATS CHIPSTATS"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code); + } + # XXX do we want to give an error if astrometry fails here? + &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); + &my_die("Unable to find expected output file: $fpaStats", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($fpaStats) or &my_die("Can't open statistics file $fpaStats: $!", $cam_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($camStats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n", $cam_id, $PS_EXIT_PROG_ERROR); + } + + # run addstar on the output fpaObjects (if a DVO database is defined) + if (defined $dvodbReal) { + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $command; + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -D CATDIR $dvodbReal"; + + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); + } + } + } +} + +my $fpaCommand = "$camtool -addprocessedexp"; +$fpaCommand .= " -cam_id $cam_id"; +$fpaCommand .= " -uri UNKNOWN"; +$fpaCommand .= " -path_base $outroot"; +$fpaCommand .= " -hostname $host" if defined $host; +$fpaCommand .= " -dbname $dbname" if defined $dbname; +$fpaCommand .= $chipStats->cmdflags(); +$fpaCommand .= $camStats->cmdflags(); + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cam_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $cam_id and not $no_update) { + my $command = "$camtool -addprocessedexp -cam_id $cam_id -uri UNKNOWN -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -path_base $outroot" if defined $outroot; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/chip_imfile.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/chip_imfile.pl (revision 22155) @@ -0,0 +1,253 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ( $exp_id, $chip_id, $class_id, $uri, $camera, $outroot, $dbname, $reduction, $verbose, + $no_update, $no_op ); +GetOptions( + 'exp_id=s' => \$exp_id, # Exposure identifier + 'chip_id=s' => \$chip_id, # Chiptool identifier + 'class_id=s' => \$class_id, # Class identifier + 'uri|u=s' => \$uri, # Input FITS file + 'camera|c=s' => \$camera, # Camera + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --chip_id --class_id --uri --camera --outroot", + -exitval => 3) unless + defined $exp_id and + defined $chip_id and + defined $class_id and + defined $uri and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +# the CHIP recipe should not perform astrometry anymore (2008.04.08) + +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use +unless ($recipe) { + &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# values to extract from output metadata and the stats to calculate +# XXX commented-out entries are not yet defined in the output files +# XXX EAM : we are removing the astrometry analysis from chip, so we +# do not bother to calculate those stats. +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "OVER_VAL", type => "mean", flag => "-bias", dtype => "float" }, + { name => "OVER_VAL", type => "stdev", flag => "-bias_stdev", dtype => "float" }, + { name => "FRINGE_0", type => "rms", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_2", dtype => "float" }, +# { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, +# { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "APMIFIT", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "DAPMIFIT", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, +# { name => "ZP??", type => "mean", flag => "-zp_mean", dtype => "float" }, +# { name => "ZP??", type => "rms", flag => "-zp_stdev", dtype => "float" }, + { name => "FWHM_X", type => "mean", flag => "-fwhm_major", dtype => "float" }, + { name => "FWHM_Y", type => "mean", flag => "-fwhm_minor", dtype => "float" }, + { name => "DT_DET", type => "sum", flag => "-dtime_detrend", dtype => "float" }, + { name => "DT_PHOT", type => "sum", flag => "-dtime_photom", dtype => "float" }, +# { name => "DT_ASTR", type => "sum", flag => "-dtime_astrom", dtype => "float" }, + { name => "NSTARS", type => "sum", flag => "-n_stars", dtype => "int" }, +# { name => "?", type => "sum", flag => "-n_extended", dtype => "int" }, +# { name => "?", type => "sum", flag => "-n_cr", dtype => "int" }, +# { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +## these names are used in ppImage, and thus may be URIs +my $outputImage = $ipprc->filename("PPIMAGE.CHIP", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputWeight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +# Run ppImage +unless ($no_op) { + ## XXX can we convert ppImage log and trace to use the filerules to generate consistent names + ## XXX also stats: output should be implied by $outroot + my $command = "$ppImage -file $uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + + ## get the ppImage recipe for this camera and CHIP reduction + $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe -"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## allow the output images to be optional, depending on the recipe / reduction class + my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS'); + if ($outputImageExpect) { + &my_die("Couldn't find expected output file: $outputImage\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + } + + my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS'); + if ($outputMaskExpect) { + &my_die("Couldn't find expected output file: $outputMask\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + } + + my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.WEIGHT.FITS'); + if ($outputWeightExpect) { + &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + } + + &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + &my_die("Couldn't find expected output file: $outputStats\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# command to update database +my $command = "$chiptool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -chip_id $chip_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage"; +$command .= " -path_base $outroot"; +$command .= " -hostname $host" if defined $host; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; # Chiptool identifier + my $chip_id = shift; # Chiptool identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + # outputImage and outroot are globals + + carp($msg); + if (defined $chip_id and defined $class_id and not $no_update) { + my $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -uri $outputImage"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_correct_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_correct_imfile.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_correct_imfile.pl (revision 22155) @@ -0,0 +1,154 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $dbname, $workdir, $reduction, + $verbose, $no_update, $no_op, $outroot, $redirect, $corr_id, $corr_type, $corr_uri ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'corr_id|u=s' => \$corr_id, + 'corr_type|u=s' => \$corr_type, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'outroot' => \$outroot, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera", + -exitval => 3) + unless defined $det_id + and defined $exp_id + and defined $class_id + and defined $det_type + and defined $exp_tag + and defined $input_uri + and defined $camera; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +if ($redirect) { + die "must suplly --outroot with --redirect-output" if !defined ($outroot); + my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); + $ipprc->redirect_output($logDest); +} + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $dvocorr = can_run('dvoApplyCorr') or (warn "Can't find dvoApplyCorr" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +# detselect -select -det_id $corr_id -class_id $class_id; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +my $outputImage = $ipprc->filename("DVOCORR.OUTPUT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + +# Run dvoApplyCorr +unless ($no_op) { + my $command = "$dvocorr -file $input_uri $outputRoot"; + $command .= " -corr $corr_uri"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); +} + +# command to update database +my $command = "$dettool -addcorrectedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addcorrectedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and defined $class_id and not $no_update) { + my $command = "$dettool -addcorrectedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_apply.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_apply.pl (revision 22155) @@ -0,0 +1,229 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line +my ( $det_id, $iter, $class_id, $value, $input_uri, $camera, $det_type, $outroot, $dbname, $verbose, + $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend ID + 'iteration|n=s' => \$iter, # Iteration + 'class_id|i=s' => \$class_id, # Class ID + 'value|v=s' => \$value, # Value to multiple (for normalisation) + 'input_uri|u=s' => \$input_uri, # Input file + 'camera|c=s' => \$camera, # Camera + 'det_type|t=s' => \$det_type, # Detrend type + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database + 'no-op' => \$no_op, # Don't do any operations + 'redirect-output' => \$redirect, # send output from script to LOG.IMFILE + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --value --input_uri --camera --det_type --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $value and + defined $input_uri and + defined $camera and + defined $det_type and + defined $outroot; + +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +$ipprc->redirect_output($logDest) if $redirect; + +my $RECIPE_PPIMAGE = 'PPIMAGE_N'; # Recipe to use with ppImage +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +# Define which detrend types we normalise +use constant DETTYPE => { + 'bias' => 'bias', + 'dark' => 'dark', + 'dark_premask' => 'dark', + 'shutter' => 'shutter', + 'flat' => 'flat', + 'domeflat' => 'flat', + 'skyflat' => 'flat', + 'flat_premask' => 'flat', + 'domeflat_premask' => 'flat', + 'skyflat_premask' => 'flat', + 'fringe' => 'fringe', + 'mask' => 'mask', + 'darkmask' => 'mask', + 'flatmask' => 'mask', + }; + +# convert supplied detrend type to a controlled namespace +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR) unless exists DETTYPE()->{lc($det_type)}; +my $det_type_real = DETTYPE()->{lc($det_type)}; + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outFile = ($det_type_real eq "mask") ? 'PPIMAGE.OUTPUT.DETMASK' : 'PPIMAGE.OUTPUT';; # XXXX something of a hack (too many places to control things...) + +my $output = $ipprc->filename($outFile, $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $b1name = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $b2name = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $statsName = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); + +# Run normalisation +unless ($no_op) { + + # we cannot use ppImage to load a normalized mask : just copy it and build the jpeg images + if ($det_type_real eq 'mask') { + $RECIPE_PPIMAGE = 'PPIMAGE_BIN'; + my $input_real = $ipprc->file_resolve($input_uri, 0); + my $output_real = $ipprc->file_resolve($output, 1); + system ("cp $input_real $output_real"); + } + + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -norm $value -stats $statsName"; + $command .= " -recipe PPIMAGE $RECIPE_PPIMAGE"; + $command .= " -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= " -F PPIMAGE.OUTPUT $outFile" unless $outFile eq 'PPIMAGE.OUTPUT'; + $command .= ' -isfringe' if $det_type_real eq 'fringe'; + $command .= ' -isdark' if $det_type_real eq 'dark'; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Can't find expected output file: $output", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($output); + &my_die("Can't find expected output file: $b1name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b1name); + &my_die("Can't find expected output file: $b2name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b2name); + &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($statsName); + + # Get the statistics on the normalised image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($statsName) or &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $statsFile; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# Command to update the database +my $command = "$dettool -addnormalizedimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $output"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addnormalizedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addnormalizedimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_calc.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_calc.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_calc.pl (revision 22155) @@ -0,0 +1,242 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run ); +use IPC::Run 0.36 qw( run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse command-line arguments +my ($det_id, $iter, $detType, $outroot, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend id + 'iteration|i=s' => \$iter, # Iteration + 'det_type|t=s' => \$detType, # Detrend type + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do operations + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_id --iteration --det_type --outroot", + -exitval => 3, + ) unless + defined $det_id and + defined $iter and + defined $detType and + defined $outroot; + +use constant STATISTIC => 'bg'; # Background statistic to use from the database + +# Define which detrend types we normalise +use constant NORMALIZE => { + 'bias' => 0, + 'dark' => 0, + 'dark_premask' => 0, + 'shutter' => 0, + 'flat' => 1, + 'domeflat' => 1, + 'skyflat' => 1, + 'flat_premask' => 1, + 'domeflat_premask' => 1, + 'skyflat_premask' => 1, + 'fringe' => 0, + 'mask' => 0, + 'darkmask' => 0, + 'flatmask' => 0, + }; + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppNormCalc = can_run('ppNormCalc') or (warn "Can't find ppNormCalc" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless exists NORMALIZE()->{lc($detType)}; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my @files; # The input files +{ + my $command = "$dettool -processedimfile"; + $command .= " -det_id $det_id"; # Command to run + $command .= " -included"; # only use the inputs for this detrend run to calculate the norm + $command .= " -dbname $dbname" if defined $dbname; + my @command = split /\s+/, $command; + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n" if $verbose; + if (not run(\@command, \$stdin, \$stdout, \$stderr)) { + &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?", + $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my @whole = split /\n/, $stdout; + my @single = (); + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from dettool", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless + defined $list; + push @files, @$list; + @single = (); + } + } +} + +my $norms; # MDC with normalisations +if (NORMALIZE()->{lc($detType)} and not $no_op) { + + my %matrix; # Matrix of statistics as a function of exposures and classes + foreach my $file (@files) { + my $exp_id = $file->{'exp_id'}; # Exposure ID + my $class_id = $file->{'class_id'}; # Class ID + my $stat = $file->{STATISTIC()}; # Statistic of interest + + # Create matrix elements + $matrix{$exp_id} = {} if not defined $matrix{$exp_id}; + $matrix{$exp_id}->{$class_id} = $stat; + } + + # Generate the input for ppNormCalc + my $normData; # Normalisation data + foreach my $exp_id (keys %matrix) { + $normData .= "$exp_id\tMETADATA\n"; + foreach my $class_id (keys %{$matrix{$exp_id}}) { + $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_id}->{$class_id} . "\n"; + } + $normData .= "END\n\n"; + } + + # Run ppNormCalc + { + my ( $stdout, $stderr ); # Buffers for running program + my @command = split /\s+/, $ppNormCalc; + print "Running [$ppNormCalc]...\n" if $verbose; + if (not run(\@command, \$normData, \$stdout, \$stderr)) { + &my_die("Unable to perform ppNormCalc: $?", $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + + # Parse the output + $norms = $mdcParser->parse($stdout); + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless $norms; + } + +} else { + # It's something that doesn't need normalisation --- just push in a normalisation of 1 + my %classes; # List of unique classes + foreach my $file (@files) { + my $class_id = $file->{'class_id'}; # Class Id + $classes{$class_id} = 1; + } + + foreach my $class_id (keys %classes) { + my %mdValue; # Metadata value for this class id + $mdValue{name} = $class_id; + $mdValue{value} = 1.0; + push @$norms, \%mdValue; + } +} + +my $commandBase = "$dettool -addnormalizedstat"; +$commandBase .= " -det_id $det_id"; +$commandBase .= " -iteration $iter"; +$commandBase .= " -dbname $dbname" if defined $dbname; + +# Process output normalisations +unless ($no_update) { + foreach my $normItem (@$norms) { + + my $className = $normItem->{name}; # Name of component + my $normalisation = $normItem->{value}; # Normalisation for component + + if ($normalisation == 0.0 or lc($normalisation) eq 'nan') { + warn("Class $className has bad normalisation: $normalisation"); + exit($PS_EXIT_SYS_ERROR); + } + + my $command = $commandBase; + $command .= " -class_id $className"; + $command .= " -norm $normalisation"; + + my @command = split /\s+/, $command; + + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n" if $verbose; + if (not run \@command, \$stdin, \$stdout, \$stderr) { + warn("Unable to perform dettool -addnormstat for $className: $?"); + exit($PS_EXIT_SYS_ERROR); + } + print $stdout . "\n" if $verbose; + } +} else { + print "skipping command: $commandBase\n"; + foreach my $normItem (@$norms) { + + my $className = $normItem->{name}; # Name of component + my $normalisation = $normItem->{value}; # Normalisation for component + + if ($normalisation == 0.0 or lc($normalisation) eq 'nan') { + warn("Class $className has bad normalisation: $normalisation"); + exit($PS_EXIT_SYS_ERROR); + } + print "$className : $normalisation\n"; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -addnormalizedstat"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_exp.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_norm_exp.pl (revision 22155) @@ -0,0 +1,197 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, $no_op, $save_temps); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration|i=s' => \$iter, + 'camera|c=s' => \$camera, + 'det_type|t=s' => \$det_type, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --camera --det_type --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $camera and + defined $det_type and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -normalizedimfile -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to get list of normalized imfiles from dettool: $error_code", $det_id, $iter, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR); + } +} + +my ($list1File, $list1Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps ); +my ($list2File, $list2Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps ); +foreach my $file (@$files) { + print $list1File ( $ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ( $ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); + +} + +# command to update the database +$command = "$dettool -addnormalizedexp"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -recip $recipe1,$recipe2"; +$command .= " -path_base $outroot "; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -addnormalizedexp: $error_code", $det_id, $iter, $error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -addnormalizedexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_process_exp.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_process_exp.pl (revision 22155) @@ -0,0 +1,215 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $det_type, $exp_tag, $camera, $outroot, $dbname, $reduction, $verbose, $no_update, + $no_op, $save_temps, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'det_type|t=s' => \$det_type, + 'exp_id|d=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'save-temps' => \$save_temps, # Save temporary files? + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --det_type --exp_id --exp_tag --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $det_type and + defined $exp_id and + defined $exp_tag and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +if ($redirect) { + my $logDest = $ipprc->filename("LOG.EXP", $outroot, "NONE") + or &my_die("Missing entry in file rules", $det_id, $exp_id, $PS_EXIT_CONFIG_ERROR); + $ipprc->redirect_output($logDest); +} + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_id, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + } +} + +# File lists for the two binnings +my ($list1File, $list1Name) = tempfile( "$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => !$save_temps ); +my ($list2File, $list2Name) = tempfile( "$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => !$save_temps ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2); +} + +# Command to update the database +$command = "$dettool -addprocessedexp"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedexp: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and not $no_update) { + my $command = "$dettool -addprocessedexp"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_process_imfile.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_process_imfile.pl (revision 22155) @@ -0,0 +1,190 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $outroot, $dbname, $reduction, + $verbose, $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $exp_id and + defined $class_id and + defined $det_type and + defined $exp_tag and + defined $input_uri and + defined $camera and + defined $outroot; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR); +# Run ppImage +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve("$outputStats") or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) + or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# command to update database +my $command = "$dettool -addprocessedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $reduction"; +$command .= " -uri $outputImage -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $exp_id and defined $class_id and not $no_update) { + my $command = "$dettool -addprocessedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_reject_exp.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_reject_exp.pl (revision 22155) @@ -0,0 +1,366 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my $ITER_LIMIT = 20; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $outroot, $filter, $dbname, $verbose, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $det_type and + defined $camera and + defined $outroot; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# these stats are used it the rejections but not passed to the database +# there is some duplication with the above, but the calculation time is minimal +my $REJSTATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "clipmean", flag => "ensMeanMean", dtype => "float" }, + { name => "bg", type => "clipstdev", flag => "ensMeanStdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "clipmean", flag => "ensMeanStdevMean", dtype => "float" }, + { name => "bg_mean_stdev", type => "clipstdev", flag => "ensMeanStdevStdev", dtype => "float" }, + { name => "bg_stdev", type => "clipmean", flag => "ensStdevMean", dtype => "float" }, + { name => "bg_stdev", type => "clipstdev", flag => "ensStdevStdev", dtype => "float" }, + { name => "bg_skewness", type => "clipmean", flag => "ensSkewness", dtype => "float" }, + { name => "bg_kurtosis", type => "clipmean", flag => "ensKurtosis", dtype => "float" }, + + ]; +my $rejstats = PS::IPP::Metadata::Stats->new($REJSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($exposures, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select exp data for this det_run + $command = "$dettool -residexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code); + } + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $exposures = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics for rejections + $rejstats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); +} + +# each image has a mean mean (average of means over all chips) +# a standard deviation mean (average of standard deviations over all chips) +# and a + + + +# we use the statistics of the ensemble to accept/reject exposurs +my $ensMeanMean = $rejstats->value_for_flag ("ensMeanMean"); # average of all exposure means (in turn averaged over all chips) +my $ensMeanStdev = $rejstats->value_for_flag ("ensMeanStdev"); # standard deviation of all exposure means +my $ensMeanStdevMean = $rejstats->value_for_flag ("ensMeanStdevMean"); # average over all exposures of the stdev of the means for each chip +my $ensMeanStdevStdev = $rejstats->value_for_flag ("ensMeanStdevStdev"); # standard deviation over all exposures of the stdev of the means for each chip +my $ensStdevMean = $rejstats->value_for_flag ("ensStdevMean"); # average over all exposures of the sum of the squares of the stdevs for each chip +my $ensStdevStdev = $rejstats->value_for_flag ("ensStdevStdev"); # standard deviation over all exposures of the sum of the squares of the stdevs for each chip + +$ipprc->define_camera($camera); +# Rejection thresholds +my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter ); +my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter ); +my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter ); + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $logName = "$outroot.log"; # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); + print $logFile "Ensemble mean $ensMeanMean +/- $ensMeanStdev\n"; + print $logFile "Ensemble stdev $ensStdevMean +/- $ensStdevStdev\n"; + print $logFile "Ensemble mean rms (over imfiles) $ensMeanStdevMean +/- $ensMeanStdevStdev\n\n"; +} + +# Go through again to do rejection, and update the database for each exposure +my $numChanges = 0; # Number of exposures with changed status +my $numReject = 0; # Number of exposures rejected +my $firstElement = 1; + +foreach my $exposure (@$exposures) { + my $mean = $exposure->{bg}; # Mean for this exposure + my $stdev = $exposure->{bg_stdev}; # Stdev for this exposure + my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure + my $expID = $exposure->{exp_id}; + my $accept = $exposure->{accept}; + my $include = $exposure->{include}; + + &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $expID; + &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $accept; + &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include; + + my $reject = 0; # Reject this exposure? + + $command = "$dettool -updateresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $expID"; + $command .= " -dbname $dbname" if defined $dbname; + + if (not $accept) { + # Rejected this at an earlier stage + unless ($no_op) { + print $logFile "Rejecting $expID based on earlier determination.\n"; + } + $reject = 1; + goto UPDATE; + } + + # Cop-out if we're not doing any operations + if ($no_op) { + # Make sure something gets rejected (just once!), just so that + # we can trace the full range of the workflow + if ($firstElement and $iter == 0) { + $reject = 1; + } + goto UPDATE; + } + + if ($reject_mean > 0 and $ensMeanStdev > 0) { + my $delta = abs($mean - $ensMeanMean); + if ($delta > ($reject_mean * $ensMeanStdev)) { + print $logFile "Rejecting $expID based on ensemble mean value: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean: "; + } + print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean\n"; + } + + if ($reject_stdev > 0 and $ensStdevStdev > 0) { + my $delta = abs($stdev - $ensStdevMean); + if ($delta > ($reject_stdev * $ensStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble stdev: "; + } + print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble stdev\n"; + } + + if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) { + my $delta = abs($meanStdev - $ensMeanStdevMean); + if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble mean stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean stdev: "; + } + print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean stdev\n"; + } + + UPDATE: + if ($reject) { + $command .= ' -reject'; + $numReject++; + } + + # Check for status changes + if ((not $include and not $reject) or ($include and $reject)) { + unless ($no_op) { + print $logFile "Status of $expID has changed.\n"; + } + $numChanges++; + } + + unless ($no_update) { + # Update + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code); + } + } +} + +# Decide if the current is sufficient to use as a master, and if we can stop iterating +my $master = 1; # This is good enough for a master +my $stop = 1; # Stop iterating + +if ($numChanges > 0) { + $master = 0; + $stop = 0; +} + +# Rejecting everything --- stop before something bad happens! +if ($numReject == scalar @$exposures) { + $master = 0; + $stop = 1; +} + +unless ($no_op) { + print $logFile "Master: $master\n"; + print $logFile "Stop: $stop\n"; + close $logFile; +} + +# attempt to prevent endless, pathological iterations +if ($iter >= $ITER_LIMIT) { + warn("iteration limit reached -- bailing out"); + exit($PS_EXIT_PROG_ERROR); +} + +## XXX for now, always stop on each iteration +$stop = 1; + +## add the summary statistics, and request a new iteration if needed +$command = "$dettool -adddetrunsummary"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -accept" if $master; +$command .= " -dbname $dbname" if defined $dbname; +### XXX for now, removed : $command .= " -again" unless $stop; +$command .= $stats->cmdflags(); + +# Put results into the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -adddetrunsummary: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and not $no_update) { + my $command = "$dettool -adddetrunsummary"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying unless extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" unless defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_reject_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_reject_imfile.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_reject_imfile.pl (revision 22155) @@ -0,0 +1,611 @@ +#!/usr/bin/env perl + +# this program has two jobs: +# 1: for the given exp_tag, generate the binned & mosaiced JPEG images +# 2: examine the collection of per-imfile statistics and reject. At the moment, +# this program (and the database) only allows rejection at the exposure level, +# not at the imfile level. + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); # tools to run UNIX programs with control over I/O +use PS::IPP::Metadata::Config; # tools to parse the psMetadataConfig files +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list +use Statistics::Descriptive; # tools for calculating basic statistical quantities +use File::Temp qw( tempfile ); # tools to construct temp files +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); # option parsing +use Pod::Usage qw( pod2usage ); + +# parse the command-line options +my ( $det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction, + $verbose, $no_update, $no_op, $save_temps, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'filter=s' => \$filter, + 'reject' => \$reject, + 'dbname|d=s' => \$dbname, # Database name + 'reduction|=s' => \$reduction, + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'save-temps' => \$save_temps, # Save temporary files? + 'redirect-output' => \$redirect, # redirect output from LOG.IMFILE + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $exp_id and + defined $exp_tag and + defined $det_type and + defined $camera and + defined $outroot; + +# load IPP config information for the specified camera +$ipprc->define_camera($camera); +if ($redirect) { + # XXX should this really be going to LOG.IMFILE? + # That's where detrend.resid.pro was sending it + my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, "NONE") + or &my_die("Missing entry from camera config", $det_id, $iter, $exp_id, $PS_EXIT_CONFIG_ERROR); + $ipprc->redirect_output($logDest); +} + + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_skewness", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "bg_kurtosis", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "bin_stdev", type => "rms", flag => "-bin_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "user_1", type => "mean", flag => "-user_1", dtype => "float" }, # fringe residual + { name => "user_2", type => "rms", flag => "-user_2", dtype => "float" }, # fringe residual + { name => "user_1", type => "stdev", flag => "-user_3", dtype => "float" }, # fringe residual + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of imfile files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select imfile data for this exp_id + $command = "$dettool -residimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -residimfile: $error_code\n"); + exit($error_code); + } + # XXX report an error message if stdout_buf is empty + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); +} + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outroot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outroot); # Binned JPEG #2 +my $logName = $ipprc->filename("LOG.EXP", $outroot); # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); +} + +# XXX in debug mode, unlink = 0 +my ($list1File, $list1Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps ); +my ($list2File, $list2Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + + +# build the JPEG images +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outroot -recipe PPIMAGE $recipe1"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outroot -recipe PPIMAGE $recipe2"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); +} + +#### measure stats and reject if needed #### + +# Rejection thresholds & related data +my $expected = rejection_limit( 'EXPECTED', $det_type, $filter ); +my $reject_imfile_mean = rejection_limit( 'IMFILE.MEAN', $det_type, $filter ); +my $reject_imfile_flux = rejection_limit( 'IMFILE.FLUX', $det_type, $filter ); +my $reject_imfile_stdev = rejection_limit( 'IMFILE.STDEV', $det_type, $filter ); +my $reject_imfile_skewness = rejection_limit( 'IMFILE.SKEWNESS', $det_type, $filter ); +my $reject_imfile_kurtosis = rejection_limit( 'IMFILE.KURTOSIS', $det_type, $filter ); +my $reject_imfile_meanstdev = rejection_limit( 'IMFILE.MEANSTDEV', $det_type, $filter ); +my $reject_imfile_snr = rejection_limit( 'IMFILE.SNR', $det_type, $filter ); +my $reject_imfile_bin_stdev = rejection_limit( 'IMFILE.BIN.STDEV', $det_type, $filter ); +my $reject_imfile_bin_snr = rejection_limit( 'IMFILE.BIN.SNR', $det_type, $filter ); +my $reject_exp_mean = rejection_limit( 'EXP.MEAN', $det_type, $filter ); +my $reject_exp_flux = rejection_limit( 'EXP.FLUX', $det_type, $filter ); +my $reject_exp_stdev = rejection_limit( 'EXP.STDEV', $det_type, $filter ); +my $reject_exp_meanstdev = rejection_limit( 'EXP.MEANSTDEV', $det_type, $filter ); +my $reject_exp_snr = rejection_limit( 'EXP.SNR', $det_type, $filter ); +my $reject_exp_bin_stdev = rejection_limit( 'EXP.BIN.STDEV', $det_type, $filter ); +my $reject_exp_bin_snr = rejection_limit( 'EXP.BIN.SNR', $det_type, $filter ); + +# storage array +my @fluxes; + +foreach my $file (@$files) { + my $name = $file->{class_id}; + my $mean = $file->{bg}; # Mean for this imfile + my $stdev = $file->{bg_stdev}; # Stdev for this imfile + my $skewness = $file->{bg_skewness}; # Skewness for this imfile + my $kurtosis = $file->{bg_kurtosis}; # Kurtosis for this imfile + my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this imfile + my $binStdev = $file->{bin_stdev}; # Binned Stdev for this imfile + + # calculate and save the fluxes + my $flux; + if ($file->{exp_time} == 0.0) { + $flux = $mean; + } else { + $flux = $mean / $file->{exp_time}; + } + push @fluxes, $flux; + + $mean -= $expected; + + last if $no_op; + + # reject exposure if, for any imfiles, the mean residual counts + # deviate from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual mean count value (eg, + # 0.0 for a bias). in general, use with ADDITIVE components + if ($reject_imfile_mean > 0) { + if (abs($mean) > $reject_imfile_mean * $stdev) { + print $logFile "Rejecting exposure based on bad imfile mean for $name: "; + $reject = 1; + } else { + print $logFile "Imfile mean for $name meets requirements: "; + } + print $logFile "$mean vs $reject_imfile_mean * $stdev\n"; + } else { + print $logFile "No rejection on imfile mean for $name\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_imfile_flux > 0) { + if (abs($flux) > $reject_imfile_flux) { + print $logFile "Rejecting exposure based on bad imfile flux for $name: "; + $reject = 1; + } else { + print $logFile "Imfile flux for $name meets requirements: "; + } + print $logFile "$flux vs $reject_imfile_flux\n"; + } else { + print $logFile "No rejection on imfile flux for $name\n"; + } + + # reject exposure if, for any imfiles, the total pixel variance is + # larger than the limit + if ($reject_imfile_stdev > 0) { + if ($stdev > $reject_imfile_stdev) { + print $logFile "Rejecting exposure based on bad imfile stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile stdev for $name meets requirements: "; + } + print $logFile "$stdev vs $reject_imfile_stdev\n"; + + } else { + print $logFile "No rejection on imfile stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the skewness is + # larger than the limit + if ($reject_imfile_skewness > 0) { + if ($stdev > $reject_imfile_skewness) { + print $logFile "Rejecting exposure based on bad imfile skewness for $name: "; + $reject = 1; + } else { + print $logFile "Imfile skewness for $name meets requirements: "; + } + print $logFile "$skewness vs $reject_imfile_skewness\n"; + + } else { + print $logFile "No rejection on imfile skewness for $name\n"; + } + + # reject exposure if, for any imfiles, the kurtosis is + # larger than the limit + if ($reject_imfile_kurtosis > 0) { + if ($stdev > $reject_imfile_kurtosis) { + print $logFile "Rejecting exposure based on bad imfile kurtosis for $name: "; + $reject = 1; + } else { + print $logFile "Imfile kurtosis for $name meets requirements: "; + } + print $logFile "$kurtosis vs $reject_imfile_kurtosis\n"; + + } else { + print $logFile "No rejection on imfile kurtosis for $name\n"; + } + + # reject exposure if, for any imfiles, the variance of the imfile + # component means is larger than the limit + if ($reject_imfile_meanstdev > 0) { + if ($meanStdev > $reject_imfile_meanstdev) { + print $logFile "Rejecting exposure based on bad imfile mean stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile mean stdev for $name meets requirements: "; + } + print $logFile "$meanStdev vs $reject_imfile_meanstdev\n"; + } else { + print $logFile "No rejection on imfile mean stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise (ie, + # the mean counts / total pixel variance) of the imfile component + # means are less than the limit. this test is sensible for images + # which have finite residual flux such as a flat-field image. + if ($reject_imfile_snr > 0) { + if ($mean < $stdev * $reject_imfile_snr) { + print $logFile "Rejecting exposure based on bad imfile S/N for $name: "; + $reject = 1; + } else { + print $logFile "Imfile S/N for $name meets requirements: "; + } + print $logFile "mean: $mean vs stdev*SNlimit: " . $stdev * $reject_imfile_snr . "\n"; + } else { + print $logFile "No rejection on imfile S/N for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise of the + # imfile component means, based on the stdev of the binned image + # is less than the limit. this test is sensible for images which + # have finite residual flux such as a flat-field image. + if ($reject_imfile_bin_stdev > 0) { + if ($binStdev > $reject_imfile_bin_stdev) { + print $logFile "Rejecting exposure based on bad imfile binned stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile binned stdev for $name meets requirements: "; + } + print $logFile "$binStdev vs $reject_imfile_bin_stdev\n"; + } else { + print $logFile "No rejection on imfile binned stdev for $name\n"; + } + if ($reject_imfile_bin_snr > 0) { + if ($mean < $binStdev * $reject_imfile_bin_snr) { + print $logFile "Rejecting exposure based on bad imfile binned S/N for $name: "; + $reject = 1; + } else { + print $logFile "Imfile binned S/N for $name meets requirements: "; + } + print $logFile "mean: $mean vs binStdev*SNlimit: " . $binStdev * $reject_imfile_bin_snr . "\n"; + } else { + print $logFile "No rejection on imfile binned S/N for $name\n"; + } +} + +# basic ensemble stats +my $mean = $stats->value_for_flag ("-bg"); +my $meanStdev = $stats->value_for_flag ("-bg_mean_stdev"); +my $stdev = $stats->value_for_flag ("-bg_stdev"); +my $binStdev = $stats->value_for_flag ("-bin_stdev"); +my $fringe_mean = $stats->value_for_flag ("-fringe_0"); +my $fringe_err = $stats->value_for_flag ("-fringe_1"); +my $fringe_mean_stdev = $stats->value_for_flag ("-fringe_2"); +my $dfringe_mean = $stats->value_for_flag ("-fringe_resid_0"); +my $dfringe_err = $stats->value_for_flag ("-fringe_resid_1"); +my $dfringe_mean_stdev = $stats->value_for_flag ("-fringe_resid_2"); + +# other stats (flux depends on bg and exp_time) +my $fluxStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means +$fluxStats->add_data(@fluxes); +my $flux = $fluxStats->mean(); # Mean of the imfile means + +# other stats +my $exp_snr = 0.0; +if ($stdev > 0) { $exp_snr = $mean / $stdev; } + +## Reject based on the exposure ensemble stats +# reject if the exposure ensemble mean is deviant +unless ($no_op) { + print $logFile "Exposure mean $mean, stdev $stdev, mean stdev $meanStdev, exp s/n: $exp_snr\n"; + + # reject exposure if the ensemble mean residual counts deviate + # from the expected value by more than N sigma, (sigma = total + # pixel variance). this test is sensible for images which should + # have a predictable nominal residual mean count value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_mean > 0) { + if (abs($mean) > $reject_exp_mean * $stdev) { + print $logFile "Rejecting exposure based on bad mean counts: "; + $reject = 1; + } else { + print $logFile "Exposure mean counts meets requirements: "; + } + print $logFile "mean: $mean, stdev: $stdev (limit is: " . $reject_exp_mean * $stdev . ") \n"; + } else { + print $logFile "No rejection for exp mean\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_flux > 0) { + if (abs($flux) > $reject_exp_flux * $stdev) { + print $logFile "Rejecting exposure based on bad mean flux: "; + $reject = 1; + } else { + print $logFile "Exposure mean flux meets requirements: "; + } + print $logFile "flux: $flux, stdev: $stdev (limit is: " . $reject_exp_flux * $stdev . ")\n"; + } else { + print $logFile "No rejection for exp mean\n"; + } + + # reject exposure if the total pixel variance is larger than the + # limit + if ($reject_exp_stdev > 0) { + if ($stdev > $reject_exp_stdev) { + print $logFile "Rejecting exposure based on bad stdev: "; + $reject = 1; + } else { + print $logFile "Exposure stdev meets requirements: "; + } + print $logFile "$stdev vs $reject_exp_stdev\n"; + } else { + print $logFile "No rejection for exp stdev\n"; + } + + # reject exposure if the variance of the imfile means is larger + # than the limit + if ($reject_exp_meanstdev > 0) { + if ($meanStdev > $reject_exp_meanstdev) { + print $logFile "Rejecting exposure based on bad mean stdev: "; + $reject = 1; + } else { + print $logFile "Exposure mean stdev meets requirements: "; + } + print $logFile "$meanStdev vs $reject_exp_meanstdev\n"; + } else { + print $logFile "No rejection for exp mean stdev\n"; + } + + # reject if the signal-to-noise is insufficient + if ($reject_exp_snr > 0) { + if (abs($mean) < abs($stdev * $reject_exp_snr)) { + print $logFile "Rejecting exposure based on poor S/N: \n"; + $reject = 1; + } else { + print $logFile "Exposure S/N meets requirements: \n"; + } + print $logFile "signal: $mean vs noise: $stdev (s/n limit is: $reject_exp_snr)\n"; + } else { + print $logFile "No rejection for exp S/N\n"; + } + # reject if the exposure ensemble stdev is deviant + if ($reject_exp_bin_stdev > 0) { + if ($binStdev > $reject_exp_bin_stdev) { + print $logFile "Rejecting exposure based on bad binned stdev: "; + $reject = 1; + } else { + print $logFile "Exposure binned stdev meets requirements: "; + } + print $logFile "$stdev vs $reject_exp_bin_stdev\n"; + } else { + print $logFile "No rejection for exp binned stdev\n"; + } + # reject if the signal-to-noise is insufficient + if ($reject_exp_bin_snr > 0) { + if (abs($mean) < abs($binStdev * $reject_exp_bin_snr)) { + print $logFile "Rejecting exposure based on poor binned S/N: \n"; + $reject = 1; + } else { + print $logFile "Exposure binned S/N meets requirements: \n"; + } + print $logFile "signal: $mean vs noise: $binStdev (s/n limit is: $reject_exp_bin_snr)\n"; + } else { + print $logFile "No rejection for exp binned S/N\n"; + } + + close $logFile; +} + +$command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outroot "; +$command .= ' -reject' if $reject; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidexp: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $exp_id and not $no_update) { + my $command = "$dettool -addresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying if not extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" if not defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ + +#### + +## this function is not well though out. it loads a collection of +## values (background mean and stdev) for a collection of imfile +## subcomponents. it then measures the mean background, the +## background stdev (rms of ensemble stdevs), and the background mean +## stdev (rms of the means). it then tries to reject based on the +## following criteria: +## (background / stdev > limit) for a single component +## ---> this will ACCEPT for a poor images (large stdev) +## (stdev > limit) +## (mean background / stdev > limit) same problem +## (mean stdev > limit) + +### I would suggest the following: +## 1) calculate the ensemble mean and stdev +## 2) for each component, is the (back - mean) / mean stdev > limit? +## (ie, is it an outlier?) +## 3) is the component stdev > limit? (absolute test) +## 4) calculate the background stdev for the ensemble (excluding rejects) +## 5) is the mean stdev > limit (poor images) +## 6) calculate the ensemble stdev (rms) of remaining (excluding rejects) +## 7) is the ensemble stdev > limit + +## for flats, the value used for stdev should be the fractional stdev +## (stdev/mean) and no rejection should be performed on the basis of +## the mean value, but could still be done on the mean stdev + +## the same logical cuts above can be applied to components in an +## imfile, imfiles in an exposure, and exposures in a stack + +### this function needs to avoid div by zero: compare Signal vs Noise*limit Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_resid.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_resid.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_resid.pl (revision 22155) @@ -0,0 +1,264 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $exp_id, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $outroot, + $dbname, $reduction, $verbose, $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'detrend=s' => \$detrend, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'mode|m=s' => \$mode, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --class_id --det_type --camera --input_uri --mode --detrend --outroot (not for 'verify' mode)", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $exp_id and + defined $exp_tag and + defined $class_id and + defined $det_type and + defined $input_uri and + defined $camera and + defined $mode and + defined $outroot and + (defined $detrend or lc($mode) eq 'verify'); + +$ipprc->define_camera($camera); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id); +if ($redirect) { + $ipprc->redirect_output($logDest); +} + +# Recipes to use as a function of detrend type and mode +$reduction = 'DETREND' unless defined $reduction; +my $recipe; # Name of recipe to use +if ($mode eq 'master') { + $recipe = uc($det_type) . '_RESID'; +} elsif ($mode eq 'verify') { + $recipe = uc($det_type) . '_VERIFY'; +} else { + &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} +$recipe = $ipprc->reduction($reduction, $recipe); + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "SAMPLE_SKEWNESS", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "SAMPLE_KURTOSIS", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "FRINGE_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "mean", flag => "-fringe_resid_0", dtype => "float" }, + { name => "FRINGE_RESID_ERR_0", type => "rms", flag => "-fringe_resid_1", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "stdev", flag => "-fringe_resid_2", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +my $BINNED_STATS = + [ + { name => "ROBUST_STDEV", type => "rms", flag => "-bin_stdev" }, + ]; +my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser + +# Flags to specify the particular detrend to use +use constant DETRENDS => { + 'bias' => '-bias', # Specify the bias frame + 'dark' => '-dark', # Specify the dark frame + 'dark_premask' => '-dark', # Specify the dark frame + 'shutter' => '-shutter', # Specify the shutter frame + 'flat' => '-flat', # Specify the flat frame + 'domeflat' => '-flat', # Specify the flat frame + 'skyflat' => '-flat', # Specify the flat frame + 'flat_premask' => '-flat', # Specify the flat frame + 'domeflat_premask' => '-flat', # Specify the flat frame + 'skyflat_premask' => '-flat', # Specify the flat frame + 'fringe' => '-fringe', # Specify the fringe frame + 'mask' => '-mask', # Specify the mask frame + 'darkmask' => '-mask', # Specify the mask frame + 'flatmask' => '-mask', # Specify the mask frame +}; + +use constant DELETE_STATS => 0; # Delete the statistics file when done? + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +# XXX use PPIMAGE.OUTPUT.RESID for output file (compressed, with pos & neg range allowed) +# my $outputName = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id); + +my $outputName = $ipprc->filename("PPIMAGE.OUTPUT.RESID", $outroot, $class_id); +my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outroot, $class_id); +my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outroot, $class_id); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id); + +# Run ppImage & ppStats +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outroot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS RESIDUAL"; + $command .= " -F PPIMAGE.OUTPUT PPIMAGE.OUTPUT.RESID"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + # Detrend to use in processing + if (lc($mode) ne 'verify') { + my $detFlag = DETRENDS->{lc($det_type)}; + &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR) unless defined $detFlag; + $command .= " $detFlag $detrend"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name); + &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name); + + # Load the raw output stats file + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # Parse the stats file contents into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # run ppStats on the binned image + $command = "$ppStats -recipe PPSTATS RESIDUAL $bin2Name"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + + # Parse the output contents into a metadata + my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # parse the binned image statistics + $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Command to update the database +my $command = "$dettool -addresidimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $recipe"; +$command .= " -uri $outputName"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); +$command .= $binnedStats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $exp_id and not $no_update) { + my $command = "$dettool -addresidimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_stack.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_stack.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/detrend_stack.pl (revision 22155) @@ -0,0 +1,259 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $det_id, $iter, $class_id, $det_type, $camera, $outroot, $dbname, $reduction, $verbose, $save_temps, + $no_update, $no_op, $redirect ); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'camera|c=s' => \$camera, + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class for processing + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'redirect-output' => \$redirect, +) or pod2usage( 2 ); + + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera --outroot", + -exitval => 3) unless + defined $det_id and + defined $iter and + defined $class_id and + defined $det_type and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); +$det_type = uc($det_type); + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) + or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); + +# optionally redirect the outputs from this script to LOG.IMFILE +$ipprc->redirect_output($logDest) if $redirect; + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, $det_type . '_STACK'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# The output file rule name depends on the detrend type +my $FILERULES = { 'FLATMASK' => 'PPMERGE.OUTPUT.MASK', + 'DARKMASK' => 'PPMERGE.OUTPUT.MASK', + 'MASK' => 'PPMERGE.OUTPUT.MASK', + 'BIAS' => 'PPMERGE.OUTPUT.BIAS', + 'DARK' => 'PPMERGE.OUTPUT.DARK', + 'DARK_PREMASK' => 'PPMERGE.OUTPUT.DARK', + 'SHUTTER' => 'PPMERGE.OUTPUT.SHUTTER', + 'FLAT' => 'PPMERGE.OUTPUT.FLAT', + 'FLAT_PREMASK' => 'PPMERGE.OUTPUT.FLAT', + 'DOMEFLAT' => 'PPMERGE.OUTPUT.FLAT', + 'SKYFLAT' => 'PPMERGE.OUTPUT.FLAT', + 'FRINGE' => 'PPMERGE.OUTPUT.FRINGE', + }; +my $output_filerule = $FILERULES->{$det_type}; # File rule for output +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $output_filerule; + + +# Get list of files to stack +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -included"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $iter, $class_id, $error_code); + } + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Generate MDC file with the inputs +my ($listFile, $listName) = $ipprc->create_temp_file("$outroot.$class_id.list", $save_temps); + +my $num = 0; +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PPIMAGE.OUTPUT.MASK", $file->{path_base}, $class_id ); # Mask name + my $weight = $ipprc->filename( "PPIMAGE.OUTPUT.WEIGHT", $file->{path_base}, $class_id ); # Weight name + + &my_die("Image $image does not exist", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + + if ($ipprc->file_exists( $mask )) { + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + } + if ($ipprc->file_exists( $weight )) { + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + } + + print $listFile "END\n\n"; +} +close $listFile; + + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +my $outputStack = $ipprc->filename($output_filerule, $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Output name +my $outputCount = $ipprc->filename("PPMERGE.OUTPUT.COUNT", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Count image +my $outputSigma = $ipprc->filename("PPMERGE.OUTPUT.SIGMA", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Stdev image +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Statistics name +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR); # Trace messages + +$command = "$ppMerge $listName $outroot"; # Command to run +$command .= " -recipe PPMERGE $recipe"; +$command .= ' -type ' . uc($det_type); # Type of stacking to perform +$command .= " -stats $outputStats"; # Statistics output filename +$command .= " -recipe PPSTATS CHIPSTATS"; +$command .= " -tracedest $traceDest -log $logDest"; +$command .= " -dbname $dbname" if defined $dbname; + +# Stack the files +unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppMerge: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Unable to find expected output file: $outputStack\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStack); + &my_die("Unable to find expected output file: $outputCount\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputCount); + &my_die("Unable to find expected output file: $outputSigma\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputSigma); + &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + + # Get the statistics on the stacked image + open(my $statsFile, $ipprc->file_resolve("$outputStats")) or + &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + my $contents = do { local $/; <$statsFile> }; # Contents of file + close($statsFile); + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse($contents) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + + $stats->parse($metadata) or + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); +} + +# Command to update the database +$command = "$dettool -addstacked"; +$command .= " -det_id $det_id -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputStack"; +$command .= " -recip $recipe"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the resultant into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addstacked: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $det_id and defined $iter and defined $class_id and not $no_update) { + my $command = "$dettool -addstacked"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/diff_skycell.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/diff_skycell.pl (revision 22155) @@ -0,0 +1,261 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($diff_id, $dbname, $outroot, $verbose, $no_update, $no_op); +GetOptions( + 'diff_id|d=s' => \$diff_id, # Diff identifier + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --diff_id --outroot", + -exitval => 3, + ) unless defined $diff_id + and defined $outroot; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC DIFFTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "DT_SUB", type => "sum", flag => "-dtime_diff", dtype => "float" }, + { name => "SUBTRACTION.NUM", type => "mean", flag => "-stamps_num", dtype => "int" }, + { name => "SUBTRACTION.RMS", type => "mean", flag => "-stamps_rms", dtype => "float" }, + { name => "NUM_SOURCES", type => "sum", flag => "-sources", dtype => "int" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$difftool -inputskyfile -diff_id $diff_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $diff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $diff_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Subtraction list contains more than two elements", $diff_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files == 2; + +# Identify the input and the template +my ($input, $inputMask, $inputWeight, $inputPath); # Input files and path +my ($template, $templateMask, $templateWeight, $templatePath, $templateSources); # Template files and path +my $tess_id; # Tesselation identifier +my $skycell_id; # Skycell identifier +my $camera; # Camera +foreach my $file (@$files) { + if (defined $file->{template} and $file->{template}) { + $template = $file->{uri}; + $templatePath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $templateMask = "PPSTACK.OUTPUT.MASK"; + $templateWeight = "PPSTACK.OUTPUT.WEIGHT"; + $templateSources = "PSPHOT.OUTPUT"; + } else { + $templateMask = "PSWARP.OUTPUT.MASK"; + $templateWeight = "PSWARP.OUTPUT.WEIGHT"; + $templateSources = "PSWARP.OUTPUT.SOURCES"; + } + } else { + $input = $file->{uri}; + $inputPath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $inputMask = "PPSTACK.OUTPUT.MASK"; + $inputWeight = "PPSTACK.OUTPUT.WEIGHT"; + } else { + $inputMask = "PSWARP.OUTPUT.MASK"; + $inputWeight = "PSWARP.OUTPUT.WEIGHT"; + } + } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } + +} + +&my_die("Unable to identify template", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $template; +&my_die("Unable to identify input", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $input; +&my_die("Unable to identify camera", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +$templateMask = $ipprc->filename($templateMask, $templatePath); +$inputMask = $ipprc->filename($inputMask, $inputPath); +$templateWeight = $ipprc->filename($templateWeight, $templatePath); +$inputWeight = $ipprc->filename($inputWeight, $inputPath); +$templateSources = $ipprc->filename($templateSources, $templatePath); + +&my_die("Couldn't find input: $template", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($template); +&my_die("Couldn't find input: $templateMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateMask); +&my_die("Couldn't find input: $templateWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateWeight); +&my_die("Couldn't find input: $input", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputWeight); +&my_die("Couldn't find input: $templateSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateSources); + +# Get the output filenames +my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot); +my $outputWeight = $ipprc->filename("PPSUB.OUTPUT.WEIGHT", $outroot); +my $outputSources = $ipprc->filename("PSPHOT.OUTPUT", $outroot); +#my $bin1Name = $ipprc->filename("PPSUB.BIN1", $outroot); +#my $bin2Name = $ipprc->filename("PPSUB.BIN2", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +# Perform subtraction +unless ($no_op) { + my $command = "$ppSub $input $template $outroot"; + $command .= " -inmask $inputMask"; + $command .= " -refmask $templateMask"; + $command .= " -inweight $inputWeight"; + $command .= " -refweight $templateWeight"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -sources $templateSources"; + $command .= " -photometry"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $diff_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the residual image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $diff_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $diff_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -uri $outputName -path_base $outroot"; + $command .= $stats->cmdflags(); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $error_code); + } + } + + # Register the run as completed + { + my $command = "$difftool -updaterun -diff_id $diff_id -state stop"; # Command to run difftool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform difftool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $diff_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $diff_id and not $no_update) { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ds9_cmf_regions.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ds9_cmf_regions.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ds9_cmf_regions.pl (revision 22155) @@ -0,0 +1,134 @@ +#!/usr/bin/env perl + +use strict; +use Astro::FITS::CFITSIO qw( :constants ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; +use Carp; + +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +my $xpaset = `which xpaset`; +my $xpaget = `which xpaget`; + +die "Unable to find xpaget and xpaset.\n" unless ($xpaset =~ /\S+/ and $xpaget =~ /\S+/); + + +my ( $filename, # Filename containing photometry + $extname, # Extension name containing photometry + $frame, # Frame number in ds9 + $colour, # Region colour + $flag_colour, # Flagged source region colour + $flag, # Flags + $radius, # Radius for circle + $save_temps + ); + +# Defaults +$colour = "blue"; +$flag_colour = "red"; +$radius = 5; +$flag = 0x3888; + +GetOptions( + 'file=s' => \$filename, + 'ext=s' => \$extname, + 'frame=s' => \$frame, + 'colour=s' => \$colour, + 'flag-colour=s' => \$flag_colour, + 'flag=o' => \$flag, + 'radius=f' => \$radius, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file --ext", + -exitval => 3) + unless defined $filename + and defined $extname; + +my $status; # Status of FITSIO calls +my $fits = Astro::FITS::CFITSIO::open_file( $filename, READONLY, $status ); # FITS file handle +check_fitsio($status); +$fits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); +my $numRows; # Number of rows in table +$fits->get_num_rows($numRows, $status) and check_fitsio($status); + +my ($xCol, $yCol, $flagCol); # Column numbers for x,y, flag +$fits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'FLAGS', $flagCol, $status) and check_fitsio($status); + +my ($x, $y, $flags); # Coordinates and flags read from table +$fits->read_col(TFLOAT, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); +$fits->read_col(TFLOAT, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); +$fits->read_col(TINT, $flagCol, 1, 1, $numRows, 0, $flags, undef, $status) and check_fitsio($status); +$fits->close_file($status); + +my ($coordFile, $coordName) = tempfile( "/tmp/ds9_cmf_regions.XXXX", UNLINK => !$save_temps ); +for (my $i = 0; $i < $numRows; $i++) { + print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$radius \# color = " . + (($$flags[$i] & $flag) ? $flag_colour : $colour) . "\n"; +} +close $coordFile; + +my @settings = settings_save("regions format", + "regions system"); # Settings to save + +xpaset("frame $frame") if defined $frame; +xpaset("regions format ds9"); +xpaset("regions system image"); +xpaset("regions load $coordName"); + +xpaset(@settings); + +print "Plotted $numRows sources.\n"; + +### Pau. + + + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + croak "CFITSIO error: $msg\n"; + } +} + +# Save specified settings +sub settings_save +{ + my @settings; # Values of settings + foreach my $setting (@_) { + my @values = xpaget($setting); + push @settings, $setting . ' ' . shift @values; + } + return @settings; +} + + +# XPA subroutines courtesy Derek Fox +sub xpaset { + foreach my $cmd (@_) { + system("xpaset -p ds9 $cmd"); + } +} +sub xpaget { + my @out; + foreach my $cmd (@_) { + my $output = `xpaget ds9 $cmd`; + push @out, $output; + } + return @out; +} + + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/fake_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/fake_imfile.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/fake_imfile.pl (revision 22155) @@ -0,0 +1,237 @@ +#!/usr/bin/env perl + +## this script is run on every imfile to perform the fake source +## analysis and the forced photometry analysis. + +## For the fake source analysis, we load the image, inject a number of +## fake source (with a flat mag distribution), save the input source, +## then perform photometry on the image to recover the input sources. +## we should save a file with the matched input and recovered +## sources. the output metadata should include an analysis of the +## recovery rate and measurement error as a function of magnitude + +## For the forced photometry, we need to use the measured astrometry +## to select the desired locations in pixel coord. + +# XXX : seeing needs to be determined from the input PSF (not currently done in ppSim) +# ppSim output -input input.fits -cmf input.cmf -psf input.psf -seeing 0.563760 -recipe PPSIM FAKEPHOT + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ( $exp_id, $fake_id, $class_id, $chiproot, $camroot, $camera, $outroot, + $dbname, $reduction, $verbose, $no_update, $no_op ); +GetOptions( + 'exp_id=s' => \$exp_id, # Exposure identifier + 'fake_id=s' => \$fake_id, # Chiptool identifier + 'class_id=s' => \$class_id, # Class identifier + 'chiproot=s' => \$chiproot, # Input Chip files (root) + 'camroot=s' => \$camroot, # Input Camera files (root) + 'camera|c=s' => \$camera, # Camera + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --fake_id --class_id --chiproot --camroot --camera --outroot", + -exitval => 3) unless + defined $exp_id and + defined $fake_id and + defined $class_id and + defined $chiproot and + defined $camroot and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, 'FAKEPHOT'); # Recipe to use +unless ($recipe) { + &my_die("Couldn't find selected reduction for FAKE: $reduction\n", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# values to extract from output metadata and the stats to calculate +# XXX not sure if this is needed by fake_imfile.pl +# my $STATS = +# [ +# # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG +# { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, +# { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, +# { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# ]; +# my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $faketool = can_run('faketool') or (warn "Can't find faketool" and $missing_tools = 1); +my $ppSim = can_run('ppSim') or (warn "Can't find ppSim" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# outroot examples (HOST components must be set) +# file://data/ipp004.0/gpc1/20080130 +# neb:///ipp004-v1/gpc1/20080130 +# neb:///*/gpc1/20080130 (volume not specified) + +# check for existing directory, generate if needed +$ipprc->outroot_prepare($outroot); + +## these names are used in ppImage, and thus may be URIs +my $chipImage = $ipprc->filename("PPIMAGE.CHIP", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $chipMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $chipWeight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $chipPSF = $ipprc->filename("PSPHOT.PSF.SAVE", $chiproot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $cameraObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +# XXX check for existence of input data +# &my_die("Couldn't find input file: $uri\n", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + +## get the PPSIM recipe for this camera and FAKEPHOT reduction +my $command = "$ppConfigDump -camera $camera -dump-recipe PPSIM -recipe PPSIM $recipe -"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR); +} +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR); + +## allow the output images to be optional, depending on the recipe / reduction class +my $skipFake = metadataLookupBool($recipeData, 'SKIP.FAKE'); + +# Run ppSim +unless ($no_op || $skipFake) { + # examine the PPSIM recipe to decide if we need to run this or opt out + + my $command = "$ppSim $outroot"; + $command .= " -input $chipImage"; + # XXX add input mask and weight to ppSim + # $command .= " -mask $chipMask"; + # $command .= " -weight $chipWeight"; + $command .= " -cmf $cameraObjects"; + $command .= " -psf $chipPSF"; + $command .= " -recipe PPSIM $recipe"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -tracedest $traceDest"; + $command .= " -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSim: $error_code", $exp_id, $fake_id, $class_id, $error_code); + } + + # XXX check for output files? + # &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + + # XXX use this to parse the output metadata : eg, detection limits, Nfakes, Nforced + # Get the statistics on the processed image + # my $statsFile; # File handle + # open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR); + # my @contents = <$statsFile>; # Contents of file + # close $statsFile; + # + # # parse the statistics MDC file + # my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + # my $metadata = $mdcParser->parse(join "", @contents); + # unless ($metadata) { + # &my_die("Unable to parse metadata config doc", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR); + # } + # + # # extract the stats from the metadata + # unless ($stats->parse($metadata)) { + # &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $fake_id, $class_id, $PS_EXIT_PROG_ERROR); + # } +} else { + print "skipping ppSim processing\n"; +} + +# command to update database +$command = "$faketool -addprocessedimfile"; +$command .= " -fake_id $fake_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -path_base $outroot"; +$command .= " -dbname $dbname" if defined $dbname; +# XXX add this after defined +# $command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform faketool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; # rawExp identifier + my $fake_id = shift; # fakeRun identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $exp_id and defined $fake_id and defined $class_id and not $no_update) { + my $command = "$faketool -addprocessedimfile"; + $command .= " -fake_id $fake_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outroot"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/flatcorr_init.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/flatcorr_init.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/flatcorr_init.pl (revision 22155) @@ -0,0 +1,212 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_init.pl +## given dbname, dvodb, filter, time range, camera, telescope, etc? +## select matching images and register as a new flatcorr run + +## should this be implemented as a ippTool program? +## this is a lot like dettool -definebyquery +## flatcorr -definebyquery +## this should queue the selected images with a mode that stops at camera +## ?? how do we block this batch from running through warp ?? + +## we also need the test / query tool: +## flatcorr -pendingcorr +## examine the chip & camera stages and check for all images to have completed (successfully or not) + +## allow a certain fraction of failures... + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dvo_id, $catdir, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'dvo_id|i=s' => \$dvo_id, + 'catdir|c=s' => \$catdir, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dvo_id --catdir --region", + -exitval => 3) unless + defined $dvo_id and + defined $catdir and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addcalrun"; +$command .= " -dvo_id $dvo_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -status SUCCESS"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/flatcorr_proc.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/flatcorr_proc.pl (revision 22155) @@ -0,0 +1,205 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_proc.pl --dbname --corr_id + +## this script does the following steps: + +# extract the details of the flatcorr run: dvodb, filter, camera, etc? + +# relphot -D CATDIR $dvodb -grid (outgrid.fits) (filter) -region 0 360 -90 90 (other parameters?) + +# dvoMakeCorr -file outgrid.fits -ref ref.fits outcorr + +# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc +# foreach $imfile () +# dettool -register_imfile -uri, etc, etc + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($corr_id, $dvodb, $region, $filter, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'corr_id|i=s' => \$corr_id, + 'dvodb|c=s' => \$dvodb, + 'region|r=s' => \$region, + 'filter|f=s' => \$filter, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --corr_id --dvodb --region --filter", + -exitval => 3) unless + defined $corr_id and + defined $dvodb and + defined $region and + defined $filter; + +# Look for programs we need +my $missing_tools; +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $dvoMakeCorr = can_run('dvoMakeCorr') or (warn "Can't find dvoMakeCorr" and $missing_tools = 1); +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $flatcorr = can_run('flatcorr') or (warn "Can't find flatcorr" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +my($outgrid, $outcorr); + +$outgrid = "$workdir/grid.$corr_id.fits"; +$outcorr = "$workdir/corr.$corr_id.fits"; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run relphot (filter) for the specified region (need to clarify the options like imfreeze) +{ + my $command = "$relphot $filter"; + $command .= "-D CATDIR $dvodb"; + $command .= "-region $RAs $RAe $DECs $DECe"; + $command .= "-grid $outgrid"; + $command .= "-imfreeze?"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform relphot -grid on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } +} + +# use one of the input raw images as a reference image +# XXX I'm not sure how this works: do I run this once per imfile for a given exp? +my ($reffile) +{ + my $command = "$flatcoor -flatcorrimfile -limit 1"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform flatcorr -flatcorrimfile: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dvo_id, $region, "RELPHOT", $status, $dbname, $PS_EXIT_PROG_ERROR);); + + my $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $dvo_id, $region, "RELPHOT", $status, $dbname, $PS_EXIT_PROG_ERROR);); + + my $imfile = $imfiles->[0]; + $reffile = $imfile->{uri}; +} + +{ + my $command = "$dvoMakeCorr -file $outgrid -ref $reffile $outcorr"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform dvoMakeCorr: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$dettool -register"; + $command .= "-det_type FLATCORR"; + $command .= "-file_level $fileLevel"; + $command .= "-workdir $workdir"; + $command .= "-inst $inst"; + $command .= " -dbname $dbname" if defined $dbname; + ## XXX what else do we need, and where do we get it? + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$flatcorr -done"; +$command .= " -corr_id $corr_id"; +$command .= " -stats UNKNOWN"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if (defined $dvo_id && defined $region && defined $last_step && defined $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/gpc_seeing.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/gpc_seeing.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/gpc_seeing.pl (revision 22155) @@ -0,0 +1,28 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: gpc_seeing.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OP'; +$RECIPE_PSPHOT = 'PSPHOT.SEEING'; + +# recommend only processing to PSFMODEL +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +# XXX otis can read the output psf model, or we can supply a program to interpret the model + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_cleanup.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_cleanup.pl (revision 22155) @@ -0,0 +1,155 @@ +#!/usr/bin/env perl + +# this script is used to cleanup the files from the different ipp +# stages. It can be called with one of two cleanup modes: clean and +# purge. the former removes temporary data files, leaving behind +# enough information for the results to be rebuilt. The latter +# removes all but basic logging data. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +# Parse the command-line arguments +my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $no_op, $help); +GetOptions('stage=s' => \$stage, # which analysis stage to clean? + 'camera|i=s' => \$camera, # user-supplied camera name + 'stage_id=s' => \$stage_id, # id for this stage (only needed for certain stages) + 'mode|m=s' => \$mode, # cleanup mode (clean / purge) + 'path_base=s' => \$path_base, # basename for files + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id stage_id --mode (mode) --path_base (path) [--dbname dbname] [--no-op] [--help]", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options:--camera (name) --stage (stage) --mode (mode) --path_base (path)", + -exitval => 3) unless + defined $camera and + defined $stage and + defined $mode and + defined $path_base; + +# $mode must be one of "clean" or "purge" +unless (($mode eq "clean") || ($mode eq "purge")) { + die "invalid cleanup mode $mode\n"; +} + +$ipprc->define_camera($camera); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $found = 0; + +# choice of files to delete depends on the stage +if ($stage eq "chip") { + + # select the imfiles for this entry + # dettool -pendingcleanupimfile -chip_id $stage_id + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanupimfile -chip_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", "chip", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "chip", $cam_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.WEIGHT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP.WEIGHT", $path_base, $class_id); + if ($mode eq "purge") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + + if (! $status) { + my $command = "$dettool -updateimfile -chip_id $stage_id -class_id $class_id -code 55"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + # call the command + $fault = 1; + } + } + + if (! $fault) { + my $command = "$dettool -updaterun -chip_id $chip_id -state clean";; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + # need to run command "chiptool -donecleanup" + } + exit 0; +} + +die "unknown stage for ipp_cleanup.pl\n"; +exit 1; + +sub delete_files +{ + my @files = shift; # reference to a list of files to unlink + + # this script is, of course, very dangerous. + foreach my $file (@$files) { + echo "unlinking $file\n"; + # unlink $file; + } + return 1; +} + +sub addFilename +{ + my $files = shift; # reference to a list of files to unlink + my $filerule = shift; # filerule to add + my $path_base = shift; # base filename + my $class_id = shift; # class_id, if needed + + my $file = $ipprc->filename($filerule, $path_base, $class_id); + + push @$files, $file; + return 1; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_darkstats.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_darkstats.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_darkstats.pl (revision 22155) @@ -0,0 +1,188 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_darkstats.pl --dbname (name) --det_id (id) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_datapath.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_datapath.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_datapath.pl (revision 22155) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +my $touch; + +GetOptions( + 'touch' => \$touch, +); + +die "No filename specified.\n" if scalar @ARGV != 1; + +my $filename = shift @ARGV; + +my $resolved = $ipprc->file_resolve($filename, $touch); +print "$resolved\n" if ($resolved); + +1; + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_detrend_combine.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_detrend_combine.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_detrend_combine.pl (revision 22155) @@ -0,0 +1,220 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Data::Dumper; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use PS::IPP::Metadata::Stats; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +# Parse command-line arguments +my ($det_type, $filelevel, $inst, $telescope, $filter, + $det_id1, $iter1, $det_id2, $iter2, $operation, $mask, + $workdir, $dbname, $no_update); +GetOptions( + 'det_type=s' => \$det_type, # Detrend type for new detrend + 'filelevel=s' => \$filelevel, # File level for new detrend + 'inst=s' => \$inst, # Instrument for new detrend + 'telescope=s' => \$telescope, # Telescope for new detrend + 'filter=s' => \$filter, # Filter name for new detrend + 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 + 'iteration1=s' => \$iter1, # Iteration for detrend 1 + 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 + 'iteration2=s' => \$iter2, # Iteration for detrend 2 + 'operation=s' => \$operation, # Operation to perform on files + 'mask' => \$mask, # Operation is on a mask + 'workdir=s' => \$workdir, # Working directory for output files + 'dbname=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_type --filelevel --inst --telescope --det_id1 --iteration1 --det_id2 --iteration2 --workdir", + -exitval => 3, + ) + unless defined $det_type + and defined $filelevel + and defined $inst + and defined $telescope + and defined $det_id1 + and defined $iter1 + and defined $det_id2 + and defined $iter2 + and defined $operation + and defined $workdir; + +$ipprc->define_camera($inst); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; + +# Look for programs we need +my $missing_tools; +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppArith = can_run('ppArith') or (warn "Can't find ppArith" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my $files1 = filelist($det_id1, $iter1); # Hash of input files for detrend 1 +my $files2 = filelist($det_id2, $iter2); # Hash of input files for detrend 2 +die("File lists for detrends have differing lengths") unless scalar keys %$files1 == scalar keys %$files2; + +my ($det_id, $iter); # Detrend identifier for the new detrend +unless ($no_update) { + my $command = "$dettool -register_detrend -det_type $det_type -filelevel $filelevel -workdir $workdir " . + "-inst $inst -telescope $telescope"; # Command to run + $command .= " -filter $filter" if defined $filter; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend: $error_code"); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or die("Unable to parse metadata config doc\n"); + my $md = parse_md_list($metadata) or die("Unable to parse metadata list\n"); + + $det_id = $$md[0]->{det_id}; + $iter = $$md[0]->{iteration}; + + die("Unable to get det_id and iteration for new detrend.\n") unless defined $det_id and defined $iter; +} else { + $det_id = 'DUMMY_DET_ID'; + $iter = 'DUMMY_ITER'; +} + +my $outRoot = caturi($workdir, "$inst.$det_id.$iter"); # Output root name +my $filerule = (defined $mask ? "PPARITH.OUTPUT.MASK" : "PPARITH.OUTPUT.IMAGE"); # File rule for ppArith + +foreach my $class_id ( keys %$files1 ) { + my $md1 = $$files1{$class_id}; + my $md2 = $$files2{$class_id}; + die("Class_id=$class_id not defined for det_id=$det_id2") unless defined $md2; + + my $uri1 = $$md1[0]->{uri}; + my $uri2 = $$md2[0]->{uri}; + + die("Unable to find input file $uri1\n") unless $ipprc->file_exists($uri1); + die("Unable to find input file $uri2\n") unless $ipprc->file_exists($uri2); + + my $outName = $ipprc->filename($filerule, $outRoot, $class_id); + my $outStats = $outRoot . '.stats'; + + my $command = "$ppArith -file1 $uri1 -op \'$operation\' -file2 $uri2 $outRoot"; # Command to run + $command .= " -stats $outStats -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= ' -mask' if defined $mask; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run ppArith: $error_code"); + } + + die("Unable to find ppArith product: $outName\n") unless $ipprc->file_exists($outName); + die("Unable to find ppArith product: $outStats\n") unless $ipprc->file_exists($outStats); + + # Get the statistics on the processed image + my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + { + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); + + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n"); + } + } + + # Register the imfile + unless ($no_update) { + my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run + $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend_imfile: $error_code"); + } + } +} + + +### Pau. + + +# Get a list of files for the given detrend +sub filelist +{ + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + + my $command = "$detselect -select -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run detselect: $error_code"); + } + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my %files; + + my $md = $mdcParser->parse( join( "", @$stdout_buf ) ); # Parsed metadata + my $list = parse_md_list( $md ); + + foreach my $item ( @$list ) { + my $class_id = $item->{class_id}; + die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if + defined $files{$class_id}; + $files{$class_id} = parse_md_list($md); + } + + return \%files; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_filename.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_filename.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_filename.pl (revision 22155) @@ -0,0 +1,52 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# use these to check the apache environment +#print "$ENV{'PATH'}\n"; +#print "$ENV{'PERL5LIB'}\n"; + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +my ($filerule, $class_id, $basename, $camera, $touch); + +GetOptions('filerule=s' => \$filerule, + 'class_id=s' => \$class_id, + 'basename=s' => \$basename, + 'camera|c=s' => \$camera, + 'touch' => \$touch, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --filerule --class_id --basename --camera", + -exitval => 3, +) unless defined $basename + and defined $filerule + and defined $class_id + and defined $camera; + +$touch = 0 unless (defined $touch); + +$ipprc->define_camera($camera); + +# print "$filerule\n"; +# print "$basename\n"; +# print "$camera\n"; +# print "$class_id\n"; + +my $filename = $ipprc->filename($filerule, $basename, $class_id); +# print "$filename\n"; +# print "touch: $touch\n"; + +my $realname = $ipprc->file_resolve( $filename, $touch ); +print "$realname\n"; + +1; + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_image_path.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_image_path.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_image_path.pl (revision 22155) @@ -0,0 +1,128 @@ +#!/usr/bin/env perl + +# ipp_image_path.pl print out the unix path name(s) for rawImage files for an exposure +# +# XXX TODO add argument to get paths to chip processed images + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use DBI; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($exp_name, $class_id, $from_registered, $dbname, $verbose); + +GetOptions( + 'exp_name|e=s' => \$exp_name, + 'class_id|c=s' => \$class_id, + 'registered|r' => \$from_registered, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_name", + -exitval => 3) unless + defined $exp_name; + +my $ota_num; +if ($class_id) { + $class_id = lc($class_id); + my $extra; + (undef, $ota_num, $extra) = $class_id =~ m/(\D*)(\d\d)(.*)/; + if ($extra || !$ota_num) { + print STDERR "$class_id is not a valid class_id\n"; + exit 1; + } +} + +# Look for commands we need +my $missing_tools; +my $neb_locate = can_run('neb-locate') + or (warn "can't find neb-locate" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $ipprc = PS::IPP::Config->new(); +my $dbh = getDBHandle(); + +# Get the list of imfiles +{ + # XXX: If there are multiple exposures with the same exposure name + # this query will return them all + my $query; + if ($from_registered) { + $query = "SELECT exp_id, class_id, uri FROM rawImfile" + . " WHERE exp_name = \'$exp_name\'"; + $query .= " AND class_id = \'xy$ota_num\'" if ($ota_num); + } else { + $query = "SELECT exp_id FROM newExp WHERE tmp_exp_name = ?"; + my $stmt = $dbh->prepare($query); + $stmt->execute($exp_name); + my $exp_ref = $stmt->fetchrow_hashref(); + if (!$exp_ref) { + print STDERR "exposure $exp_name not found\n"; + exit 1; + } + $stmt->finish(); + $query = "SELECT exp_id, tmp_class_id, uri FROM newImfile WHERE exp_id = $exp_ref->{exp_id}"; + $query .= " AND tmp_class_id = \'ota$ota_num\'" if ($ota_num); + } + + my $stmt = $dbh->prepare($query); + $stmt->execute(); + my $nfiles = 0; + while (my $ref = $stmt->fetchrow_hashref()) { + my $uri = $ref->{uri}; + my $path; + my ($scheme) = $uri =~/^(path|neb|file):/; + + if (!$scheme) { + $path = $uri; + } else { + $path = $ipprc->file_resolve($uri); + } + if ($path) { + # remove the leading "file://" if present + $path =~ s/^file\:\/\///; + } + $nfiles++; + print "$path\n"; + } + if (!$nfiles) { + my $cstr = defined($class_id) ? $class_id . " " : ""; + print STDERR "exposure $exp_name " . $cstr . "not found\n"; + exit 2; + } +} + +sub getDBHandle { + my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER"); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD"); + if (!$dbname) { + $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME"); + } + + die "database configuration set up" unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_inject_expname.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_inject_expname.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_inject_expname.pl (revision 22155) @@ -0,0 +1,190 @@ +#!/usr/bin/env perl + +# this program injects a set of files the db, assuming the list of +# files all belong to a single exposure the user supplies a temporary +# telescope and camera name. these are used for informational +# purposes only until the registration step can determine the true +# telescope and camera name from the image headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $from_db, $dbname, $no_op, $verbose, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'from=s' => \$from_db, # Source Database name + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "inject files based on query to another IPP db", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 [--from dbname] [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (exp_name)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( + -msg => "Required options: --exp_tag --cam_id --camera --outroot", + -exitval => 3, + ) unless + defined $from_db; + +my $exp_name = $ARGV[0]; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; +my $regtool = can_run('regtool') or die "Can't find regtool\n"; + +# tess_id needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($tess_id) { + $tess_id = &fixpath ($tess_id); +} + +# dvodb needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($dvo_db) { + $dvo_db = &fixpath ($dvo_db); +} + +# workdir needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($workdir) { + $workdir = &fixpath ($workdir); +} + +# if workdir is not defined, assign the current path +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +if (! $telescope) { + $telescope = "UNKNOWN"; +} + +if (! $camera) { + $camera = "UNKNOWN"; +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# query the from_db for the desired imfiles: +# regtool -dbname $from_db -processedimfile -exp_name $exp_name + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$regtool -dbname $from_db -processedimfile -exp_name $exp_name"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die "Unable to perform regtool for $exp_name: $error_code"; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + die "Unable to parse metadata config doc $exp_name: $error_code"; + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + die "Unable to parse metadata list: $exp_name: $error_code"; +} + +## inject the exposure + +# the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values +my $command_exp = "$pxinject -newExp"; +$command_exp .= " -tmp_exp_name $exp_name"; +$command_exp .= " -tmp_inst $camera"; +$command_exp .= " -tmp_telescope $telescope"; +$command_exp .= " -workdir $workdir"; +$command_exp .= " -reduction $reduction" if defined $reduction; +$command_exp .= " -dvodb $dvo_db" if defined $dvo_db; +$command_exp .= " -tess_id $tess_id" if defined $tess_id; +$command_exp .= " -end_stage $end_stage" if defined $end_stage; +$command_exp .= " -label $label" if defined $label; +$command_exp .= " -dbname $dbname" if defined $dbname; + +my $exp_id = 0; +unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag +} else { + print "skipping command: $command_exp\n"; +} + +# now inject the imfiles one at a time +foreach my $file (@$files) { + + my $uri = $file->{uri}; + my $class_id = $file->{class_id}; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id $class_id"; + $command_imfile .= " -uri $uri"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } +} + +# the class_id used here is temporary : register replaces it with the true class_id +my $command_update = "$pxinject -updatenewExp"; +$command_update .= " -exp_id $exp_id"; +$command_update .= " -state run"; +$command_update .= " -dbname $dbname" if defined ($dbname); + +unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; +} else { + print "skipping command: $command_update\n"; +} + +exit 0; + +sub fixpath { + my $path = shift; + my $valid = 0; + $valid |= ($path =~ m|^file://|); + $valid |= ($path =~ m|^path://|); + $valid |= ($path =~ m|^neb://|); + $valid |= ($path =~ m|^/|); + + if (!$valid) { + $path = File::Spec->rel2abs( $path ); + } + return $path; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_inject_fileset.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_inject_fileset.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_inject_fileset.pl (revision 22155) @@ -0,0 +1,177 @@ +#!/usr/bin/env perl + +# this program injects a set of files the db, assuming the list of +# files all belong to a single exposure the user supplies a temporary +# telescope and camera name. these are used for informational +# purposes only until the registration step can determine the true +# telescope and camera name from the image headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing + ) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +# XXX why are these required? +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless @ARGV > 0; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# tess_id needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($tess_id) { + $tess_id = &fixpath ($tess_id); +} + +# dvodb needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($dvo_db) { + $dvo_db = &fixpath ($dvo_db); +} + +# workdir needs to be an absolute path, or it must start with file://, path://, or neb:// +if ($workdir) { + $workdir = &fixpath ($workdir); +} + +# if workdir is not defined, assign the current path +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +if (! $telescope) { + $telescope = "UNKNOWN"; +} + +if (! $camera) { + $camera = "UNKNOWN"; +} + +# use the first file name as the exp_name (strip off .fits) +my $num = 0; +my $exp_name; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + if (! $exp_name) { + # strip off the extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $file ); + ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + print "exp_name : $exp_name.\n"; + } + $num ++; +} + +print "$num files in fileset.\n"; + +## inject the exposure + +# the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values +my $command_exp = "$pxinject -newExp"; +$command_exp .= " -tmp_exp_name $exp_name"; +$command_exp .= " -tmp_inst $camera"; +$command_exp .= " -tmp_telescope $telescope"; +$command_exp .= " -workdir $workdir"; +$command_exp .= " -reduction $reduction" if defined $reduction; +$command_exp .= " -dvodb $dvo_db" if defined $dvo_db; +$command_exp .= " -tess_id $tess_id" if defined $tess_id; +$command_exp .= " -end_stage $end_stage" if defined $end_stage; +$command_exp .= " -label $label" if defined $label; +$command_exp .= " -dbname $dbname" if defined $dbname; + +my $exp_id = 0; +unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag +} else { + print "skipping command: $command_exp\n"; +} + +# now inject the imfiles one at a time +for (my $i = 0; $i < @ARGV; $i++) { + + my $file = $ARGV[$i]; + + my $absfile = File::Spec->rel2abs( $file ); + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id file.$i"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } +} + +# the class_id used here is temporary : register replaces it with the true class_id +my $command_update = "$pxinject -updatenewExp"; +$command_update .= " -exp_id $exp_id"; +$command_update .= " -state run"; +$command_update .= " -dbname $dbname" if defined ($dbname); + +unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; +} else { + print "skipping command: $command_update\n"; +} + +exit 0; + +sub fixpath { + my $path = shift; + my $valid = 0; + $valid |= ($path =~ m|^file://|); + $valid |= ($path =~ m|^path://|); + $valid |= ($path =~ m|^neb://|); + $valid |= ($path =~ m|^/|); + + if (!$valid) { + $path = File::Spec->rel2abs( $path ); + } + return $path; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_maskscript.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_maskscript.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_maskscript.pl (revision 22155) @@ -0,0 +1,190 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_maskscript.pl --dbname (name) --det_id (id) --iter (iteration) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +# I could determine the camera from a query for the detrun +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_mops_translate.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_mops_translate.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_mops_translate.pl (revision 22155) @@ -0,0 +1,255 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Math::Trig; +use Data::Dumper; + +use constant EXTNAME => 'MOPS_TRANSIENT_DETECTIONS'; # Extension name for output table +use constant POSITION_ERROR => 0.2 / 3600; # Error in positions, degrees +use constant ZERO_POINT => 25; # Magnitude zero point +use constant MAG_ERROR => 0.1; # Error in magnitudes +use constant OBSERVATORY_CODE => 566; # IAU Observatory Code +use constant FAKE_LIMITING_MAG => 23.0; # Fake limiting magnitude to report +use constant FAKE_DETECTION_EFFICIENCY => 0.0; # Fake detection efficiency to report + +my ( $input, # Name of input file with IPP photometry + $extname, # Name of extension containing photometry + $skycell, # Skycell file with WCS + $output, # Name of output file + $save_temps, # Save temporary files? + ); + +GetOptions( + 'input=s' => \$input, + 'extname=s' => \$extname, + 'skycell=s' => \$skycell, + 'output=s' => \$output, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --input --extname --skycell --output", + -exitval => 3) + unless defined $input + and defined $extname + and defined $skycell + and defined $output; + +# Specification of columns to write +my $columns = [ { name => 'RA_DEG', type => 'D' }, # Right ascension + { name => 'DEC_DEG', type => 'D' }, # Declination + { name => 'RA_SIG', type => 'D' }, # Error in right ascension + { name => 'DEC_SIG', type => 'D' }, # Error in declination + { name => 'FLUX', type => 'D' }, # Flux + { name => 'FLUX_SIG', type => 'D' }, # Error in flux + { name => 'ANG', type => 'D' }, # Angle + { name => 'ANG_SIG', type => 'D' }, # Error in angle + { name => 'LEN', type => 'D' }, # Length + { name => 'LEN_SIG', type => 'D' }, # Error in length + ]; + +# Header translation table +my $headers = { + 'FPA.RA' => { name => 'RA', type => TSTRING, comment => 'Right ascension of boresight'}, + 'FPA.DEC' => { name => 'DEC', type => TSTRING, comment => 'Declination of boresight' }, + 'FPA.OBS' => { name => 'FPA_ID', type => TSTRING, comment => 'Exposure identifier' }, + 'FPA.FILTER' => { name => 'FILTER', type => TSTRING, comment => 'Filter name' }, + 'EXPTIME' => { name => 'EXPTIME', type => TDOUBLE, comment => 'Exposure time' }, + 'FPA.POSANGLE' => { name => 'ROTANGLE', type => TDOUBLE, comment => 'Position angle' }, + 'FPA.ALT' => { name => 'TEL_ALT', type => TDOUBLE, comment => 'Telescope altitude' }, + 'FPA.AZ' => { name => 'TEL_AZ', type => TDOUBLE, comment => 'Telescope azimuth' }, + 'MJD-OBS' => { name => 'MJD-OBS', type => TDOUBLE, comment => 'Time of exposure' }, + }; + + +# Read the skycell header for the WCS +my ($wcsheader, $status) = Astro::FITS::CFITSIO::fits_read_header( $skycell ); + +# Get the useful header keywords +my $naxis1 = $$wcsheader{'NAXIS1'} or die("Can't find NAXIS1"); +my $naxis2 = $$wcsheader{'NAXIS2'} or die("Can't find NAXIS2"); +my $ctype1 = $$wcsheader{'CTYPE1'} or die("Can't find CTYPE1"); +my $ctype2 = $$wcsheader{'CTYPE2'} or die("Can't find CTYPE2"); +my $cdelt1 = $$wcsheader{'CDELT1'} or die("Can't find CDELT1"); +my $cdelt2 = $$wcsheader{'CDELT2'} or die("Can't find CDELT2"); +my $crval1 = deg2rad($$wcsheader{'CRVAL1'}) or die("Can't find CRVAL1"); +my $crval2 = deg2rad($$wcsheader{'CRVAL2'}) or die("Can't find CRVAL2"); +my $crpix1 = $$wcsheader{'CRPIX1'} or die("Can't find CRPIX1"); +my $crpix2 = $$wcsheader{'CRPIX2'} or die("Can't find CRPIX2"); +my $pc11 = $$wcsheader{'PC001001'} or die("Can't find PC001001"); +my $pc12 = $$wcsheader{'PC001002'} or die("Can't find PC001002"); +my $pc21 = $$wcsheader{'PC002001'} or die("Can't find PC002001"); +my $pc22 = $$wcsheader{'PC002002'} or die("Can't find PC002002"); +my $crota1 = $$wcsheader{'CROTA1'}; +my $crota2 = $$wcsheader{'CROTA2'}; + +die("Unexpected projection: $ctype1 and $ctype2.") unless $ctype1 =~ /^\'RA---TAN\s*\'$/ and + $ctype2 =~ /^\'DEC--TAN\s*\'$/; +die("Can't determine size of skycell ($naxis1,$naxis2)") unless $naxis1 > 0 and $naxis2 > 0; +die("Can't determine scale of skycell ($cdelt1,$cdelt2)") if not defined $cdelt1 or $cdelt1 == 0 or + not defined $cdelt2 or $cdelt2 == 0; +die("We don't know how to handle rotations ($crota1,$crota2)") if defined $crota1 or defined $crota2; + + +# Read the input table +my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle +my $inHeader = $inFits->read_header(); # Header for input +check_fitsio($status); +$inFits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); +my $numRows; # Number of rows in table +$inFits->get_num_rows($numRows, $status) and check_fitsio($status); + +my ($xCol, $yCol, $magCol); # Column numbers for x and y +$inFits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); +$inFits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); +$inFits->get_colnum(0, 'PSF_INST_MAG', $magCol, $status) and check_fitsio($status); + +my ($xType, $yType, $magType); # Column types +$inFits->get_coltype($xCol, $xType, undef, undef, $status) and check_fitsio($status); +$inFits->get_coltype($yCol, $yType, undef, undef, $status) and check_fitsio($status); +$inFits->get_coltype($magCol, $magType, undef, undef, $status) and check_fitsio($status); + +my ($x, $y, $mag); # Sources arrays +$inFits->read_col($xType, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); +$inFits->read_col($yType, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); +$inFits->read_col($magType, $magCol, 1, 1, $numRows, 0, $mag, undef, $status) and check_fitsio($status); + +$inFits->close_file( $status ); + +# Parse the list of columns +my @colNames; # Names of columns +my @colTypes; # Types of columns +my %colData; # Data for each column +foreach my $colSpec ( @$columns) { + push @colNames, $colSpec->{name}; + push @colTypes, $colSpec->{type}; + $colData{$colSpec->{name}} = []; +} + + +# Convert the input data into the output formats +for (my $i = 0; $i < $numRows; $i++) { + my ($ra, $dec) = pixels_to_sky($$x[$i], $$y[$i]); # Sky coordinates + + push @{$colData{'RA_DEG'}}, $ra; + push @{$colData{'DEC_DEG'}}, $dec; + push @{$colData{'RA_SIG'}}, POSITION_ERROR; + push @{$colData{'DEC_SIG'}}, POSITION_ERROR; + push @{$colData{'FLUX'}}, $$mag[$i] + ZERO_POINT; + push @{$colData{'FLUX_SIG'}}, MAG_ERROR; + push @{$colData{'ANG'}}, 0.0; + push @{$colData{'ANG_SIG'}}, 0.0; + push @{$colData{'LEN'}}, 0.0; + push @{$colData{'LEN_SIG'}}, 0.0; +} + + +# Write the output +unlink "$output" if -e "$output"; +my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); # Output file handle +check_fitsio( $status ); + +# Write the table +$outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, EXTNAME, + $status ); +check_fitsio( $status ); + +# Write the header keywords +foreach my $keyword ( keys %$headers ) { + my $value = $inHeader->{$keyword}; # Header keyword value + unless (defined $value) { + print "Can't find header keyword $keyword\n"; + next; + } + $value =~ s/\'//g; + my $name = $headers->{$keyword}->{name}; # New name + my $type = $headers->{$keyword}->{type}; # Type + my $comment = $headers->{$keyword}->{comment}; # Comment + $outFits->write_key( $type, $name, $value, $comment, $status ); + check_fitsio( $status ); +} + +# Stellar PSF +my $fwhm = 0.5 * 3600 * ($inHeader->{'FWHM_X'} * $cdelt1 + $inHeader->{'FWHM_Y'} * $cdelt2); # FWHM for star +$outFits->write_key( TSTRING, 'STARPSF', $fwhm, 'Stellar PSF (arcsec)', $status ); +check_fitsio( $status ); + +# Observatory code +$outFits->write_key( TINT, 'OBSCODE', OBSERVATORY_CODE, 'IAU observatory code', $status ); + +# Limiting magnitude +$outFits->write_key( TINT, 'LIMITMAG', FAKE_LIMITING_MAG, 'Limiting magnitude (FAKE)', $status ); + +# Detection efficiency +$outFits->write_key( TINT, 'DE1', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE2', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE3', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE4', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE5', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE6', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE7', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE8', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE9', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); +$outFits->write_key( TINT, 'DE10', FAKE_DETECTION_EFFICIENCY, 'Detection efficiency (FAKE)', $status ); + +# Write the data +for (my $i = 0; $i < scalar @colNames; $i++) { + my $colName = $colNames[$i];# Column name + $outFits->write_col( TDOUBLE, $i + 1, 1, 1, $numRows, $colData{$colName}, $status ); + check_fitsio( $status ); +} + +$outFits->close_file( $status ); + +### Pau. + + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + die "CFITSIO error: $msg\n"; + } +} + +# Convert pixel coordinates to sky coordinates +sub pixels_to_sky +{ + my ($x, $y) = @_; # Coordinates + + # Pixel coordinate relative to reference + $x -= $crpix1; + $y -= $crpix2; + + # Coordinates on tangent plane + my $xi = $pc11 * ($x) + $pc12 * ($y); + my $eta = $pc21 * ($x) + $pc22 * ($y); + $xi *= $cdelt1; + $eta *= $cdelt2; + + # Coordinates on rotated celestial sphere + my $phi = atan2($eta,$xi) + pi/2; + my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2)); + + # Coordinates on celestial sphere + my $ra = $crval1 + atan2(cos($theta) * sin($phi), + sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi)); + my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi)); + + return (rad2deg($ra), rad2deg($dec)); +} + + +__END__ + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_camera.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_camera.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_camera.pl (revision 22155) @@ -0,0 +1,85 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $camera_exp = can_run('camera_exp.pl') or (warn "Can't find camera_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Camera exposure processing +my $list; +{ + my $command = "$camtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get camera exposure list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from camtool.\n"; +} + +foreach my $item (@$list) { + my $cam_id = $item->{cam_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.cm.$cam_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$camera_exp --cam_id $cam_id --exp_tag $exp_tag --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do camera processing on $cam_id: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_chip.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_chip.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_chip.pl (revision 22155) @@ -0,0 +1,105 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --dbname --workdir", + -exitval => 3, + ) unless + defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $chip = can_run('chip_imfile.pl') or (warn "Can't find chip_imfile.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Imfile processing +my @whole; # The whole list for processing +{ + my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 2 imfile list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); +} + +my @single = (); + +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from chiptool.\n"; + + foreach my $item (@$list) { + my $chip_id = $item->{chip_id}; + my $exp_id = $item->{exp_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $reduction = $item->{reduction}; + my $workdir = $item->{workdir}; + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + + my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.ch.$chip_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$chip --chip_id $chip_id --exp_id $exp_id --exp_tag $exp_tag --class_id $class_id --uri $uri --dbname $dbname --camera $camera --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success; + } + + @single = (); + + } +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_detrend.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_detrend.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_detrend.pl (revision 22155) @@ -0,0 +1,377 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long; +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +my ($dbname, # Database name to use + $workdir_global, # Global working directory + $verbose, # Produce verbose output? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'workdir=s' => \$workdir_global, + 'verbose' => \$verbose, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $detrend_process_imfile = can_run('detrend_process_imfile.pl') or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1); +my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1); +my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1); +my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1); +my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1); +my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1); +my $detrend_resid = can_run('detrend_resid.pl') or (warn "Can't find detrend_resid.pl" and $missing_tools = 1); +my $detrend_reject_imfile = can_run('detrend_reject_imfile.pl') or (warn "Can't find detrend_reject_imfile.pl" and $missing_tools = 1); +my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Process raw imfiles +{ + my $command = "$dettool -toprocessedimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detproc.$det_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Process raw exposures +{ + my $command = "$dettool -toprocessedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detproc.$det_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_process_exp --det_id $det_id --det_type $det_type --exp_tag $exp_tag --exp_id $exp_id --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success; + } +} + +# Stack +{ + my $command = "$dettool -tostacked -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get stack list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + + my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Calculate normalisation +{ + my $command = "$dettool -tonormalizedstat -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalise calculation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.normstat.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Apply normalisation +{ + my $command = "$dettool -tonormalize -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalisation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.norm.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Examine normalised exposure +{ + my $command = "$dettool -tonormalizedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalised exposures list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.normexp.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Get residuals +{ + my $command = "$dettool -toresidimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual processing list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $detrend = $item->{det_uri}; + my $uri = $item->{uri}; + my $mode = $item->{mode}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detresid.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode --dbname $dbname --outroot $outroot"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --detrend $detrend" if defined $detrend; + $command .= " --verbose" if defined $verbose; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Reject based on imfiles +{ + my $command = "$dettool -toresidexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual imfile list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", $exp_tag, "$exp_tag.detresid.$det_id.$iteration" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_reject_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success; + } +} + +# Reject based on exposures +{ + my $command = "$dettool -todetrunsummary -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual exposure list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "$camera.$det_type.$det_id", "$camera.$det_type.$det_id.$iteration.detreject" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success; + } +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_diff.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_diff.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_diff.pl (revision 22155) @@ -0,0 +1,95 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $diff_skycell = can_run('diff_skycell.pl') or + (warn "Can't find diff_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image differencing +{ + my $list; + my $command = "$difftool -todiffskyfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of diffs: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from difftool.\n"; + + foreach my $item (@$list) { + my $diff_id = $item->{diff_id}; + my $skycell_id = $item->{skycell_id}; + my $tess_id = $item->{tess_id}; + my $workdir = $item->{workdir}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "tess_$tess_id", $skycell_id, "$tess_id.$skycell_id.dif.$diff_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$diff_skycell --diff_id $diff_id --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do difference for $diff_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject.pl (revision 22155) @@ -0,0 +1,142 @@ +#!/usr/bin/env perl + +# this program injects a list of single-file exposures into the db, +# taking the filename (without .fits) as the exp_tag. the user +# supplies a temporary telescope and camera name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and camera name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless + defined $telescope and + defined $camera; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + my $absfile = File::Spec->rel2abs( $file ); + inject($absfile, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +print "$num files injected.\n"; + +sub inject +{ + my $absfile = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile ); + my ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -label $label" if defined $label; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my $exp_id = 0; + unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag + } else { + print "skipping command: $command_exp\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id fpa"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + } else { + print "skipping command: $command_update\n"; + } + + return 1; +} + + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject_mosaic.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 22155) @@ -0,0 +1,168 @@ +#!/usr/bin/env perl + +# this program injects a set of multi-file exposures into the db. the +# program takes a list of base exposure names and injects all files +# associated with the exposure. It constructs the expected filenames +# from the exposure tag and rules for the camera. This program is for +# the test only since it requires too much information at the inject +# stage. use 'ipp_serial_inject.pl' for single-file images and +# 'ipp_serial_inject_split.pl' for multiple file images in split +# format + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($camera, # Camera used + $telescope, # Telescope used + $tess_id, # Tessellation identifier + $dbname, # Database name + $workdir, # Working directory + $path, # Path to data + ); +GetOptions( + 'camera|c=s' => \$camera, + 'telescope|t=s' => \$telescope, + 'workdir=s' => \$workdir, + 'tess_id=s' => \$tess_id, + 'path=s' => \$path, + 'dbname=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --camera --telescope --workdir --path --dbname", + -exitval => 3, + ) unless defined $camera + and defined $telescope + and defined $workdir + and defined $path + and defined $dbname; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $pxinject = can_run('pxinject') or (warn "Can't find pxinject" and $missing_tools = 1); + +if (scalar @ARGV == 0) { + die "No exposures provided.\n"; +} + +# Inject new data into the database +my @classes; # Names of the classes +my @files; # What to add to the filename for each class +my $imfiles; +my $add_dir; # Add directory name to get file name? +if ($camera eq "MEGACAM") { + for (my $i = 0; $i < 36; $i++) { + push @classes, sprintf("ccd%02d", $i); + push @files, sprintf(".ccd%02d", $i); + } +} elsif ($camera eq "MCSHORT") { + @classes = ( 'ccd12', 'ccd13', 'ccd14', 'ccd21', 'ccd22', 'ccd23' ); + @files = ( '.ccd12', '.ccd13', '.ccd14', '.ccd21', '.ccd22', '.ccd23' ); +} elsif ($camera eq "CTIO_MOSAIC2") { + @classes = (); + @files = (); +} elsif ($camera eq "TC3") { + @classes = ( 'CCID58-1-06b2', 'CCID45-1-14A', 'CCID45-1-11A', 'CCID45-1-22A', + 'CCID45-1-04C', 'CCID45-1-13A', 'CCID45-1-05A', 'CCID45-1-19A' ); + @files = ( '00', '01', '10', '11', '20', '21', '30', '31' ); + $add_dir = 1; +} elsif ($camera eq "SIMMOSAIC") { + @classes = ( 'Chip00', 'Chip01', 'Chip10', 'Chip11' ); + @files = ( '.Chip00', '.Chip01', '.Chip10', '.Chip11' ); +} elsif ($camera eq "SIMTEST") { + @classes = (); + @files = (); +} elsif ($camera eq "GPC1") { + for (my $i = 0; $i < 8; $i++) { + for (my $j = 0; $j < 8; $j++) { + if (($i == 0 or $i == 7) and ($j == 0 or $j == 7)) { + # Excluding corner chips + next; + } + push @classes, "XY$i$j"; + push @files, "$i$j"; + } + } + $add_dir = 1; +} else { + die "Unrecognised camera name: $camera.\nDid you mean to use ipp_serial_inject.pl?\n"; +} + +foreach my $exp_name ( @ARGV ) { + my $command = "$pxinject -newExp -tmp_exp_name $exp_name -tmp_inst $camera -tmp_telescope $telescope -workdir $workdir -tess_id $tess_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name: $error_code\n" if not $success; + + my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + for (my $i = 0; $i < scalar @classes; $i++) { + my $class_id = $classes[$i]; + my $file_id = $files[$i]; + my $filename = $exp_name . $file_id . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id $class_id -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name $class_id: $error_code\n" if not $success; + } + + if (scalar @classes == 0) { + my $filename = $exp_name . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id fpa -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code\n" if not $success; + } + + # Update the exposure to run + { + my $command = "$pxinject -updatenewExp -exp_id $exp_id -state run"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to activate $exp_name: $error_code\n" if not $success; + } + +} + +END { + my $status = $?; +system("sync") == 0 + or die "failed to execute sync: $!" ; +$? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject_split.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject_split.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_inject_split.pl (revision 22155) @@ -0,0 +1,138 @@ +#!/usr/bin/env perl + +# this program injects a list of multi-file exposures into the db. +# the program takes a list of directory names (dir), and injects all +# dir/*.fits. It takes the directory name as the exp_tag. the user +# supplies a temporary telescope and instrument name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and instrument name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $dbname, $help); +GetOptions('camera|c=s' => \$camera, # Camera used + 'telescope|t=s' => \$telescope, # Telescope used + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'dbname|d=s' => \$dbname, # Database name + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many image exposures (directories) into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--dbname name] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( + -msg => "Required options: --camera --telescope", + -exitval => 3) unless + defined $telescope and + defined $camera; + +# Look for programs we need +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $filedir ( @ARGV ) { + # check for filedir existence + if (! -e $filedir) { die "file dir $filedir not found\n"; } + my $absfiledir = File::Spec->rel2abs( $filedir ); + inject($absfiledir, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +sub inject +{ + my $absfiledir = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my @files = <$absfiledir/*.fits>; + my $Nfiles = scalar @files; + + print "absfiledir: $absfiledir\n"; + print "Nfiles: $Nfiles\n"; + + # the absfiledir is of the form path://PATH/data/foo/expname + my ( $vol, $path, $exp_name ) = File::Spec->splitpath( $absfiledir ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + + foreach my $absfile (@files) { + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + my ( $tmpvol, $tmppath, $filename ) = File::Spec->splitpath( $absfile ); + my ( $class_name ) = $filename =~ /(.*)\.fits/; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id $class_name"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + + return 1; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_mops.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_mops.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_mops.pl (revision 22155) @@ -0,0 +1,121 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use DBI; + +use IPC::Cmd 0.36 qw( can_run run ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; +use File::Temp qw( tempfile ); + +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); + +# Look for programs we need +my $missing_tools; +my $mops = can_run('ipp_mops_translate.pl') or (warn "Can't find ipp_mops_translate.pl" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +my ( $dbhost, # Database host + $dbname, # Database name + $dbuser, # Database user + $dbpass, # Database p/w + $camera, # Camera used + $outroot, # Output directory + $fileset, # File set + $verbose, # Verbose output? + $no_update, # Don't update state? + $no_op, # Don't do any operations? + $save_temps # Save temporary files? + ); + +GetOptions( + 'dbhost=s' => \$dbhost, + 'dbname=s' => \$dbname, + 'dbuser=s' => \$dbuser, + 'dbpass=s' => \$dbpass, + 'camera=s' => \$camera, + 'outroot=s' => \$outroot, + 'fileset=s' => \$fileset, + 'verbose' => \$verbose, + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbhost --dbname --dbuser --dbpass --camera --outroot --fileset", + -exitval => 3) + unless defined $dbhost + and defined $dbname + and defined $dbuser + and defined $dbpass + and defined $camera + and defined $outroot + and defined $fileset; + +$ipprc->define_camera($camera); + +my $dbsrc = 'DBI:mysql:database=' . $dbname . ';host=' . $dbhost . + ';mysql_socket=/var/run/mysqld/mysqld.sock'; +my $db = DBI->connect($dbsrc, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1 } ) or + die "Unable to connect to database: $DBI::errstr"; + +my $sql = "SELECT exp_id, diff_id, skycell_id, warp_id, diffSkyfile.path_base AS diff_path_base, warpSkyfile.path_base AS warp_path_base FROM diffSkyfile JOIN diffRun using(diff_id) JOIN diffInputSkyfile USING(diff_id,tess_id,skycell_id) JOIN warpRun USING(warp_id,tess_id) JOIN warpSkyfile USING(warp_id,skycell_id,tess_id) JOIN fakeRun using(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE diffSkyfile.fault = 0 AND rawExp.camera = \'$camera\';"; + +my $rows = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr"; +$db->disconnect; + +print "Selected " . scalar @$rows . " rows.\n"; + +$ipprc->outroot_prepare( $outroot ); +my $outrootResolved = $ipprc->file_resolve( $outroot ); +my ($dsFile, $dsName) = tempfile( "$outrootResolved.dslist.XXXX", UNLINK => !$save_temps); + +foreach my $row ( @$rows ) { + my $exp_id = $row->{exp_id}; + my $diff_id = $row->{diff_id}; + my $skycell_id = $row->{skycell_id}; + my $warp_id = $row->{warp_id}; + my $diff_base = $row->{diff_path_base}; + my $warp_base = $row->{warp_path_base}; + + my $input = $ipprc->filename("PSPHOT.OUTPUT", $diff_base); + die "Can't find $input\n" unless $ipprc->file_exists($input); + $input = $ipprc->file_resolve($input); + + my $skycell = $ipprc->filename("SKYCELL.TEMPLATE", $warp_base); + die "Can't find $skycell\n" unless $ipprc->file_exists($skycell); + $skycell = $ipprc->file_resolve($skycell); + + my $output = caturi( $outroot, "mops.$exp_id.$skycell_id.$diff_id.fits" ); + $output = $ipprc->file_resolve($output); + + unless ($no_op) { + $ipprc->file_prepare($output); + my $command = "$mops --input $input --extname SkyChip.psf --skycell $skycell --output $output"; + my $success = run( command => $command, verbose => $verbose ); + die "Couldn't translate $input\n" unless $success; + } + + print $dsFile "${output}|ipp-mops\n"; +} +close $dsFile; + +# Register new files with the data store +unless ($no_update) { + my $command = "$dsreg --add --product mops_transient_detections --type MOPS_TRANSIENT_DETECTIONS --fileset $fileset --list $dsName --copy --abspath --dbname DataStore"; + my $success = run( command => $command, verbose => $verbose ); + die "Couldn't register files with data store.\n" unless $success; +} + + +__END__ + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_register.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_register.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_register.pl (revision 22155) @@ -0,0 +1,118 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + ); +GetOptions( + 'dbname|d=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $register_imfile = can_run('register_imfile.pl') or (warn "Can't find register_imfile.pl" and $missing_tools = 1); +my $register_exp = can_run('register_exp.pl') or (warn "Can't find register_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 0 imfile processing +{ + my $command = "$regtool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 imfile list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_id = $item->{exp_id}; + my $exp_name = $item->{tmp_exp_name}; + my $class_id = $item->{tmp_class_id}; + my $uri = $item->{uri}; + + my $command = "$register_imfile --exp_id $exp_id --tmp_class_id $class_id --tmp_exp_name $exp_name --uri $uri --dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 imfile processing on $exp_name $class_id: $error_code\n" if not $success; + } + + + @single = (); + + } + } +} + +# Phase 0 exposure processing +{ + my $command = "$regtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 exposure list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{tmp_exp_name} . '.' . $item->{exp_id}; + my $exp_id = $item->{exp_id}; + + my $command = "$register_exp --exp_tag $exp_tag --exp_id $exp_id --dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_stack.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_stack.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_stack.pl (revision 22155) @@ -0,0 +1,98 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + $save_temps, # Save temporary files? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $stack_skycell = can_run('stack_skycell.pl') or + (warn "Can't find stack_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image stacking +{ + my $list; + my $command = "$stacktool -tosum -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of stacks: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from stacktool.\n"; + + foreach my $item (@$list) { + my $stack_id = $item->{stack_id}; + my $workdir = $item->{workdir}; + my $tess_id = $item->{tess_id}; + my $skycell_id = $item->{skycell_id}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "tess_$tess_id", $skycell_id, "$tess_id.$skycell_id.stk.$stack_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$stack_skycell --stack_id $stack_id --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --save-temps" if defined $save_temps; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); +### die "Unable to do stack for $stack_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_warp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_warp.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_serial_warp.pl (revision 22155) @@ -0,0 +1,128 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $verbose, # Verbose operations? + $workdir_global, # Global working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname=s' => \$dbname, + 'verbose' => \$verbose, + 'workdir' => \$workdir_global, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or + (warn "Can't find warptool" and $missing_tools = 1); +my $warp_skycell = can_run('warp_skycell.pl') or + (warn "Can't find warp_skycell.pl" and $missing_tools = 1); +my $warp_overlap = can_run('warp_overlap.pl') or + (warn "Can't find warp_overlap.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Calculate overlaps +{ + my $list; + my $command = "$warptool -tooverlap -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for which to calculate overlaps: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + my $tess_id = $item->{tess_id}; + + my $command = "$warp_overlap --warp_id $warp_id --camera $camera --tess_id $tess_id --dbname $dbname"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warp overlaps on $warp_id: $error_code\n" if not $success; + } +} + + +# Warping proper +{ + my $list; + my $command = "$warptool -towarped -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for warping: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $skycell_id = $item->{skycell_id}; + my $tess_id = $item->{tess_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + + $workdir = $workdir_global unless defined $workdir and $workdir ne "NULL"; + die "No working directory specified.\n" unless defined $workdir; + + my $outroot = caturi( $workdir, "tess_$tess_id", $skycell_id, "$tess_id.$skycell_id.wrp.$warp_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$warp_skycell --warp_id $warp_id --skycell_id $skycell_id --tess_id $tess_id --camera $camera --dbname $dbname --outroot $outroot"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do warp processing on $warp_id,$skycell_id: $error_code\n" if not $success; + } +} + + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_simulation_data.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_simulation_data.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipp_simulation_data.pl (revision 22155) @@ -0,0 +1,215 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Math::Trig; +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($name, # Base name for output images + $camera, # Name of camera to use + $telescope, # Telescope name + $tess_id, # Tessellation identifier + $dbname, # Database name + $path, # Path to data + $workdir, # Working directory for data + $no_cal, # Don't produce calibration files + $no_update # Don't update the database + ); + +GetOptions( + 'name=s' => \$name, + 'camera=s' => \$camera, + 'telescope=s' => \$telescope, + 'tess_id=s' => \$tess_id, + 'dbname=s' => \$dbname, + 'path=s' => \$path, + 'workdir=s' => \$workdir, + 'no-cal' => \$no_cal, + 'no-update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --name --path --camera --telescope --dbname", + -exitval => 3, + ) unless + defined $name and + defined $path and + defined $camera and + defined $telescope and + defined $dbname; + +$workdir = $path if not defined $workdir; + +# Look for programs we need +my $missing_tools; +my $ppSim = can_run('ppSim') + or (warn "Can't find ppSim" and $missing_tools = 1); +my $inject = can_run('ipp_serial_inject_mosaic.pl') + or (warn "Can't find ipp_serial_inject_mosaic.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Number of bias images +use constant BIAS => 20; +# Dark exposure times +use constant DARK => [ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300 ]; +# Flat-field image characteristics +use constant FLAT => [ + { + filter => 'r', + exptime => [ 0.1, 0.1, 0.1, 0.1, 0.1, 0.5, 1, 2, 5, 10, 20, 20, 20, 20, 20 ], + }, + { + filter => 'i', + exptime => [ 20, 20, 20, 20, 20 ], + } + ]; +# Object image characteristics +use constant OBJECT => [ + { + filter => 'r', + exptime => [ 5, 10, 10, 10, 10, 10, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.15, + sky => 20.86, + dither => 40, + }, + { + filter => 'i', + exptime => [ 5, 30, 30, 30, 30, 30, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.00, + sky => 20.15, + dither => 40, + }, + ]; +use constant SCALE => 0.2; # Plate scale + +############################################################################################################# +### Now do the work +############################################################################################################# + +my $counter = 0; +my $tessellation = (defined $tess_id) ? "--tess_id $tess_id" : ""; # Additional switch for tessellation + +# Generate bias images +for (my $i = 0; $i < BIAS; $i++) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type BIAS $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate dark images +foreach my $exptime ( @{DARK()} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run ( command => "$ppSim -camera $camera -type DARK -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate flat images +foreach my $set ( @{FLAT()} ) { + my $filter = $set->{filter}; # Name of filter + foreach my $exptime ( @{$set->{exptime}} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type FLAT -filter $filter -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } + } +} + +# Generate object images +foreach my $set ( @{OBJECT()} ) { + my $filter = $set->{filter}; # Name of filter + my $ra0 = $set->{ra}; # Base Right Ascension (deg) + my $dec0 = $set->{dec}; # Base Declination (deg) + my $pa = $set->{pa}; # Position angle (deg) + my $zp = $set->{zp}; # Zero point + my $scale = SCALE(); # Plate scale (arcsec/pix) + my $sky = 10**( -0.4 * ( $set->{sky} - $zp ) ) * $scale**2; # Sky background (counts/s) + my $dither = $set->{dither} / 3600; # Dither size (deg) + + for (my $i = 0; $i < scalar @{$set->{exptime}}; $i++) { + my $exptime = ${$set->{exptime}}[$i]; # Exposure time + my $seeing = ${$set->{seeing}}[$i]; # Seeing (pix) + my $ra = $ra0 + (2*rand() - 1) * $dither * cos(deg2rad($dec0)); # RA with dither + my $dec = $dec0 + (2*rand() - 1) * $dither; # Dec with dither + + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + run( command => "$ppSim -camera $camera -type OBJECT -filter $filter -exptime $exptime " . + "-skyrate $sky -ra $ra -dec $dec -pa $pa -scale $scale -zp $zp -seeing $seeing $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "$tessellation --workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + + +### Pau. + + +# Generate a filename from the base name and counter +sub filename +{ + my $base = shift; # Base name + my $num = shift; # Number + my $workdir = shift; # Working directory + my $name = sprintf("$base%04d", $num); + $num++; + return ( $name, $num ); +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipprc.txt =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipprc.txt (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/ipprc.txt (revision 22155) @@ -0,0 +1,16 @@ + +these scripts use the following library functions to work with the IPP Config system: + +$ipprc->convert_filename_absolute( $tess_dir ); +$ipprc->convert_filename_relative( $absfile ); +$ipprc->define_camera($camera); +$ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output +$ipprc->file_create_append( $logName ); +$ipprc->file_exists( $skyfile ); +$ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +$ipprc->file_resolve ($path_base); +$ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); +$ipprc->outroot_prepare($outroot); +$ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +$ipprc->rejection( $name, $det_type, $filter ); +$ipprc->tessellation_catdir( $tess_id ); # Tessellation catdir for DVO Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/isp_trans.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/isp_trans.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/isp_trans.pl (revision 22155) @@ -0,0 +1,30 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: isp_trans.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OA'; # Recipe to use (switch to OBDSFRA when detrend images are ready) +$RECIPE_PSPHOT = 'PSPHOT.SUMMIT'; +$CALDIR = '/data/alala.0/ipp/ippRefs/catdir.synth.bright'; # source of photometric calibration data +$IMTABLE = 'images.dat'; # source of photometric calibration data + +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +vsystem ("addstar -incal -image -D CAMERA isp -D IMAGE_TABLE $IMTABLE -D CATDIR $CALDIR $output.smf"); +if ($status) { die "failure getting calibration from addstar\n"; } + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/magic_tree.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/magic_tree.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/magic_tree.pl (revision 22155) @@ -0,0 +1,329 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +use Math::Trig; +use File::Temp qw( tempfile ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant MAX_FIELDS => 4; # Maximum number of fields to be in a node + +# Parse the command-line arguments +my ($magic_id, $tess_id, $camera, $ra0, $dec0, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op); +GetOptions( + 'magic_id=s' => \$magic_id, # Magic identifier + 'tess_id=s' => \$tess_id, # Tessellation identifier + 'camera=s' => \$camera, # Camera name + 'ra=f' => \$ra0, # Boresight right ascension, radians + 'dec=f' => \$dec0, # Boresight declination, radians + 'dbname=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'save-temps' => \$save_temps, # Save temporary files? + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_id --camera --outroot", + -exitval => 3) unless + defined $magic_id and + defined $tess_id and + defined $ra0 and + defined $dec0 and + defined $camera and + defined $outroot; + +$ipprc->define_camera($camera); + + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +### Get a list of skycells +my @skycells; # List of skycells +{ + my $command = "$magictool -inputskyfile -magic_id $magic_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); + + my $inputs = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); + + foreach my $input ( @$inputs ) { + push @skycells, $input->{skycell_id}; + } +} + +### For each skycell, project centre of skycell onto tangent plane of boresight +my @fields; +foreach my $skycell_id ( @skycells ) { + my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); + $ipprc->skycell_file($tess_id, $skycell_id, $skyFile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR); + my $skyfileResolved = $ipprc->file_resolve( $skyfile ); + my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved ); + &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status; + + # Get the useful header keywords + my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR); + my $naxis2 = $$header{'NAXIS2'} or &my_die("Can't find NAXIS2", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR); + my $crota1 = $$header{'CROTA1'}; + my $crota2 = $$header{'CROTA2'}; + + &my_die("Unexpected projection: $ctype1 and $ctype2.", $magic_id, $PS_EXIT_SYS_ERROR) unless + $ctype1 =~ /^\'RA---TAN\s*\'$/ and $ctype2 =~ /^\'DEC--TAN\s*\'$/; + &my_die("Can't determine size of skycell ($naxis1,$naxis2)", $magic_id, $PS_EXIT_SYS_ERROR) unless + $naxis1 > 0 and $naxis2 > 0; + &my_die("Can't determine scale of skycell ($cdelt1,$cdelt2)", $magic_id, $PS_EXIT_SYS_ERROR) if + not defined $cdelt1 or $cdelt1 == 0 or not defined $cdelt2 or $cdelt2 == 0; + &my_die("We don't know how to handle rotations ($crota1,$crota2)", $magic_id, $PS_EXIT_SYS_ERROR) + if defined $crota1 or defined $crota2; + + # Relative coordinates of centre of the field + my $x = $naxis1 - $crpix1; + my $y = $naxis2 - $crpix2; + + # Coordinates on tangent plane + my $xi = $pc11 * ($x) + $pc12 * ($y); + my $eta = $pc21 * ($x) + $pc22 * ($y); + $xi *= $cdelt1; + $eta *= $cdelt2; + + # Coordinates on rotated celestial sphere + my $phi = atan2($eta,$xi) + pi/2; + my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2)); + + # Coordinates on celestial sphere + $crval1 = deg2rad($crval1); + $crval2 = deg2rad($crval2); + my $ra = $crval1 + atan2(cos($theta) * sin($phi), + sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi)); + my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi)); + + # Rotate to boresight + my $phi_new = atan2(cos($dec) * sin($ra - $ra0), + sin($dec) * cos($dec0) + cos($dec) * sin($dec0) * cos($ra - $ra0)); + my $theta_new = asin(sin($dec) * sin($dec0) - cos($dec) * cos($dec0) * cos($ra - $ra0)); + + # Project + my $rad = 180 / pi * cot($theta_new); + my $xi_new = $rad * sin($phi); + my $eta_new = - $rad * cos($phi); + + my $field = { id => $skycell_id, + xi => $xi_new, + eta => $eta_new, + }; + + push @fields, $field; +} + +### Subdivide list of positions into kd-tree +my $root = { # Root node of tree + contents => \@fields, # Contents of node + position => 'root', # Position in tree + children => {}, # Children of node +}; +my @tasks = ( $root ); +while (scalar @tasks > 0) { + my $node = shift @tasks; + divide_node($node, \@tasks); +} + +### Format tree for magictool +my $mdcTree = print_node($root); # The tree in MDC format +my ($treeFile, $treeName) = tempfile( "magictree.${magic_id}.XXXX", UNLINK => !$save_temps ); +print $treeFile, $mdcTree; +close $treeFile; + +### Input tree into database +{ + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -dep_file $treeName"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform magictool -inputtree: $error_code"); + exit($error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + +### Pau. + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_id = shift; # Magic identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $magic_id and not $no_update) { + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +# Divide a list into two, returning the lower and upper parts +sub divide_list +{ + my $list = shift; # List to divide + my $index = shift; # Name of index for sorting + + my @sorted = sort { $$a{$index} <=> $$b{$index} } @$list; # Sorted list + my $median = int(scalar @sorted / 2); # Median point of list + my @upper = splice(@sorted, $median); # Upper part of the sorted list + + return (\@sorted, \@upper); +} + +# Create a new node, add it to the parent, and add it to the task list if required +sub new_node +{ + my $parent = shift; # The parent node + my $contents = shift; # Contents of the new node + my $position = shift; # Position description + my $tasks = shift; # Tasks to do + + my $node = { + contents => $contents, + position => $parent->{position} . '_' . $position, + children => {}, + }; + + $parent->{children}->{$position} = $node; + + push @$tasks, $node if scalar @$contents > 4; + + return $node; +} + +# Divide a node +sub divide_node +{ + my $node = shift; # Node to divide + my $tasks = shift; # Tasks to do + + my $position = $node->{position}; + + my $contents = $node->{contents} or die "Can't find contents of node."; # Contents of node + + my ($lower, $upper) = divide_list($contents, 'xi'); + + if (scalar @$lower > 4) { + my ($ll, $lr) = divide_list($lower, 'eta'); + new_node($node, $ll, 'll', $tasks); + new_node($node, $lr, 'lr', $tasks); + } else { + new_node($node, $lower, 'L', $tasks); + } + + if (scalar @$upper > 4) { + my ($ul, $ur) = divide_list($upper, 'eta'); + new_node($node, $ul, 'ul', $tasks); + new_node($node, $ur, 'ur', $tasks); + } else { + new_node($node, $upper, 'U', $tasks); + } + + $node->{contents} = undef; + + return $node; +} + +# Print the contents of a node +sub print_node +{ + my $node = shift; # Node to print + + my $position = $node->{position}; # Position of node + + my $output = "$position\t\tMULTI\n"; # Output text + + if (defined $node->{contents}) { + foreach my $field ( @{$node->{contents}} ) { + my $skycell_id = $field->{id}; # Skycell name + $output .= "$position\t\tSTR\t$skycell_id\n"; + $output .= "$skycell_id\t\tSTR\tNULL\t\# $field->{xi},$field->{eta}\n"; + } + } else { + foreach my $div ( keys %{$node->{children}} ) { + $output .= "$position\t\tSTR\t${position}_$div\n"; + $output .= print_node($node->{children}->{$div}); + } + } + + return $output; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/mdc2list.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/mdc2list.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/mdc2list.pl (revision 22155) @@ -0,0 +1,77 @@ +#!/usr/bin/env perl + +# Program to convert a PS metadata config syntax (usually from one of +# the IPP tools) into a flat tab-delimited list suitable for parsing +# using "split". + +# Copyright (C) 2006 Joshua Hoblitt, Paul A. Price. + +use strict; +use warnings; + +use PS::IPP::Metadata::Config; # Supplies the metadata config parser. + +die "Program to convert a PS metadata config file (usually from one of the IPP\n" . + "tools) into a flat tab-delimited list suitable for parsing using \'split\'.\n\n" . + "Usage: $0 [INFILE [OUTFILE]]\n" if ((join "", @ARGV) =~ /--help/ or scalar @ARGV > 2); + +my $inFile; # Input file +my $outFile; # Output file +if (scalar @ARGV >= 1) { + my $inName = shift @ARGV; + open $inFile, $inName or die "Can't open $inName: $!\n"; + if (scalar @ARGV == 1) { + my $outName = shift @ARGV; + open $outFile, ">", $outName or die "Can't open $outName: $!\n"; + } else { + $outFile = *STDOUT; + } +} else { + $inFile = *STDIN; + $outFile = *STDOUT; +} + +my @input = <$inFile>; # Contents of the metadata config file +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $md = $mdcParser->parse(join "", @input) + or die "unable to parse metadata config doc"; +my $hashes = mds2hashes($md); # An array of hashes +foreach my $pending (@$hashes) { + foreach my $key (keys %$pending) { + print $outFile ( $pending->{$key} . "\t"); + } + print $outFile "\n"; +} + +### Pau. + + +# Given an array of MDs, return an array of hashes +sub mds2hashes +{ + my $mds = shift; # Reference to the metadatas + my @array; # The array of hashes, to be returned + foreach my $md (@$mds) { + my $values = md2hash($md->{value}); + push @array, $values; + } + return \@array; +} + +# Convert the metadata to a hash; in effect, strips out the comment, type and class fields. +sub md2hash +{ + my $values = shift; # Reference to the metadata + my %hash; # Hash, to be returned + foreach my $data (@$values) { + $hash{$data->{name}} = $data->{value}; + } + return \%hash; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/register_exp.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/register_exp.pl (revision 22155) @@ -0,0 +1,222 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw( freeze thaw ); +use File::Basename qw( basename ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); + +my ($cache, $exp_id, $exp_tag, $dbname, $verbose, $no_update, $no_op, $logfile); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|t=s' => \$exp_tag, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'logfile=s' => \$logfile, +) or pod2usage( 2 ); + +$ipprc->redirect_output($logfile) if $logfile; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --exp_tag", + -exitval => 3) unless + defined $exp_id and + defined $exp_tag; + +# add -detrend UNLESS type is one of SCIENCE listed below (eg, OBJECT) +my @SCIENCE = ( "object", "science" ); # Observation types to NOT mark as detrend +my $DETREND_FLAG = "-end_stage reg"; # Flag to use to mark detrend exposure + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ # register imfile + # label STATISTIC CHIPTOOL FLAG + { name => "exp_name", type => "constant", flag => "-exp_name", dtype => "string" }, # File level + { name => "telescope", type => "constant", flag => "-telescope", dtype => "string" }, # File level + { name => "camera", type => "constant", flag => "-inst", dtype => "string" }, # File level + { name => "filelevel", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "object", type => "constant", flag => "-object", dtype => "string" }, + { name => "exp_type", type => "constant", flag => "-exp_type", dtype => "string" }, # File level + { name => "filter", type => "constant", flag => "-filter", dtype => "string" }, # File level + { name => "comment", type => "constant", flag => "-comment", dtype => "string" }, # ObsComment + { name => "dateobs", type => "constant", flag => "-dateobs", dtype => "string" }, # File level + { name => "ccd_temp", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "exp_time", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "sat_pixel_frac", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # Fraction of saturated pixels + { name => "airmass", type => "mean", flag => "-airmass", dtype => "float" }, # Airmass + { name => "ra", type => "mean", flag => "-ra", dtype => "float" }, # Right ascension + { name => "decl", type => "mean", flag => "-decl", dtype => "float" }, # Declination + { name => "posang", type => "mean", flag => "-posang", dtype => "float" }, # Position angle + { name => "alt", type => "mean", flag => "-alt", dtype => "float" }, # Altitude + { name => "az", type => "mean", flag => "-az", dtype => "float" }, # Azimuth + { name => "m1_x", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "m1_y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "m1_z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "m1_tip", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "m1_tilt", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "m2_x", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "m2_y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "m2_z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "m2_tip", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "m2_tilt", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "env_temperature", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "env_humidity", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "env_wind_speed", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "env_wind_dir", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "-teltemp_m1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "-teltemp_m1cell", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "-teltemp_m2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "-teltemp_spider", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "-teltemp_truss", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "-teltemp_extra", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "pon_time", type => "mean", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, # background + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, # Azimuth + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, # Azimuth + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for commands we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "can't find regtool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +# Get the list of imfiles & their stats +{ + my $command = "$regtool -processedimfile -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -processedimfile on exposure id $exp_id: $error_code"); + exit ($error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $PS_EXIT_CONFIG_ERROR); + } +} + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_name")) eq "NULL") { &my_die ("exp_name not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedexp"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_tag $exp_tag"; +$command .= " -hostname $host" if defined $host; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +my $exp_type = $stats->value_for_flag ("-exp_type"); + +# Add the detrend flag, if needed +{ + my $object = 0; # Is it an object exposure? + foreach my $scienceType (@SCIENCE) { + if (lc($exp_type) =~ /$scienceType/) { + $object = 1; + last; + } + } + $command .= " $DETREND_FLAG" unless $object; +} + +# Output results to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to run regtool -addprocessedexp for $exp_id: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +### Pau. + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exit_code = shift; + + carp($msg); + if (defined $exp_id and not $no_update) { + my $command = "$regtool -addprocessedexp -exp_id $exp_id -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/register_imfile.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/register_imfile.pl (revision 22155) @@ -0,0 +1,244 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $dbname, $verbose, $no_update, $no_op, $logfile); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'tmp_class_id|i=s' => \$tmp_class_id, + 'tmp_exp_name|n=s' => \$tmp_exp_name, + 'uri|u=s' => \$uri, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'logfile=s' => \$logfile, +) or pod2usage( 2 ); + +$ipprc->redirect_output($logfile) if $logfile; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --tmp_class_id --tmp_exp_name --uri", + -exitval => 3) unless + defined $exp_id and + defined $tmp_class_id and + defined $tmp_exp_name and + defined $uri; + + +my $RECIPE = "REGISTER"; # Recipe to use for ppStats + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "FILE.LEVEL", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "CLASS.ID", type => "constant", flag => "-class_id", dtype => "string" }, # Real Class ID + { name => "FPA.OBJECT", type => "constant", flag => "-object", dtype => "string" }, # Object + { name => "FPA.OBSTYPE", type => "constant", flag => "-exp_type", dtype => "string" }, # Exposure type + { name => "FPA.FILTER", type => "constant", flag => "-filter", dtype => "string" }, # Filter used + { name => "FPA.COMMENT", type => "constant", flag => "-comment", dtype => "string" }, # Obs Comment + { name => "FPA.AIRMASS", type => "constant", flag => "-airmass", dtype => "float" }, # Airmass + { name => "FPA.RA", type => "constant", flag => "-ra", dtype => "float" }, # Right ascension + { name => "FPA.DEC", type => "constant", flag => "-decl", dtype => "float" }, # Declination + { name => "FPA.ALT", type => "constant", flag => "-alt", dtype => "float" }, # Altitude + { name => "FPA.AZ", type => "constant", flag => "-az", dtype => "float" }, # Azimuth + { name => "FPA.POSANGLE", type => "constant", flag => "-posang", dtype => "float" }, # Position angle + { name => "FPA.TIME", type => "constant", flag => "-dateobs", dtype => "string" }, # Date of observation (UTC) + { name => "FPA.TELESCOPE", type => "constant", flag => "-telescope", dtype => "string" }, # Telescope + { name => "FPA.CAMERA", type => "constant", flag => "-inst", dtype => "string" }, # Instrument + { name => "FPA.M1X", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "FPA.M1Y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "FPA.M1Z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "FPA.M1TIP", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "FPA.M1TILT", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "FPA.M2X", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "FPA.M2Y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "FPA.M2Z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "FPA.M2TIP", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "FPA.M2TILT", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "FPA.ENV.TEMP", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "FPA.ENV.HUMID", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "FPA.ENV.WIND", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "FPA.ENV.DIR", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "FPA.TELTEMP.M1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "FPA.TELTEMP.M1CELL", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "FPA.TELTEMP.M2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "FPA.TELTEMP.SPIDER", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "FPA.TELTEMP.TRUSS", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "FPA.TELTEMP.EXTRA", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "FPA.PON.TIME", type => "constant", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "CHIP.TEMP", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "CELL.EXPOSURE", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "SAT_PIXEL_FRAC", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # fraction of saturated pixels + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "Can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') + or (warn "Can't find ppStats" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +# XXX why is this being cached? +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +my $now_time = localtime(); +printf STDERR "\nstarting ppStats: %s\n", $now_time if $verbose; + +# Run ppStats on the input file +{ + # extract the data from the image header; -level is used to get FILE.LEVEL and CLASS.ID + my $command = "$ppStats $uri -recipe PPSTATS $RECIPE -level"; # Command to run ppStats + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); + } +} + +$now_time = localtime(); +printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose; + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-class_id")) eq "NULL") { &my_die ("class_id not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_name $tmp_exp_name"; # keep the supplied exp_name (could be derived from the file) +$command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID +$command .= " -uri $uri "; +$command .= " -hostname $host" if defined $host; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +$now_time = localtime(); +printf STDERR "\nrunning regtool update: %s\n", $now_time if $verbose; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +$now_time = localtime(); +printf STDERR "\ndone with regtool update: %s\n", $now_time if $verbose; + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exp_name = shift; + my $tmp_class_id = shift; + my $uri = shift; + my $exit_code = shift; + + # for failed imfiles, we insert UNKNOWN for inst, telescope, class_id + + carp($msg); + if (defined $exp_id && defined $tmp_class_id and not $no_update) { + my $command = "$regtool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -exp_name $exp_name"; + $command .= " -tmp_class_id $tmp_class_id"; + $command .= " -uri $uri "; + $command .= " -telescope UNKNOWN"; + $command .= " -inst UNKNOWN"; + $command .= " -class_id $tmp_class_id"; + $command .= " -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/stack_skycell.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/stack_skycell.pl (revision 22155) @@ -0,0 +1,256 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($stack_id, $dbname, $outroot, $verbose, $no_update, $no_op, $save_temps); +GetOptions( + 'stack_id|d=s' => \$stack_id, # Stack identifier + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --stack_id --outroot", + -exitval => 3, + ) unless defined $stack_id + and defined $outroot; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC STACKTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, +# { name => "DT_STACK", type => "sum", flag => "-dtime_stack", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $ppStack = can_run('ppStack') or (warn "Can't find ppStack" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for stacking +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$stacktool -inputskyfile -stack_id $stack_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -inputskyfile: $error_code", $stack_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $stack_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files >= 2; + +# Parse the list of input files to get the tesselation, skycell identifiers and camera +my $skycell_id; # Skycell identifier +my $tess_id; # Tesselation identifier +my $camera; # Camera +foreach my $file (@$files) { + # skip warps which are speicified as 'ignored' + if ($file->{ignored}) { next; } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } +} + +&my_die("Can't find camera", $stack_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +# Generate MDC file with the inputs +my ($listFile, $listName) = tempfile( "$tess_id.$skycell_id.stk$stack_id.list.XXXX", + UNLINK => !$save_temps ); +my $num = 0; +my $inputSources; # Sources to use as stamps +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weight = $ipprc->filename( "PSWARP.OUTPUT.WEIGHT", $file->{path_base} ); # Weight name + my $psf = $ipprc->filename( "PSPHOT.PSF.SAVE", $file->{path_base} ); # Weight name + + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); + &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight ); + &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $psf ); + + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + print $listFile "\tPSF\tSTR\t" . $psf . "\n"; + + ### XXX NEED TO UPDATE THESE appropriately + print $listFile "\tWEIGHTING\tF32\t" . 1.0 . "\n"; + + print $listFile "END\n\n"; + + # XXX details about supplying sources, etc, need to be specified in the libraries + unless (defined $inputSources) { + $inputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); + &my_die("Source file $inputSources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $inputSources ); + } + +} + +# Get the output filenames +my $outputName = $ipprc->filename("PPSTACK.OUTPUT", $outroot); +my $outputMask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $outroot); +my $outputWeight = $ipprc->filename("PPSTACK.OUTPUT.WEIGHT", $outroot); +my $outputSources = $ipprc->filename("PSPHOT.OUTPUT", $outroot, "none"); +#my $bin1Name = $ipprc->filename("PPSTACK.BIN1", $outroot); +#my $bin2Name = $ipprc->filename("PPSTACK.BIN2", $outroot); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +# Perform stacking +unless ($no_op) { + my $command = "$ppStack $listName $outroot"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSUB STACK"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -sources $inputSources"; + $command .= " -photometry"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStack: $error_code", $stack_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the stacked image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $stack_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $stack_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the stack result + { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -uri $outputName -path_base $outroot"; + $command .= $stats->cmdflags(); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -addsumskyfile: $error_code", $stack_id, $error_code); + } + } + + # Register the run as completed + { + my $command = "$stacktool -updaterun -stack_id $stack_id -state stop"; # Command to run stacktool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform stacktool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stack_id = shift; # Stack identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $stack_id and not $no_update) { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/summit_copy.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/summit_copy.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/summit_copy.pl (revision 22155) @@ -0,0 +1,163 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use Sys::Hostname; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +## report the program and machine +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +# Parse the command-line arguments +my ( $uri, $filename, $compress, $bytes, $md5, $nebulous, $exp_name, $inst, $telescope, $class, $class_id, $end_stage, $workdir, + $dbname, $verbose, $no_update, $no_op, $timeout ); +GetOptions( + 'uri=s' => \$uri, # source location of file on data store + 'filename=s' => \$filename, # target location of file on local system + 'compress' => \$compress, # request file in compressed format + 'bytes=s' => \$bytes, # reported file size in bytes + 'md5=s' => \$md5, # reported md5 checksum + 'nebulous' => \$nebulous, # use nebulous for the target file + 'exp_name=s' => \$exp_name, # Exposure name + 'inst=s' => \$inst, # Instrument + 'telescope=s' => \$telescope, # Telescope + 'class=s' => \$class, # Class level + 'class_id=s' => \$class_id, # Class identifier + 'end_stage=s' => \$end_stage, # end of processing + 'workdir=s' => \$workdir, # workdir + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'timeout=s' => \$timeout, # passed through to dsget +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id --workdir", + -exitval => 3) + unless defined $uri + and defined $filename + and defined $exp_name + and defined $inst + and defined $telescope + and defined $class + and defined $class_id + and defined $workdir; + +# Look for programs we need +my $missing_tools; +my $dsget = can_run('dsget') + or (warn "Can't find dsget" and $missing_tools = 1); +my $pztool = can_run('pztool') + or (warn "Can't find pztool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# dsget command +my $command; +$command = "$dsget --uri $uri --filename $filename"; +$command .= " --compress" if defined $compress; +$command .= " --bytes $bytes" if defined $bytes; +$command .= " --nebulous" if defined $nebulous; +$command .= " --md5 $md5" if defined $md5; +$command .= " --timeout $timeout" if defined $timeout; + +# run command +unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my_die("Unable to perform dsget: $error_code", + $exp_name, + $inst, + $telescope, + $class, + $class_id, + $uri, + $error_code + ); + } +} else { + print "skipping command: $command\n"; +} + +# command to update database +$command = "$pztool -copydone"; +$command .= " -exp_name $exp_name"; +$command .= " -inst $inst"; +$command .= " -telescope $telescope"; +$command .= " -class $class"; +$command .= " -class_id $class_id"; +$command .= " -uri $filename"; +$command .= " -workdir $workdir"; +$command .= " -end_stage $end_stage" if defined $end_stage; +$command .= " -dbname $dbname" if defined $dbname; + +# update the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform $command: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_name = shift; # Chiptool identifier + my $inst = shift; # Chiptool identifier + my $telescope = shift; # Class identifier + my $class = shift; # Class identifier + my $class_id = shift; # Class identifier + my $uri = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + warn $msg; + unless ($no_update) { + # command to update database + my $command; + $command = "$pztool -copydone"; + $command .= " -exp_name $exp_name"; + $command .= " -inst $inst"; + $command .= " -telescope $telescope"; + $command .= " -class $class"; + $command .= " -class_id $class_id"; + $command .= " -uri $uri"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + + system ($command); + } + + exit $exit_code; +} + +# XXX: I don't think that we need this - JH +#END { +# my $status = $?; +# system("sync") == 0 +# or die "failed to execute sync: $!" ; +# $? = $status; +#} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/warp_overlap.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/warp_overlap.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/warp_overlap.pl (revision 22155) @@ -0,0 +1,300 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); +use File::Spec; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $camera, $tess_dir, $dbname, $verbose, $no_update, $no_op, $save_temps); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'camera|c=s' => \$camera, # Camera name + 'tess_dir=s' => \$tess_dir, # Tessellation directory + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --camera --tess_dir", + -exitval => 3, +) unless defined $warp_id + and defined $camera + and defined $tess_dir; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component imfiles for exposure +my $imfiles; +{ + my $command = "$warptool -imfile -warp_id $warp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -imfile: $error_code", $warp_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source filerule (eg, PSASTRO.OUTPUT, PSASTRO.OUTPUT.MEF) +my $astromAccept; # Accept the astrometry unconditionally? +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + $astromAccept = metadataLookupBool($metadata, 'ASTROM.ACCEPT'); +} + +# Determine the imfile/skycell overlaps +my @overlaps = (); +unless ($no_op) { + # Calculate the overlaps between imfiles and skycells + + # tess_dir is the DVO db holding the tessalations + # this may be an abstract name in site.config:TESSELLATIONS, a URI, or an absolute path. + # convert this to an absolute path + my $tess_dir_abs = $ipprc->tessellation_catdir( $tess_dir ); # Tessellation catdir for DVO + $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); + + my %unique_skycells = (); # Identified skycells (all unique by virtue of hash property) + + # astrometry is always determined at the camera stage; we have a MEF astrometry file from psastro + my $imfile = $imfiles->[0]; + my $camRoot = $imfile->{cam_path_base}; + my $astromFile = $ipprc->filename($astromSource, $camRoot); # Astrometry file + + my @matchlist = get_overlaps($astromFile, $tess_dir_abs, $astromAccept); # List of overlaps + if (! @matchlist) { + &my_die("Unable to perform dvoImageOverlaps: missing astrometry", $warp_id, $PS_EXIT_DATA_ERROR); + } + # Match each of the imfiles to this list (the input images may be split, but the astrometry is not) + # tess_dir (not tess_dir_abs) is supplied here so it may be written to the db + foreach my $imfile (@$imfiles) { + extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_dir, \@overlaps, \%unique_skycells); + } +} else { + # create an overlap with an entry for each skycell:imfile match + foreach my $imfile (@$imfiles) { + my %overlap = (); + $overlap{skycell_id} = 'default'; + $overlap{tess_dir} = 'default'; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @overlaps, \%overlap; + } +} + +# If no overlaps are found, we +# keep running this step over and over (a successful warptool -addoverlap prevents +# successive warptime -imfile from running. +&my_die("Unable to find any overlaps", $warp_id, $PS_EXIT_PROG_ERROR) if scalar @overlaps == 0; + +# Generate a MDC file with the overlaps +my ($overlapFile, $overlapName) = tempfile( 'overlaps.wrp' . $warp_id . '.mdc.XXXX', UNLINK => !$save_temps ); +print $overlapFile "warpSkyCellMap MULTI\n\n"; +foreach my $overlap (@overlaps) { + print $overlapFile "warpSkyCellMap METADATA\n"; + print $overlapFile " warp_id S32 $warp_id\n"; + print $overlapFile " skycell_id STR $overlap->{skycell_id}\n"; + # XXX convert tess_id here to tess_dir when db scheme is updated + print $overlapFile " tess_id STR $overlap->{tess_dir}\n"; + print $overlapFile " cam_id S32 $overlap->{cam_id}\n"; + print $overlapFile " class_id STR $overlap->{class_id}\n"; + print $overlapFile " fault S16 $overlap->{fault}\n"; + print $overlapFile "END\n\n"; +} +close $overlapFile; + +system "cat $overlapName" if $verbose; + +# Add the processed file to the database +unless ($no_update) { + my $command = "$warptool -addoverlap -mapfile $overlapName"; # Command to run warptool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addoverlap: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if (defined $warp_id and not $no_update) { + my $command = "$warptool -addoverlap"; + $command .= " -warp_id $warp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Run dvoImageOverlaps to get the overlaps; return the output +sub get_overlaps +{ + my $filename = shift; # Filename on which to run dvoImageOverlaps + my $tess_dir_abs = shift; # Tessellation directory + my $accept = shift; # Do we use the -accept-astrom flag? + + my $command = "$dvoImageOverlaps -D CATDIR $tess_dir_abs " . $ipprc->file_resolve($filename); + $command .= ' -accept-astrom' if $accept; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { + print "Unable to calculate overlaps from $filename\n"; + print "STDOUT:\n"; + foreach my $line (@$stdout_buf) { + print ">>>$line"; + } + print "STDERR:\n"; + foreach my $line (@$stderr_buf) { + print ">>>$line"; + } + return 0; + } + return split ('\n', (join "", @$stdout_buf)); +} + +# Extract a list of overlaps for an imfile +# +# The command "dvoImageOverlaps -accept-astrom -D CATDIR tessellation (megacam)" returns: +# 729534pa.cmf[ccd00.hdr] : skycell.051.fits +# PSASTRO.OUTPUT[CMF.HEAD] : SKYCELL +# [CMF.HEAD] is optionally used for MEF files (SIMPLE or MOSAIC) +sub extract_overlaps +{ + my $matches = shift; # Reference to list of skycells from dvoImageOverlaps + my $imfile = shift; # Imfile information + my $filename = shift; # Filename used with dvoImageOverlaps + my $tess_dir = shift; # Tessellation identifier + my $overlaps = shift; # Reference to list of overlaps + my $unique_skycells = shift; # Reference to hash of found skycells + + # Get rid of the path + my @dirlist = File::Spec->splitdir( $filename ); # The elements of the full path + $filename = pop @dirlist; + + # Work out how to identify this imfile in the output + my $fileLevel = $imfile->{filelevel}; + my $entry; # How to identify this imfile in the dvoImageOverlaps output + if (lc($fileLevel) eq "chip") { + my $class_id = $imfile->{class_id}; + my $chipRoot = $ipprc->file_resolve( $imfile->{chip_path_base} ); + my $extname = $ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output + + $entry = $filename . '\[' . $extname . '\]'; + print STDERR "entry: $entry, class: $class_id, extname: $extname, chiproot: $chipRoot\n" if $verbose; + } else { + $entry = $filename; + print STDERR "entry: $entry\n" if $verbose; + } + + my @skycells = &select_skycells($entry, @$matches); # Matching skycells + my $Nskycells = @skycells; + printf STDERR "Nskycells: $Nskycells\n" if $verbose; + foreach my $skycell (@skycells) { + my %overlap = (); # Overlap information for warptool + $overlap{skycell_id} = $skycell; + $overlap{tess_dir} = $tess_dir; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @$overlaps, \%overlap; + + printf STDERR "overlap: %s : %s , %s\n", $skycell, $imfile->{cam_id}, $imfile->{class_id} if $verbose; + + $unique_skycells->{$skycell} = 1; + } + + return; +} + +# Find skycells in the list that come from a particular entry +sub select_skycells +{ + my $entry = shift; # File+Ext to search for + my @list = @_; # List of "File+Ext : skycell" + + my @skycells = (); + my %unique = (); # Ensure we only return unique skycells for this entry + + foreach my $line (@list) { + if ($line =~ m|$entry|) { + my ($skycell) = $line =~ m|$entry\S*\s+:\s+(\S+)|; + if (not defined $unique{$skycell}) { + push @skycells, $skycell; + $unique{$skycell} = 1; + } + } + } + return @skycells; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/warp_skycell.pl (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/scripts/warp_skycell.pl (revision 22155) @@ -0,0 +1,262 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $skycell_id, $tess_dir, $camera, $dbname, $outroot, $verbose, $no_update, $no_op, $save_temps); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'skycell_id|s=s' => \$skycell_id, # Skycell identifier + 'tess_dir|s=s' => \$tess_dir, # Tesselation identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'outroot=s' => \$outroot, # Output root name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --skycell_id --tess_dir --camera --outroot", + -exitval => 3, +) unless defined $warp_id + and defined $skycell_id + and defined $tess_dir + and defined $camera + and defined $outroot; + +$ipprc->define_camera($camera); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC WARPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "DT_WARP", type => "sum", flag => "-dtime_warp", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + { name => "RANGE.XMIN", type => "mean", flag => "-xmin", dtype => "int" }, + { name => "RANGE.XMAX", type => "mean", flag => "-xmax", dtype => "int" }, + { name => "RANGE.YMIN", type => "mean", flag => "-ymin", dtype => "int" }, + { name => "RANGE.YMAX", type => "mean", flag => "-ymax", dtype => "int" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component imfiles for exposure +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $imfiles; +{ + # XXX change -tess_id to -tess_dir when db schema is updated + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_dir"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -scmap: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); +} + +my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id ); +my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id); +my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.WEIGHT", $outroot, $skycell_id); +my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id); +my $outputPSF = $ipprc->filename("PSPHOT.PSF.SAVE", $outroot, $skycell_id); +my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id ); +my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id ); +my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id ); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); +my $logDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); + +my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); +$ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + +# Get list of filenames +my $outrootResolved = $ipprc->file_resolve( $outroot ); +my ($imageFile, $imageName) = tempfile( "$outrootResolved.image.list.XXXX", UNLINK => !$save_temps); +my ($maskFile, $maskName) = tempfile( "$outrootResolved.mask.list.XXXX", UNLINK => !$save_temps); +my ($weightFile, $weightName) = tempfile( "$outrootResolved.weight.list.XXXX", UNLINK => !$save_temps); +my ($astromFile, $astromName) = tempfile( "$outrootResolved.astrom.list.XXXX", UNLINK => !$save_temps); + +my $wrote_astrom = 0; +foreach my $imfile (@$imfiles) { + my $image = $imfile->{uri}; # Image name + my $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + my $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + + &my_die("Couldn't find input file: $image", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image); + &my_die("Couldn't find input file: $mask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($mask); + + my $astrom; # Astrometry file + if ($astromSource eq 'PSASTRO.OUTPUT.MEF') { + $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base}); + } elsif ($astromSource eq 'PSASTRO.OUTPUT') { + my $chipRoot = $imfile->{chip_path_base}; + my $classID = $imfile->{class_id}; + $astrom = $ipprc->filename($astromSource, $chipRoot, $classID); # Astrometry file + } + + &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom); + + print $imageFile "$image\n"; + print $maskFile "$mask\n"; + print $weightFile "$weight\n"; + + if (!$wrote_astrom) { + print $astromFile "$astrom\n"; + # XXX simplify this? only ever have one file for astrometry (must be MEF) + if ($astromSource eq 'PSASTRO.OUTPUT.MEF') { + $wrote_astrom = 1; + } + } +} +close $imageFile; +close $maskFile; +close $weightFile; +close $astromFile; + + +# Run pswarp +my $accept = 1; # Accept the skycell? +unless ($no_op) { + my $command = "$pswarp"; + $command .= " -list $imageName"; + $command .= " -masklist $maskName"; + $command .= " -weightlist $weightName"; + $command .= " -astromlist $astromName"; + $command .= " $outroot $skyFile"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -psf"; # Turn on PSF determination + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code); + } + + # Check first for the stats file, and if the ACCEPT flag is set. + &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + # Get the statistics on the warped image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $contents = join "", @contents; + + my $metadata = $mdcParser->parse($contents) + or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); + $accept = metadataLookupBool($metadata, "ACCEPT"); + if ($accept) { + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); + + &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); +# &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); +# &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + } +} + +unless ($no_update) { + # XXX change -tess_id to -tess_dir when db is updated + my $command = "$warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_dir"; + $command .= " -ignore" if not $accept; # Completed succesfully, but can't produce product + $command .= " -uri $outputImage -path_base $outroot" if $accept; + $command .= $stats->cmdflags() if $accept; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $skycell_id = shift; # Skycell identifier + my $tess_dir = shift; # Tesselation identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if (defined $warp_id and defined $skycell_id and defined $tess_dir and not $no_update) { + # XXX change -tess_id to -tess_dir when db is updated + my $command = "$warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_dir -code $exit_code"; + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branches/eam_branch_20080618/ippScripts/t/00_distribution.t =================================================================== --- /branches/eam_branches/eam_branch_20080618/ippScripts/t/00_distribution.t (revision 22155) +++ /branches/eam_branches/eam_branch_20080618/ippScripts/t/00_distribution.t (revision 22155) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005 Joshua Hoblitt +# +# $Id: 00_distribution.t,v 1.1 2007-01-30 01:52:16 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use lib qw( ./lib ./t ); + +use Test::More; + +# example taken from Test::Distribution Pod + +BEGIN { + eval { + require Test::Distribution; + }; + if($@) { + plan skip_all => 'Test::Distribution not installed'; + } else { + import Test::Distribution; # not => qw( podcover ); + } +}