Index: /branches/eam_branch_20080121/ippScripts/.cvsignore =================================================================== --- /branches/eam_branch_20080121/ippScripts/.cvsignore (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/.cvsignore (revision 16175) @@ -0,0 +1,6 @@ +Build +META.yml +Makefile +Makefile.PL +_build +blib Index: /branches/eam_branch_20080121/ippScripts/Build.PL =================================================================== --- /branches/eam_branch_20080121/ippScripts/Build.PL (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/Build.PL (revision 16175) @@ -0,0 +1,72 @@ +use Module::Build; +# See perldoc Module::Build for details of how this works + +Module::Build->new( + module_name => 'ippscripts', + dist_version => '0.01', + author => 'Joshua Hoblitt ', + license => 'gpl', + create_makefile_pl => 'passthrough', + requires => { + 'Astro::FITS::CFITSIO' => 0, + 'Cache::File' => 0, + 'Carp' => 0, + 'Data::Dumper' => 0, + 'File::Basename' => 0, + 'File::Spec' => 0, + 'File::Temp' => 0, + 'Getopt::Long' => 0, + 'IPC::Cmd' => 0, + 'IPC::Run' => 0, + 'Math::Trig' => 0, + 'Pod::Usage' => 0, + 'PS::IPP::Config' => 0, + 'PS::IPP::Operations' => 0, + 'PS::IPP::Metadata::Config' => 0, + 'Statistics::Descriptive' => 0, + 'Storable' => 0, + 'Sys::Hostname' => 0, + }, + recommends => { + 'Test::Distribution' => '1.22', + }, + script_files => [qw( + scripts/ipp_datapath.pl + scripts/ipp_filename.pl + scripts/mdc2list.pl + scripts/register_imfile.pl + scripts/register_exp.pl + scripts/detrend_process_imfile.pl + scripts/detrend_process_exp.pl + scripts/detrend_stack.pl + scripts/detrend_norm_calc.pl + scripts/detrend_norm_apply.pl + scripts/detrend_norm_exp.pl + scripts/detrend_resid.pl + scripts/detrend_reject_imfile.pl + scripts/detrend_reject_exp.pl + scripts/chip_imfile.pl + scripts/camera_exp.pl + scripts/warp_overlap.pl + scripts/warp_skycell.pl + scripts/diff_skycell.pl + scripts/stack_skycell.pl + scripts/magic_tree.pl + scripts/ipp_serial_inject.pl + scripts/ipp_serial_inject_split.pl + scripts/ipp_serial_inject_mosaic.pl + scripts/ipp_serial_register.pl + scripts/ipp_serial_detrend.pl + scripts/ipp_serial_chip.pl + scripts/ipp_serial_camera.pl + scripts/ipp_serial_warp.pl + scripts/ipp_serial_diff.pl + scripts/ipp_serial_stack.pl + scripts/ipp_simulation_data.pl + scripts/isp_trans.pl + scripts/ds9_cmf_regions.pl + scripts/ipp_detrend_combine.pl + )], + dist_abstract => 'Scripts for running the Pan-STARRS IPP', + dist_author => 'Paul Price ', +)->create_build_script; Index: /branches/eam_branch_20080121/ippScripts/Changes =================================================================== --- /branches/eam_branch_20080121/ippScripts/Changes (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/Changes (revision 16175) @@ -0,0 +1,6 @@ +Revision history for Perl module ippScripts + +0.01 Thu Feb 23 13:55:57 2006 + - help scripts split out of the ippTools package + + Index: /branches/eam_branch_20080121/ippScripts/LICENSE =================================================================== --- /branches/eam_branch_20080121/ippScripts/LICENSE (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/LICENSE (revision 16175) @@ -0,0 +1,260 @@ +The General Public License (GPL) +Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. 675 Mass Ave, +Cambridge, MA 02139, USA. Everyone is permitted to copy and distribute +verbatim copies of this license document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to share +and change it. By contrast, the GNU General Public License is intended to +guarantee your freedom to share and change free software--to make sure the +software is free for all its users. This General Public License applies to most of +the Free Software Foundation's software and to any other program whose +authors commit to using it. (Some other Free Software Foundation software is +covered by the GNU Library General Public License instead.) You can apply it to +your programs, too. + +When we speak of free software, we are referring to freedom, not price. Our +General Public Licenses are designed to make sure that you have the freedom +to distribute copies of free software (and charge for this service if you wish), that +you receive source code or can get it if you want it, that you can change the +software or use pieces of it in new free programs; and that you know you can do +these things. + +To protect your rights, we need to make restrictions that forbid anyone to deny +you these rights or to ask you to surrender the rights. These restrictions +translate to certain responsibilities for you if you distribute copies of the +software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis or for a +fee, you must give the recipients all the rights that you have. You must make +sure that they, too, receive or can get the source code. And you must show +them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and (2) offer +you this license which gives you legal permission to copy, distribute and/or +modify the software. + +Also, for each author's protection and ours, we want to make certain that +everyone understands that there is no warranty for this free software. If the +software is modified by someone else and passed on, we want its recipients to +know that what they have is not the original, so that any problems introduced by +others will not reflect on the original authors' reputations. + +Finally, any free program is threatened constantly by software patents. We wish +to avoid the danger that redistributors of a free program will individually obtain +patent licenses, in effect making the program proprietary. To prevent this, we +have made it clear that any patent must be licensed for everyone's free use or +not licensed at all. + +The precise terms and conditions for copying, distribution and modification +follow. + +GNU GENERAL PUBLIC LICENSE +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND +MODIFICATION + +0. This License applies to any program or other work which contains a notice +placed by the copyright holder saying it may be distributed under the terms of +this General Public License. The "Program", below, refers to any such program +or work, and a "work based on the Program" means either the Program or any +derivative work under copyright law: that is to say, a work containing the +Program or a portion of it, either verbatim or with modifications and/or translated +into another language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not covered by +this License; they are outside its scope. The act of running the Program is not +restricted, and the output from the Program is covered only if its contents +constitute a work based on the Program (independent of having been made by +running the Program). Whether that is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source code as +you receive it, in any medium, provided that you conspicuously and appropriately +publish on each copy an appropriate copyright notice and disclaimer of warranty; +keep intact all the notices that refer to this License and to the absence of any +warranty; and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and you may at +your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of it, thus +forming a work based on the Program, and copy and distribute such +modifications or work under the terms of Section 1 above, provided that you also +meet all of these conditions: + +a) You must cause the modified files to carry prominent notices stating that you +changed the files and the date of any change. + +b) You must cause any work that you distribute or publish, that in whole or in +part contains or is derived from the Program or any part thereof, to be licensed +as a whole at no charge to all third parties under the terms of this License. + +c) If the modified program normally reads commands interactively when run, you +must cause it, when started running for such interactive use in the most ordinary +way, to print or display an announcement including an appropriate copyright +notice and a notice that there is no warranty (or else, saying that you provide a +warranty) and that users may redistribute the program under these conditions, +and telling the user how to view a copy of this License. (Exception: if the +Program itself is interactive but does not normally print such an announcement, +your work based on the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If identifiable +sections of that work are not derived from the Program, and can be reasonably +considered independent and separate works in themselves, then this License, +and its terms, do not apply to those sections when you distribute them as +separate works. But when you distribute the same sections as part of a whole +which is a work based on the Program, the distribution of the whole must be on +the terms of this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest your rights to +work written entirely by you; rather, the intent is to exercise the right to control +the distribution of derivative or collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program with the +Program (or with a work based on the Program) on a volume of a storage or +distribution medium does not bring the other work under the scope of this +License. + +3. You may copy and distribute the Program (or a work based on it, under +Section 2) in object code or executable form under the terms of Sections 1 and 2 +above provided that you also do one of the following: + +a) Accompany it with the complete corresponding machine-readable source +code, which must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange; or, + +b) Accompany it with a written offer, valid for at least three years, to give any +third party, for a charge no more than your cost of physically performing source +distribution, a complete machine-readable copy of the corresponding source +code, to be distributed under the terms of Sections 1 and 2 above on a medium +customarily used for software interchange; or, + +c) Accompany it with the information you received as to the offer to distribute +corresponding source code. (This alternative is allowed only for noncommercial +distribution and only if you received the program in object code or executable +form with such an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for making +modifications to it. For an executable work, complete source code means all the +source code for all modules it contains, plus any associated interface definition +files, plus the scripts used to control compilation and installation of the +executable. However, as a special exception, the source code distributed need +not include anything that is normally distributed (in either source or binary form) +with the major components (compiler, kernel, and so on) of the operating system +on which the executable runs, unless that component itself accompanies the +executable. + +If distribution of executable or object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the source +code from the same place counts as distribution of the source code, even though +third parties are not compelled to copy the source along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program except as +expressly provided under this License. Any attempt otherwise to copy, modify, +sublicense or distribute the Program is void, and will automatically terminate +your rights under this License. However, parties who have received copies, or +rights, from you under this License will not have their licenses terminated so long +as such parties remain in full compliance. + +5. You are not required to accept this License, since you have not signed it. +However, nothing else grants you permission to modify or distribute the Program +or its derivative works. These actions are prohibited by law if you do not accept +this License. Therefore, by modifying or distributing the Program (or any work +based on the Program), you indicate your acceptance of this License to do so, +and all its terms and conditions for copying, distributing or modifying the +Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the Program), +the recipient automatically receives a license from the original licensor to copy, +distribute or modify the Program subject to these terms and conditions. You +may not impose any further restrictions on the recipients' exercise of the rights +granted herein. You are not responsible for enforcing compliance by third parties +to this License. + +7. If, as a consequence of a court judgment or allegation of patent infringement +or for any other reason (not limited to patent issues), conditions are imposed on +you (whether by court order, agreement or otherwise) that contradict the +conditions of this License, they do not excuse you from the conditions of this +License. If you cannot distribute so as to satisfy simultaneously your obligations +under this License and any other pertinent obligations, then as a consequence +you may not distribute the Program at all. For example, if a patent license would +not permit royalty-free redistribution of the Program by all those who receive +copies directly or indirectly through you, then the only way you could satisfy +both it and this License would be to refrain entirely from distribution of the +Program. + +If any portion of this section is held invalid or unenforceable under any particular +circumstance, the balance of the section is intended to apply and the section as +a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any patents or other +property right claims or to contest validity of any such claims; this section has +the sole purpose of protecting the integrity of the free software distribution +system, which is implemented by public license practices. Many people have +made generous contributions to the wide range of software distributed through +that system in reliance on consistent application of that system; it is up to the +author/donor to decide if he or she is willing to distribute software through any +other system and a licensee cannot impose that choice. + +This section is intended to make thoroughly clear what is believed to be a +consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in certain countries +either by patents or by copyrighted interfaces, the original copyright holder who +places the Program under this License may add an explicit geographical +distribution limitation excluding those countries, so that distribution is permitted +only in or among countries not thus excluded. In such case, this License +incorporates the limitation as if written in the body of this License. + +9. The Free Software Foundation may publish revised and/or new versions of the +General Public License from time to time. Such new versions will be similar in +spirit to the present version, but may differ in detail to address new problems or +concerns. + +Each version is given a distinguishing version number. If the Program specifies a +version number of this License which applies to it and "any later version", you +have the option of following the terms and conditions either of that version or of +any later version published by the Free Software Foundation. If the Program does +not specify a version number of this License, you may choose any version ever +published by the Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free programs +whose distribution conditions are different, write to the author to ask for +permission. For software which is copyrighted by the Free Software Foundation, +write to the Free Software Foundation; we sometimes make exceptions for this. +Our decision will be guided by the two goals of preserving the free status of all +derivatives of our free software and of promoting the sharing and reuse of +software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS +NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE +COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM +"AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR +IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE +ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, +YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR +CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED +TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY +WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS +PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM +(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY +OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS Index: /branches/eam_branch_20080121/ippScripts/MANIFEST =================================================================== --- /branches/eam_branch_20080121/ippScripts/MANIFEST (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/MANIFEST (revision 16175) @@ -0,0 +1,38 @@ +Build.PL +Changes +LICENSE +MANIFEST +README +scripts/ipp_datapath.pl +scripts/ipp_filename.pl +scripts/mdc2list.pl +scripts/register_imfile.pl +scripts/register_exp.pl +scripts/detrend_process_imfile.pl +scripts/detrend_process_exp.pl +scripts/detrend_stack.pl +scripts/detrend_norm_calc.pl +scripts/detrend_norm_apply.pl +scripts/detrend_norm_exp.pl +scripts/detrend_resid.pl +scripts/detrend_reject_imfile.pl +scripts/detrend_reject_exp.pl +scripts/chip_imfile.pl +scripts/camera_exp.pl +scripts/warp_overlap.pl +scripts/warp_skycell.pl +scripts/diff_skycell.pl +scripts/stack_skycell.pl +scripts/ipp_serial_inject.pl +scripts/ipp_serial_inject_mosaic.pl +scripts/ipp_serial_register.pl +scripts/ipp_serial_detrend.pl +scripts/ipp_serial_chip.pl +scripts/ipp_serial_camera.pl +scripts/ipp_serial_warp.pl +scripts/ipp_serial_diff.pl +scripts/ipp_serial_stack.pl +scripts/ipp_simulation_data.pl +scripts/isp_trans.pl +scripts/ds9_cmf_regions.pl +t/00_distribution.t Index: /branches/eam_branch_20080121/ippScripts/README =================================================================== --- /branches/eam_branch_20080121/ippScripts/README (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/README (revision 16175) @@ -0,0 +1,15 @@ +pod2text foo.pm > README + +If this is still here it means the programmer was too lazy to create the readme file. + +You can create it now by using the command shown above from this directory. + +At the very least you should be able to use this set of instructions +to install the module... + +perl Build.PL +./Build +./Build test +./Build install + +If you are on a windows box you should use 'nmake' rather than 'make'. Index: /branches/eam_branch_20080121/ippScripts/scripts/calibrate_dvo.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/calibrate_dvo.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/calibrate_dvo.pl (revision 16175) @@ -0,0 +1,205 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dvo_id, $catdir, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'dvo_id|i=s' => \$dvo_id, + 'catdir|c=s' => \$catdir, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dvo_id --catdir --region", + -exitval => 3) unless + defined $dvo_id and + defined $catdir and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addcalrun"; +$command .= " -dvo_id $dvo_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -status SUCCESS"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if ($dvo_id && $region && $last_step && $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/camera_exp.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/camera_exp.pl (revision 16175) @@ -0,0 +1,319 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($exp_tag, $cam_id, $recipe, $camera, $dbname, $workdir, $reduction, $dvodb, $no_update, $no_op); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'cam_id=s' => \$cam_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --cam_id --camera", + -exitval => 3, + ) unless defined $exp_tag + and defined $cam_id + and defined $camera; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2'); # Recipe to use +&my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +my $CHIPSTATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + +# these should be coming from the psastro results for Nimfile > 1, + { name => "sigma_ra", type => "rms", flag => "-sigma_ra", dtype => "float" }, + { name => "sigma_dec", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "n_stars", type => "sum", flag => "-n_stars", dtype => "int" }, + { name => "n_astrom", type => "sum", flag => "-n_astrom", dtype => "int" }, + + { name => "fwhm", type => "mean", flag => "-fwhm", dtype => "float" }, + +# these are not defined for the database table camProcessedExp +# { name => "ap_resid", type => "mean", flag => "-ap_resid", dtype => "float" }, +# { name => "ap_resid_stdev", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, +# { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, +# { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, +# { name => "fringe_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + ]; +my $chipStats = PS::IPP::Metadata::Stats->new($CHIPSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); + +# test for addstar and psastro: +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$camtool -pendingimfile -cam_id $cam_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", $cam_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + unless ($chipStats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $cam_id, $PS_EXIT_PROG_ERROR); + } +} + +my ($list1File, $list1Name) = tempfile( "$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => 1 ); # For binning 1 +my ($list2File, $list2Name) = tempfile( "$exp_tag.cm.$cam_id.b2.list.XXXX", UNLINK => 1 ); # For binning 2 +my ($list3File, $list3Name) = tempfile( "$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => 1 ); # For astrometry + +my $chipObjects; +my $chipObjectsExist = 0; +foreach my $file (@$files) { + # use the path_base as OUTPUT root and convert the filenames with ipprc->filename: + my $class_id = $file->{class_id}; + + # If there is only one chip, we use this name for the input to addstar + $chipObjects = $ipprc->filename("PSASTRO.OUTPUT", $file->{path_base}, $class_id); + print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n"); + print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n"); + print $list3File ($chipObjects . "\n"); + + # if any of the output chip astrometry files exist, we can run psastro / addstar below + if ($ipprc->file_exists($chipObjects)) { + $chipObjectsExist = 1; + } +} +close $list1File; +close $list2File; +close $list3File; + +# Output products +$workdir = caturi( $workdir, $exp_tag ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$exp_tag.cm.$cam_id", $workdir, ${$files}[0]->{path_base} ); + +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1 +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2 +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT.MEF", $outputRoot); # MEF psastro output + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".log"; # Log messages + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->convert_filename_absolute( $dvodb ); +} + +unless ($no_op) { + + ## build the output JPEG images first so we get them even if the astrometry fails + + # Make the jpeg for binning 1 + { + my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg1); + } + + # Make the jpeg for binning 2 + { + my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg2); + } + + # check recipe for PSASTRO.MOSAIC.MODE; run mosaic astrometry if + # requested by the camera's PSASTRO recipe + my $mosaicAstrom; # run mosaic-level astrometry? + { + my $command = "$ppConfigDump -camera $camera -dump-recipe PSASTRO -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR); + $mosaicAstrom = metadataLookupBool($metadata, 'PSASTRO.MOSAIC.MODE'); + } + + # only run psastro / addstar if any of the output chip astrometry files exist (should we test for successful astrometry?) + if ($chipObjectsExist) { + if ($mosaicAstrom) { + # run psastro +mosastro on the set of chips or copy the chipObjects to fpaObjects + # XXX note that this is wrong if imfiles are cells + # XXX add a ppStats call which will collect the astrometry stats + my $command; + $command = "$psastro -list $list3Name $outputRoot"; + $command .= " +mosastro -chipastro"; + $command .= " -F PSASTRO.OUTPUT PSASTRO.OUTPUT.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code); + } + # XXX do we want to give an error if astrometry fails here? + &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); + } + + # do we supply chipObjects or fpaObjects to addstar? + # run addstar on either the single chip output or the single fpa output + if (defined $dvodbReal) { + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $command; + $command = "$addstar -D CAMERA $camdir "; + $command .= " -D CATDIR $dvodbReal"; + + if ($mosaicAstrom) { + $command .= " $ipprc->file_resolve($fpaObjects)"; + } else { + print STDERR "ERROR: addstar needs to be update to support the -list input structure\n"; + $command .= " -list $list3File"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); + } + } + } +} + +my $fpaCommand = "$camtool -addprocessedexp"; +$fpaCommand .= " -cam_id $cam_id"; +$fpaCommand .= " -uri UNKNOWN"; +$fpaCommand .= " -path_base $outputRoot"; +$fpaCommand .= " -dbname $dbname" if defined $dbname; +$fpaCommand .= $chipStats->cmdflags(); + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedexp: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $cam_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($cam_id and not $no_update) { + my $command = "$camtool -addprocessedexp -cam_id $cam_id -uri UNKNOWN -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/chip_imfile.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/chip_imfile.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/chip_imfile.pl (revision 16175) @@ -0,0 +1,210 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($exp_id, $chip_id, $class_id, $uri, $exp_tag, $camera, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'exp_id=s' => \$exp_id, # Exposure identifier + 'chip_id=s' => \$chip_id, # Chiptool identifier + 'class_id=s' => \$class_id, # Class identifier + 'uri|u=s' => \$uri, # Input FITS file + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction=s' => \$reduction, # Reduction class + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --exp_tag --chip_id --class_id --uri --camera", + -exitval => 3) unless + defined $exp_id and + defined $chip_id and + defined $class_id and + defined $uri and + defined $exp_tag and + defined $camera; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use +unless ($recipe) { + &my_die("Couldn't find selected reduction for CHIP: $reduction\n", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "FRINGE_0", type => "rms", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_2", dtype => "float" }, + { name => "OVER_VAL", type => "mean", flag => "-bias", dtype => "float" }, + { name => "FWHM_X", type => "mean", flag => "-fwhm", dtype => "float" }, + { name => "FWHM_Y", type => "mean", flag => "-fwhm_range", dtype => "float" }, + { name => "APMIFIT", type => "mean", flag => "-ap_resid", dtype => "float" }, + { name => "DAPMIFIT", type => "rms", flag => "-ap_resid_stdev", dtype => "float" }, + { name => "CERROR", type => "rms", flag => "-sigma_ra", dtype => "float" }, + { name => "CERROR", type => "rms", flag => "-sigma_dec", dtype => "float" }, + { name => "NSTARS", type => "sum", flag => "-n_stars", dtype => "int" }, + { name => "NASTRO", type => "sum", flag => "-n_astrom", dtype => "int" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + +$workdir = caturi( $workdir, $exp_tag ) if defined $workdir; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag.ch.$chip_id", $workdir, $uri ); + +my $outputImage = $ipprc->filename("PPIMAGE.CHIP", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputWeight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.log"; # Log messages + +# Run ppImage +unless ($no_op) { + my $command = "$ppImage -file $uri $outputRoot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + &my_die("Couldn't find expected output file: $outputMask\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputBin1\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + &my_die("Couldn't find expected output file: $outputStats\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new(); # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# command to update database +my $command = "$chiptool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -chip_id $chip_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage"; +$command .= " -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; # Chiptool identifier + my $chip_id = shift; # Chiptool identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($chip_id and $class_id and not $no_update) { + my $command = "$chiptool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -chip_id $chip_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -uri $outputImage"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_correct_imfile.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_correct_imfile.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_correct_imfile.pl (revision 16175) @@ -0,0 +1,151 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'corr_id|u=s' => \$corr_id, + 'corr_type|u=s' => \$corr_type, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera", + -exitval => 3) + unless defined $det_id + and defined $exp_id + and defined $class_id + and defined $det_type + and defined $exp_tag + and defined $input_uri + and defined $camera; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $dvocorr = can_run('dvoApplyCorr') or (warn "Can't find dvoApplyCorr" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +# detselect -select -det_id $corr_id -class_id $class_id; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); +my $outputImage = $ipprc->filename("DVOCORR.OUTPUT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + +# Run dvoApplyCorr +unless ($no_op) { + my $command = "$dvocorr -file $input_uri $outputRoot"; + $command .= " -corr $corr_uri"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); +} + +# command to update database +my $command = "$dettool -addcorrectedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputImage -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addcorrectedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $exp_id and $class_id and not $no_update) { + my $command = "$dettool -addcorrectedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_apply.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_apply.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_apply.pl (revision 16175) @@ -0,0 +1,197 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line +my ($det_id, $iter, $class_id, $value, $input_uri, $camera, $det_type, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend ID + 'iteration|n=s' => \$iter, # Iteration + 'class_id|i=s' => \$class_id, # Class ID + 'value|v=s' => \$value, # Value to multiple (for normalisation) + 'input_uri|u=s' => \$input_uri, # Input file + 'camera|c=s' => \$camera, # Camera + 'det_type|t=s' => \$det_type, # Detrend type + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database + 'no-op' => \$no_op, # Don't do any operations + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --value --input_uri --camera --det_type", + -exitval => 3) + unless defined $det_id + and defined $iter + and defined $class_id + and defined $value + and defined $input_uri + and defined $camera + and defined $det_type; + +$ipprc->define_camera($camera); + +my $RECIPE_PPIMAGE = 'PPIMAGE_N'; # Recipe to use with ppImage +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +my $isMaskType = 0; +if (lc($det_type) eq "mask") { $isMaskType = 1; } +if (lc($det_type) eq "darkmask") { $isMaskType = 1; } +if (lc($det_type) eq "flatmask") { $isMaskType = 1; } + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +my $outputRoot = $ipprc->file_prepare( "$camera.$det_type.norm.$det_id.$iter", $workdir, $input_uri ); + +my $output = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id); +my $b1name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id); +my $b2name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id); +my $statsName = $ipprc->filename("PPIMAGE.STATS", $outputRoot, $class_id); + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.log"; # Log messages + +# Run normalisation +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outputRoot"; + $command .= " -norm $value -stats $statsName"; + $command .= " -recipe PPIMAGE $RECIPE_PPIMAGE"; + $command .= " -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= " -F PPIMAGE.OUTPUT PPIMAGE.OUTPUT.DETMASK" if ($isMaskType); # XXXX something of a hack (too many places to control things...) + $command .= ' -isfringe' if lc($det_type) eq 'fringe'; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Can't find expected output file: $output", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($output); + &my_die("Can't find expected output file: $b1name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b1name); + &my_die("Can't find expected output file: $b2name", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b2name); + &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($statsName); + + # Get the statistics on the normalised image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($statsName) or &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $statsFile; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents); + unless ($metadata) { + &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + } +} + +# Command to update the database +my $command = "$dettool -addnormalizedimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $output"; +$command .= " -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addnormalizedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and $class_id and not $no_update) { + my $command = "$dettool -addnormalizedimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_calc.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_calc.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_calc.pl (revision 16175) @@ -0,0 +1,226 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run ); +use IPC::Run 0.36 qw( run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse command-line arguments +my ($det_id, $iter, $detType, $workdir, $dbname, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, # Detrend id + 'iteration|i=s' => \$iter, # Iteration + 'det_type|t=s' => \$detType, # Detrend type + 'workdir|w=s' => \$workdir, # Working directory for output files + 'dbname|d=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do operations + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_id --iteration --det_type", + -exitval => 3, + ) + unless defined $det_id + and defined $iter + and defined $detType; + +use constant STATISTIC => 'bg'; # Background statistic to use from the database + +# Define which detrend types we normalise +use constant NORMALIZE => { + 'bias' => 0, + 'dark' => 0, + 'shutter' => 0, + 'flat' => 1, + 'domeflat' => 1, + 'skyflat' => 1, + 'fringe' => 0, + 'mask' => 0, + 'darkmask' => 0, + 'flatmask' => 0, + }; + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppNormCalc = can_run('ppNormCalc') or (warn "Can't find ppNormCalc" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless exists NORMALIZE()->{lc($detType)}; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my @files; # The input files +{ + my $command = "$dettool -processedimfile -det_id $det_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my @command = split /\s+/, $command; + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n"; + if (not run(\@command, \$stdin, \$stdout, \$stderr)) { + &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?", + $det_id, $iter, $PS_EXIT_SYS_ERROR); + } +# print $stdout . "\n"; + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my @whole = split /\n/, $stdout; + my @single = (); + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from dettool", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless + defined $list; + push @files, @$list; + @single = (); + } + } +} + +my $norms; # MDC with normalisations +if (NORMALIZE()->{lc($detType)} and not $no_op) { + + my %matrix; # Matrix of statistics as a function of exposures and classes + foreach my $file (@files) { + my $exp_id = $file->{'exp_id'}; # Exposure ID + my $class_id = $file->{'class_id'}; # Class ID + my $stat = $file->{STATISTIC()}; # Statistic of interest + + # Create matrix elements + $matrix{$exp_id} = {} if not defined $matrix{$exp_id}; + $matrix{$exp_id}->{$class_id} = $stat; + } + + # Generate the input for ppNormCalc + my $normData; # Normalisation data + foreach my $exp_id (keys %matrix) { + $normData .= "$exp_id\tMETADATA\n"; + foreach my $class_id (keys %{$matrix{$exp_id}}) { + $normData .= "\t" . $class_id . "\tF32\t" . $matrix{$exp_id}->{$class_id} . "\n"; + } + $normData .= "END\n\n"; + } + + # Run ppNormCalc + { + my ( $stdout, $stderr ); # Buffers for running program + my @command = split /\s+/, $ppNormCalc; + print "Running [$ppNormCalc]...\n"; + if (not run(\@command, \$normData, \$stdout, \$stderr)) { + &my_die("Unable to perform ppNormCalc: $?", $det_id, $iter, $PS_EXIT_SYS_ERROR); + } + print $stdout . "\n"; + + # Parse the output + $norms = $mdcParser->parse($stdout); + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless $norms; + } + +} else { + # It's something that doesn't need normalisation --- just push in a normalisation of 1 + my %classes; # List of unique classes + foreach my $file (@files) { + my $class_id = $file->{'class_id'}; # Class Id + $classes{$class_id} = 1; + } + + foreach my $class_id (keys %classes) { + my %mdValue; # Metadata value for this class id + $mdValue{name} = $class_id; + $mdValue{value} = 1.0; + push @$norms, \%mdValue; + } +} + +my $commandBase = "$dettool -addnormalizedstat"; +$commandBase .= " -det_id $det_id"; +$commandBase .= " -iteration $iter"; +$commandBase .= " -dbname $dbname" if defined $dbname; + +# Process output normalisations +unless ($no_update) { + foreach my $normItem (@$norms) { + + my $className = $normItem->{name}; # Name of component + my $normalisation = $normItem->{value}; # Normalisation for component + + my $command = $commandBase; + $command .= " -class_id $className"; + $command .= " -norm $normalisation"; + + my @command = split /\s+/, $command; + + my ( $stdin, $stdout, $stderr ); # Buffers for running program + print "Running [$command]...\n"; + if (not run \@command, \$stdin, \$stdout, \$stderr) { + warn("Unable to perform dettool -addnormstat for $className: $?"); + exit($PS_EXIT_SYS_ERROR); + } + print $stdout . "\n"; + } +} else { + print "skipping command: $commandBase\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and not $no_update) { + my $command = "$dettool -addnormalizedstat"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_exp.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_exp.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_norm_exp.pl (revision 16175) @@ -0,0 +1,196 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration|i=s' => \$iter, + 'camera|c=s' => \$camera, + 'det_type|t=s' => \$det_type, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction|=s' => \$reduction, + 'no-update' => \$no_update, + 'no-op' => \$no_op, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --camera --det_type", + -exitval => 3) + unless defined $det_id + and defined $iter + and defined $camera + and defined $det_type; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -normalizedimfile -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to get list of normalized imfiles from dettool: $error_code", $det_id, $iter, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR); + } +} + +my ($list1File, $list1Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b1.list.XXXX", UNLINK => 1 ); +my ($list2File, $list2Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b2.list.XXXX", UNLINK => 1 ); +foreach my $file (@$files) { + print $list1File ( $ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ( $ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# Output products +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$camera.$det_type.norm.$det_id.$iter", $workdir ); +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); + +} + +# command to update the database +$command = "$dettool -addnormalizedexp"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -recip $recipe1,$recipe2"; +$command .= " -path_base $outputRoot "; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -addnormalizedexp: $error_code", $det_id, $iter, $error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and not $no_update) { + my $command = "$dettool -addprocessedimfile"; + $command .= " -det_id $det_id"; + $command .= " -iter $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_process_exp.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_process_exp.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_process_exp.pl (revision 16175) @@ -0,0 +1,206 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $exp_id, $det_type, $exp_tag, $camera, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'det_type|t=s' => \$det_type, + 'exp_id|d=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction|=s' => \$reduction, + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --det_type --exp_id --exp_tag --camera", + -exitval => 3) + unless defined $det_id + and defined $det_type + and defined $exp_id + and defined $exp_tag + and defined $camera; + +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_id, $error_code); + } + + # convert stdout to a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the stats in the metadata + unless ($stats->parse($metadata)) { + &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR); + } +} + +# File lists for the two binnings +my ($list1File, $list1Name) = tempfile( "$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => 1 ); +my ($list2File, $list2Name) = tempfile( "$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => 1 ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + +# Output files +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, ${$files}[0]->{path_base} ); +my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1 +my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2 + +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2); +} + +# Command to update the database +$command = "$dettool -addprocessedexp"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedexp: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $exp_id and not $no_update) { + my $command = "$dettool -addprocessedexp"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_process_imfile.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_process_imfile.pl (revision 16175) @@ -0,0 +1,188 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'exp_tag|=s' => \$exp_tag, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction=s' => \$reduction, # Reduction class + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera", + -exitval => 3) + unless defined $det_id + and defined $exp_id + and defined $class_id + and defined $det_type + and defined $exp_tag + and defined $input_uri + and defined $camera; + +# XXX this exits with status = 0 on failure +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} +$ppImage .= " -dbname $dbname" if defined $dbname; + +&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri ); + +my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.log"; # Log messages + +# Run ppImage +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outputRoot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code); + } + + &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1); + &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2); + + # Get the statistics on the processed image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve("$outputStats") or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # parse the statistics MDC file + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) + or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # extract the stats from the metadata + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# command to update database +my $command = "$dettool -addprocessedimfile"; +$command .= " -det_id $det_id"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $reduction"; +$command .= " -uri $outputImage -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addprocessedimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $exp_id and $class_id and not $no_update) { + my $command = "$dettool -addprocessedimfile"; + $command .= " -det_id $det_id"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_reject_exp.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_reject_exp.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_reject_exp.pl (revision 16175) @@ -0,0 +1,356 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); +use Statistics::Descriptive; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my $ITER_LIMIT = 20; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $det_type, $camera, $filter, $workdir, $dbname, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'filter=s' => \$filter, + 'workdir|w=s' => \$workdir, # Working directory for output files + 'dbname|d=s' => \$dbname, # Database name + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera", + -exitval => 3) + unless defined $det_id + and defined $iter + and defined $det_type + and defined $camera; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# these stats are used it the rejections but not passed to the database +# there is some duplication with the above, but the calculation time is minimal +my $REJSTATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "ensMeanMean", dtype => "float" }, + { name => "bg", type => "stdev", flag => "ensMeanStdev", dtype => "float" }, + { name => "bg_mean_stdev", type => "mean", flag => "ensMeanStdevMean", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "ensMeanStdevStdev", dtype => "float" }, + { name => "bg_stdev", type => "mean", flag => "ensStdevMean", dtype => "float" }, + { name => "bg_stdev", type => "stdev", flag => "ensStdevStdev", dtype => "float" }, + { name => "bg_skewness", type => "mean", flag => "ensSkewness", dtype => "float" }, + { name => "bg_kurtosis", type => "mean", flag => "ensKurtosis", dtype => "float" }, + + ]; +my $rejstats = PS::IPP::Metadata::Stats->new($REJSTATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component files +my ($exposures, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select exp data for this det_run + $command = "$dettool -residexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -residexp: $error_code", $det_id, $iter, $error_code); + } + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $exposures = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); + + # Parse the statistics for rejections + $rejstats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR); +} + +# we use the statistics of the ensemble to accept/reject exposurs +my $ensMeanMean = $rejstats->value_for_flag ("ensMeanMean"); +my $ensMeanStdev = $rejstats->value_for_flag ("ensMeanStdev"); +my $ensMeanStdevMean = $rejstats->value_for_flag ("ensMeanStdevMean"); +my $ensMeanStdevStdev = $rejstats->value_for_flag ("ensMeanStdevStdev"); +my $ensStdevMean = $rejstats->value_for_flag ("ensStdevMean"); +my $ensStdevStdev = $rejstats->value_for_flag ("ensStdevStdev"); + +$ipprc->define_camera($camera); +# Rejection thresholds +my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter ); +my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter ); +my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter ); + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +my $logName = $ipprc->file_prepare( "$camera.$det_type.$det_id.$iter.detreject.log", $workdir ); # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); + print $logFile "Ensemble mean $ensMeanMean +/- $ensMeanStdevMean, stdev $ensStdevMean\n\n"; +} + +# Go through again to do rejection, and update the database for each exposure +my $numChanges = 0; # Number of exposures with changed status +my $numReject = 0; # Number of exposures rejected +my $firstElement = 1; + +foreach my $exposure (@$exposures) { + my $mean = $exposure->{bg}; # Mean for this exposure + my $stdev = $exposure->{bg_stdev}; # Stdev for this exposure + my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure + my $expID = $exposure->{exp_id}; + my $accept = $exposure->{accept}; + my $include = $exposure->{include}; + + &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $expID; + &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $accept; + &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include; + + my $reject = 0; # Reject this exposure? + + $command = "$dettool -updateresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $expID"; + $command .= " -dbname $dbname" if defined $dbname; + + if (not $accept) { + # Rejected this at an earlier stage + unless ($no_op) { + print $logFile "Rejecting $expID based on earlier determination.\n"; + } + $reject = 1; + goto UPDATE; + } + + # Cop-out if we're not doing any operations + if ($no_op) { + # Make sure something gets rejected (just once!), just so that + # we can trace the full range of the workflow + if ($firstElement and $iter == 0) { + $reject = 1; + } + goto UPDATE; + } + + if ($reject_mean > 0 and $ensMeanStdev > 0) { + my $delta = abs($mean - $ensMeanMean); + if ($delta > ($reject_mean * $ensMeanStdev)) { + print $logFile "Rejecting $expID based on ensemble mean value: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean: "; + } + print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean\n"; + } + + if ($reject_stdev > 0 and $ensStdevStdev > 0) { + my $delta = abs($stdev - $ensStdevMean); + if ($delta > ($reject_stdev * $ensStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble stdev: "; + } + print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n"; + } else { + print $logFile "No rejection of $expID for ensemble stdev\n"; + } + + if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) { + my $delta = abs($meanStdev - $ensMeanStdevMean); + if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) { + print $logFile "Rejecting $expID based on ensemble mean stdev: "; + $reject = 1; + #goto UPDATE; + } else { + print $logFile "$expID OK against ensemble mean stdev: "; + } + print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n"; + } else { + print $logFile "No rejection of $expID for ensemble mean stdev\n"; + } + + UPDATE: + if ($reject) { + $command .= ' -reject'; + $numReject++; + } + + # Check for status changes + if ((not $include and not $reject) or ($include and $reject)) { + unless ($no_op) { + print $logFile "Status of $expID has changed.\n"; + } + $numChanges++; + } + + unless ($no_update) { + # Update + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -updateresidexp: $error_code", $det_id, $iter, $error_code); + } + } +} + +# Decide if the current is sufficient to use as a master, and if we can stop iterating +my $master = 1; # This is good enough for a master +my $stop = 1; # Stop iterating + +if ($numChanges > 0) { + $master = 0; + $stop = 0; +} + +# Rejecting everything --- stop before something bad happens! +if ($numReject == scalar @$exposures) { + $master = 0; + $stop = 1; +} + +unless ($no_op) { + print $logFile "Master: $master\n"; + print $logFile "Stop: $stop\n"; + close $logFile; +} + +# attempt to prevent endless, pathological iterations +if ($iter >= $ITER_LIMIT) { + warn("iteration limit reached -- bailing out"); + exit($PS_EXIT_PROG_ERROR); +} + +## add the summary statistics, and request a new iteration if needed +$command = "$dettool -adddetrunsummary"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -accept" if $master; +$command .= " -dbname $dbname" if defined $dbname; +$command .= " -again" unless $stop; +$command .= $stats->cmdflags(); + +# Put results into the database +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -adddetrunsummary: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and not $no_update) { + my $command = "$dettool -adddetrunsummary"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying unless extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" unless defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_reject_imfile.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_reject_imfile.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_reject_imfile.pl (revision 16175) @@ -0,0 +1,599 @@ +#!/usr/bin/env perl + +# this program has two jobs: +# 1: for the given exp_tag, generate the binned & mosaiced JPEG images +# 2: examine the collection of per-imfile statistics and reject. At the moment, +# this program (and the database) only allows rejection at the exposure level, +# not at the imfile level. + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); # tools to run UNIX programs with control over I/O +use PS::IPP::Metadata::Config; # tools to parse the psMetadataConfig files +use PS::IPP::Metadata::Stats; + +use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list +use Statistics::Descriptive; # tools for calculating basic statistical quantities +use File::Temp qw( tempfile ); # tools to construct temp files + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); # tools to parse the IPP configuration information +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); # option parsing +use Pod::Usage qw( pod2usage ); + +# parse the command-line options +my ($det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'det_type|t=s' => \$det_type, + 'camera=s' => \$camera, + 'filter=s' => \$filter, + 'reject' => \$reject, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction|=s' => \$reduction, + 'no-update' => \$no_update, + 'no-op' => \$no_op, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --det_type --camera", + -exitval => 3) + unless defined $det_id + and defined $iter + and defined $exp_id + and defined $exp_tag + and defined $det_type + and defined $camera; + +# load IPP config information for the specified camera +$ipprc->define_camera($camera); + +# Recipes to use based on reduction class +$reduction = 'DETREND' unless defined $reduction; + +my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1; + +my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID_' . uc($det_type)); # Recipe to use +&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2; + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg_mean_stdev", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "bg_skewness", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "bg_kurtosis", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "bin_stdev", type => "rms", flag => "-bin_stdev", dtype => "float" }, + { name => "fringe_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "fringe_1", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "fringe_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "user_1", type => "mean", flag => "-user_1", dtype => "float" }, # fringe residual + { name => "user_2", type => "rms", flag => "-user_2", dtype => "float" }, # fringe residual + { name => "user_1", type => "stdev", flag => "-user_3", dtype => "float" }, # fringe residual + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of imfile files +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + # dettool command to select imfile data for this exp_id + $command = "$dettool -residimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -residimfile: $error_code\n"); + exit($error_code); + } + # XXX report an error message if stdout_buf is empty + + # Parse the stdout buffer into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # parse the file info in the metadata + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR); +} + +# Output products +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detresid.$det_id.$iter", $workdir, ${$files}[0]->{path_base} ); +my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1 +my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2 +my $logName = $ipprc->filename("LOG.EXP", $outputRoot); # Name for log + +my $logFile; +unless ($no_op) { + $logFile = $ipprc->file_create_append( $logName ); +} + +# XXX in debug mode, unlink = 0 +my ($list1File, $list1Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => 1 ); +my ($list2File, $list2Name) = tempfile( "$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => 1 ); +foreach my $file (@$files) { + print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n"); + print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n"); +} +close $list1File; +close $list2File; + + +# build the JPEG images +unless ($no_op) { + # Make the jpeg for binning 1 + $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name); + + # Make the jpeg for binning 2 + $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code); + } + &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name); +} + +#### measure stats and reject if needed #### + +# Rejection thresholds & related data +my $expected = rejection_limit( 'EXPECTED', $det_type, $filter ); +my $reject_imfile_mean = rejection_limit( 'IMFILE.MEAN', $det_type, $filter ); +my $reject_imfile_flux = rejection_limit( 'IMFILE.FLUX', $det_type, $filter ); +my $reject_imfile_stdev = rejection_limit( 'IMFILE.STDEV', $det_type, $filter ); +my $reject_imfile_skewness = rejection_limit( 'IMFILE.SKEWNESS', $det_type, $filter ); +my $reject_imfile_kurtosis = rejection_limit( 'IMFILE.KURTOSIS', $det_type, $filter ); +my $reject_imfile_meanstdev = rejection_limit( 'IMFILE.MEANSTDEV', $det_type, $filter ); +my $reject_imfile_snr = rejection_limit( 'IMFILE.SNR', $det_type, $filter ); +my $reject_imfile_bin_stdev = rejection_limit( 'IMFILE.BIN.STDEV', $det_type, $filter ); +my $reject_imfile_bin_snr = rejection_limit( 'IMFILE.BIN.SNR', $det_type, $filter ); +my $reject_exp_mean = rejection_limit( 'EXP.MEAN', $det_type, $filter ); +my $reject_exp_flux = rejection_limit( 'EXP.FLUX', $det_type, $filter ); +my $reject_exp_stdev = rejection_limit( 'EXP.STDEV', $det_type, $filter ); +my $reject_exp_meanstdev = rejection_limit( 'EXP.MEANSTDEV', $det_type, $filter ); +my $reject_exp_snr = rejection_limit( 'EXP.SNR', $det_type, $filter ); +my $reject_exp_bin_stdev = rejection_limit( 'EXP.BIN.STDEV', $det_type, $filter ); +my $reject_exp_bin_snr = rejection_limit( 'EXP.BIN.SNR', $det_type, $filter ); + +# storage array +my @fluxes; + +foreach my $file (@$files) { + my $name = $file->{class_id}; + my $mean = $file->{bg}; # Mean for this imfile + my $stdev = $file->{bg_stdev}; # Stdev for this imfile + my $skewness = $file->{bg_skewness}; # Skewness for this imfile + my $kurtosis = $file->{bg_kurtosis}; # Kurtosis for this imfile + my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this imfile + my $binStdev = $file->{bin_stdev}; # Binned Stdev for this imfile + + # calculate and save the fluxes + my $flux; + if ($file->{exp_time} == 0.0) { + $flux = $mean; + } else { + $flux = $mean / $file->{exp_time}; + } + push @fluxes, $flux; + + $mean -= $expected; + + last if $no_op; + + # reject exposure if, for any imfiles, the mean residual counts + # deviate from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual mean count value (eg, + # 0.0 for a bias). in general, use with ADDITIVE components + if ($reject_imfile_mean > 0) { + if (abs($mean) > $reject_imfile_mean * $stdev) { + print $logFile "Rejecting exposure based on bad imfile mean for $name: "; + $reject = 1; + } else { + print $logFile "Imfile mean for $name meets requirements: "; + } + print $logFile "$mean vs $reject_imfile_mean * $stdev\n"; + } else { + print $logFile "No rejection on imfile mean for $name\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_imfile_flux > 0) { + if (abs($flux) > $reject_imfile_flux) { + print $logFile "Rejecting exposure based on bad imfile flux for $name: "; + $reject = 1; + } else { + print $logFile "Imfile flux for $name meets requirements: "; + } + print $logFile "$flux vs $reject_imfile_flux\n"; + } else { + print $logFile "No rejection on imfile flux for $name\n"; + } + + # reject exposure if, for any imfiles, the total pixel variance is + # larger than the limit + if ($reject_imfile_stdev > 0) { + if ($stdev > $reject_imfile_stdev) { + print $logFile "Rejecting exposure based on bad imfile stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile stdev for $name meets requirements: "; + } + print $logFile "$stdev vs $reject_imfile_stdev\n"; + + } else { + print $logFile "No rejection on imfile stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the skewness is + # larger than the limit + if ($reject_imfile_skewness > 0) { + if ($stdev > $reject_imfile_skewness) { + print $logFile "Rejecting exposure based on bad imfile skewness for $name: "; + $reject = 1; + } else { + print $logFile "Imfile skewness for $name meets requirements: "; + } + print $logFile "$skewness vs $reject_imfile_skewness\n"; + + } else { + print $logFile "No rejection on imfile skewness for $name\n"; + } + + # reject exposure if, for any imfiles, the kurtosis is + # larger than the limit + if ($reject_imfile_kurtosis > 0) { + if ($stdev > $reject_imfile_kurtosis) { + print $logFile "Rejecting exposure based on bad imfile kurtosis for $name: "; + $reject = 1; + } else { + print $logFile "Imfile kurtosis for $name meets requirements: "; + } + print $logFile "$kurtosis vs $reject_imfile_kurtosis\n"; + + } else { + print $logFile "No rejection on imfile kurtosis for $name\n"; + } + + # reject exposure if, for any imfiles, the variance of the imfile + # component means is larger than the limit + if ($reject_imfile_meanstdev > 0) { + if ($meanStdev > $reject_imfile_meanstdev) { + print $logFile "Rejecting exposure based on bad imfile mean stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile mean stdev for $name meets requirements: "; + } + print $logFile "$meanStdev vs $reject_imfile_meanstdev\n"; + } else { + print $logFile "No rejection on imfile mean stdev for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise (ie, + # the mean counts / total pixel variance) of the imfile component + # means are less than the limit. this test is sensible for images + # which have finite residual flux such as a flat-field image. + if ($reject_imfile_snr > 0) { + if ($mean < $stdev * $reject_imfile_snr) { + print $logFile "Rejecting exposure based on bad imfile S/N for $name: "; + $reject = 1; + } else { + print $logFile "Imfile S/N for $name meets requirements: "; + } + print $logFile "mean: $mean vs stdev*SNlimit: " . $stdev * $reject_imfile_snr . "\n"; + } else { + print $logFile "No rejection on imfile S/N for $name\n"; + } + + # reject exposure if, for any imfiles, the signal-to-noise of the + # imfile component means, based on the stdev of the binned image + # is less than the limit. this test is sensible for images which + # have finite residual flux such as a flat-field image. + if ($reject_imfile_bin_stdev > 0) { + if ($binStdev > $reject_imfile_bin_stdev) { + print $logFile "Rejecting exposure based on bad imfile binned stdev for $name: "; + $reject = 1; + } else { + print $logFile "Imfile binned stdev for $name meets requirements: "; + } + print $logFile "$binStdev vs $reject_imfile_bin_stdev\n"; + } else { + print $logFile "No rejection on imfile binned stdev for $name\n"; + } + if ($reject_imfile_bin_snr > 0) { + if ($mean < $binStdev * $reject_imfile_bin_snr) { + print $logFile "Rejecting exposure based on bad imfile binned S/N for $name: "; + $reject = 1; + } else { + print $logFile "Imfile binned S/N for $name meets requirements: "; + } + print $logFile "mean: $mean vs binStdev*SNlimit: " . $binStdev * $reject_imfile_bin_snr . "\n"; + } else { + print $logFile "No rejection on imfile binned S/N for $name\n"; + } +} + +# basic ensemble stats +my $mean = $stats->value_for_flag ("-bg"); +my $meanStdev = $stats->value_for_flag ("-bg_mean_stdev"); +my $stdev = $stats->value_for_flag ("-bg_stdev"); +my $binStdev = $stats->value_for_flag ("-bin_stdev"); +my $fringe_mean = $stats->value_for_flag ("-fringe_0"); +my $fringe_err = $stats->value_for_flag ("-fringe_1"); +my $fringe_mean_stdev = $stats->value_for_flag ("-fringe_2"); +my $dfringe_mean = $stats->value_for_flag ("-fringe_resid_0"); +my $dfringe_err = $stats->value_for_flag ("-fringe_resid_1"); +my $dfringe_mean_stdev = $stats->value_for_flag ("-fringe_resid_2"); + +# other stats (flux depends on bg and exp_time) +my $fluxStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means +$fluxStats->add_data(@fluxes); +my $flux = $fluxStats->mean(); # Mean of the imfile means + +# other stats +my $exp_snr = 0.0; +if ($stdev > 0) { $exp_snr = $mean / $stdev; } + +## Reject based on the exposure ensemble stats +# reject if the exposure ensemble mean is deviant +unless ($no_op) { + print $logFile "Exposure mean $mean, stdev $stdev, mean stdev $meanStdev, exp s/n: $exp_snr\n"; + + # reject exposure if the ensemble mean residual counts deviate + # from the expected value by more than N sigma, (sigma = total + # pixel variance). this test is sensible for images which should + # have a predictable nominal residual mean count value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_mean > 0) { + if (abs($mean) > $reject_exp_mean * $stdev) { + print $logFile "Rejecting exposure based on bad mean counts: "; + $reject = 1; + } else { + print $logFile "Exposure mean counts meets requirements: "; + } + print $logFile "mean: $mean, stdev: $stdev (limit is: " . $reject_exp_mean * $stdev . ") \n"; + } else { + print $logFile "No rejection for exp mean\n"; + } + + # reject exposure if, for any imfiles, the mean residual flux + # deviates from the expected value by more than N sigma, (sigma = + # total pixel variance). this test is sensible for images which + # should have a predictable nominal residual flux value (eg, 0.0 + # for a bias). in general, use with ADDITIVE components + if ($reject_exp_flux > 0) { + if (abs($flux) > $reject_exp_flux * $stdev) { + print $logFile "Rejecting exposure based on bad mean flux: "; + $reject = 1; + } else { + print $logFile "Exposure mean flux meets requirements: "; + } + print $logFile "flux: $flux, stdev: $stdev (limit is: " . $reject_exp_flux * $stdev . ")\n"; + } else { + print $logFile "No rejection for exp mean\n"; + } + + # reject exposure if the total pixel variance is larger than the + # limit + if ($reject_exp_stdev > 0) { + if ($stdev > $reject_exp_stdev) { + print $logFile "Rejecting exposure based on bad stdev: "; + $reject = 1; + } else { + print $logFile "Exposure stdev meets requirements: "; + } + print $logFile "$stdev vs $reject_exp_stdev\n"; + } else { + print $logFile "No rejection for exp stdev\n"; + } + + # reject exposure if the variance of the imfile means is larger + # than the limit + if ($reject_exp_meanstdev > 0) { + if ($meanStdev > $reject_exp_meanstdev) { + print $logFile "Rejecting exposure based on bad mean stdev: "; + $reject = 1; + } else { + print $logFile "Exposure mean stdev meets requirements: "; + } + print $logFile "$meanStdev vs $reject_exp_meanstdev\n"; + } else { + print $logFile "No rejection for exp mean stdev\n"; + } + + # reject if the signal-to-noise is insufficient + if ($reject_exp_snr > 0) { + if (abs($mean) < abs($stdev * $reject_exp_snr)) { + print $logFile "Rejecting exposure based on poor S/N: \n"; + $reject = 1; + } else { + print $logFile "Exposure S/N meets requirements: \n"; + } + print $logFile "signal: $mean vs noise: $stdev (s/n limit is: $reject_exp_snr)\n"; + } else { + print $logFile "No rejection for exp S/N\n"; + } + # reject if the exposure ensemble stdev is deviant + if ($reject_exp_bin_stdev > 0) { + if ($binStdev > $reject_exp_bin_stdev) { + print $logFile "Rejecting exposure based on bad binned stdev: "; + $reject = 1; + } else { + print $logFile "Exposure binned stdev meets requirements: "; + } + print $logFile "$stdev vs $reject_exp_bin_stdev\n"; + } else { + print $logFile "No rejection for exp binned stdev\n"; + } + # reject if the signal-to-noise is insufficient + if ($reject_exp_bin_snr > 0) { + if (abs($mean) < abs($binStdev * $reject_exp_bin_snr)) { + print $logFile "Rejecting exposure based on poor binned S/N: \n"; + $reject = 1; + } else { + print $logFile "Exposure binned S/N meets requirements: \n"; + } + print $logFile "signal: $mean vs noise: $binStdev (s/n limit is: $reject_exp_bin_snr)\n"; + } else { + print $logFile "No rejection for exp binned S/N\n"; + } + + close $logFile; +} + +$command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_id $exp_id"; +$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot "; +$command .= ' -reject' if $reject; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +unless ($no_update) { + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidexp: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and $exp_id and not $no_update) { + my $command = "$dettool -addresidexp"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +# Retrieve the requested rejection limit, dying if not extant +sub rejection_limit +{ + my $name = shift; # Rejection limit to + my $type = shift; # Type of exposure + my $filter = shift; # Filter + + my $value = $ipprc->rejection( $name, $det_type, $filter ); + if (not defined $value) { + $filter = "(no filter)" if not defined $filter; + die "Unable to determine $name rejection limit for $det_type with $filter.\n"; + } + + return $value; +} + +__END__ + +#### + +## this function is not well though out. it loads a collection of +## values (background mean and stdev) for a collection of imfile +## subcomponents. it then measures the mean background, the +## background stdev (rms of ensemble stdevs), and the background mean +## stdev (rms of the means). it then tries to reject based on the +## following criteria: +## (background / stdev > limit) for a single component +## ---> this will ACCEPT for a poor images (large stdev) +## (stdev > limit) +## (mean background / stdev > limit) same problem +## (mean stdev > limit) + +### I would suggest the following: +## 1) calculate the ensemble mean and stdev +## 2) for each component, is the (back - mean) / mean stdev > limit? +## (ie, is it an outlier?) +## 3) is the component stdev > limit? (absolute test) +## 4) calculate the background stdev for the ensemble (excluding rejects) +## 5) is the mean stdev > limit (poor images) +## 6) calculate the ensemble stdev (rms) of remaining (excluding rejects) +## 7) is the ensemble stdev > limit + +## for flats, the value used for stdev should be the fractional stdev +## (stdev/mean) and no rejection should be performed on the basis of +## the mean value, but could still be done on the mean stdev + +## the same logical cuts above can be applied to components in an +## imfile, imfiles in an exposure, and exposures in a stack + +### this function needs to avoid div by zero: compare Signal vs Noise*limit Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_resid.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_resid.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_resid.pl (revision 16175) @@ -0,0 +1,254 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $exp_id, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $dbname, $workdir, $reduction, $no_update, $no_op); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|=s' => \$exp_tag, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'detrend=s' => \$detrend, + 'input_uri|u=s' => \$input_uri, + 'camera|c=s' => \$camera, + 'mode|m=s' => \$mode, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction=s' => \$reduction, # Reduction class + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --class_id --det_type --camera --input_uri --mode --detrend (not for 'verify' mode)", + -exitval => 3) + unless defined $det_id + and defined $iter + and defined $exp_id + and defined $exp_tag + and defined $class_id + and defined $det_type + and defined $input_uri + and defined $camera + and defined $mode + and (defined $detrend or lc($mode) eq 'verify'); + +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type and mode +$reduction = 'DETREND' unless defined $reduction; +my $recipe; # Name of recipe to use +if ($mode eq 'master') { + $recipe = uc($det_type) . '_RESID'; +} elsif ($mode eq 'verify') { + $recipe = uc($det_type) . '_VERIFY'; +} else { + &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} +$recipe = $ipprc->reduction($reduction, $recipe); + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "SAMPLE_SKEWNESS", type => "mean", flag => "-bg_skewness", dtype => "float" }, + { name => "SAMPLE_KURTOSIS", type => "mean", flag => "-bg_kurtosis", dtype => "float" }, + { name => "FRINGE_0", type => "mean", flag => "-fringe_0", dtype => "float" }, + { name => "FRINGE_ERR_0", type => "rms", flag => "-fringe_1", dtype => "float" }, + { name => "FRINGE_0", type => "stdev", flag => "-fringe_2", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "mean", flag => "-fringe_resid_0", dtype => "float" }, + { name => "FRINGE_RESID_ERR_0", type => "rms", flag => "-fringe_resid_1", dtype => "float" }, + { name => "FRINGE_RESID_0", type => "stdev", flag => "-fringe_resid_2", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +my $BINNED_STATS = + [ + { name => "ROBUST_STDEV", type => "rms", flag => "-bin_stdev" }, + ]; +my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser + +# Flags to specify the particular detrend to use +use constant DETRENDS => { + 'bias' => '-bias', # Specify the bias frame + 'dark' => '-dark', # Specify the dark frame + 'shutter' => '-shutter', # Specify the shutter frame + 'flat' => '-flat', # Specify the flat frame + 'domeflat' => '-flat', # Specify the flat frame + 'skyflat' => '-flat', # Specify the flat frame + 'fringe' => '-fringe', # Specify the fringe frame + 'mask' => '-mask', # Specify the mask frame + 'darkmask' => '-mask', # Specify the mask frame + 'flatmask' => '-mask', # Specify the mask frame +}; + +use constant DELETE_STATS => 0; # Delete the statistics file when done? + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppStats = can_run('ppStats') or (warn "Can't find ppStats" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} +$ppImage .= " -dbname $dbname" if defined $dbname; + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detresid.$det_id.$iter", $workdir, $input_uri ); + +my $outputName = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id); +my $bin1Name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id); +my $bin2Name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id); +my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outputRoot, $class_id); + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".$class_id.log"; # Log messages + +# Run ppImage & ppStats +unless ($no_op) { + my $command = "$ppImage -file $input_uri $outputRoot"; + $command .= " -recipe PPIMAGE $recipe"; + $command .= " -recipe PPSTATS RESIDUAL"; + $command .= " -stats $outputStats"; + $command .= " -tracedest $traceDest -log $logDest"; + + # Detrend to use in processing + if (lc($mode) ne 'verify') { + my $detFlag = DETRENDS->{lc($det_type)}; + &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR) unless defined $detFlag; + $command .= " $detFlag $detrend"; + } + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName); + &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name); + &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name); + + # Load the raw output stats file + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + # Parse the stats file contents into a metadata + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # Parse the statistics on the residual image + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # run ppStats on the binned image + $command = "$ppStats -recipe PPSTATS RESIDUAL $bin2Name"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code); + } + + # Parse the output contents into a metadata + my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); + + # parse the binned image statistics + $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR); +} + +# Command to update the database +my $command = "$dettool -addresidimfile"; +$command .= " -det_id $det_id"; +$command .= " -iteration $iter"; +$command .= " -exp_id $exp_id"; +$command .= " -class_id $class_id"; +$command .= " -recip $recipe"; +$command .= " -uri $outputName"; +$command .= " -path_base $outputRoot"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); +$command .= $binnedStats->cmdflags(); + +# Add the processed file to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addresidimfile: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $exp_id = shift; # Exposure tag + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and $exp_id and not $no_update) { + my $command = "$dettool -addresidimfile"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -exp_id $exp_id"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/detrend_stack.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/detrend_stack.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/detrend_stack.pl (revision 16175) @@ -0,0 +1,208 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($det_id, $iter, $class_id, $det_type, $camera, $dbname, $workdir, $reduction, $no_update, $no_op, $quiet); +GetOptions( + 'det_id|d=s' => \$det_id, + 'iteration=s' => \$iter, + 'class_id|i=s' => \$class_id, + 'det_type|t=s' => \$det_type, + 'camera|c=s' => \$camera, + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'reduction=s' => \$reduction, # Reduction class for processing + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'quiet' => \$quiet, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera", + -exitval => 3) + unless defined $det_id + and defined $iter + and defined $class_id + and defined $det_type + and defined $camera; + +my $verbose = 1; +if (defined $quiet) { $verbose = 0; } + +$ipprc->define_camera($camera); + +# Recipes to use as a function of detrend type +$reduction = "DETREND" unless defined $reduction; +my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_STACK'); # Recipe name to use + +# values to extract from output metadata and the stats to calculate +# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A) +# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0 (B) +# XXX (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1 +my $STATS = + [ + # KEYWORD STATISTIC CHIPTOOL FLAG + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, + # { name => "bg_mean_stdev", type => "rms", flag => "-bg_mean_stdev" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppMerge = can_run('ppMerge') or (warn "Can't find ppMerge" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of files to stack +my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf); +{ + $command = "$dettool -processedimfile -included"; + $command .= " -det_id $det_id"; + $command .= " -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $iter, $class_id, $error_code); + } + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR); +} + +$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir; + +my $outputRoot = $ipprc->file_prepare( "$camera.$det_type.$det_id.$iter.$class_id", $workdir, ${$files}[0]->{uri} ); +my $outputStack = $outputRoot . '.fits'; # Output name +my $outputStats = $outputRoot . '.stats'; # Statistics name + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".log"; # Log messages + +$command = "$ppMerge $outputStack"; # Command to run +foreach my $file (@$files) { + $command .= ' ' . $file->{uri}; +} +$command .= " -recipe PPMERGE $recipe"; +$command .= ' -type ' . uc($det_type); # Type of stacking to perform +$command .= " -stats $outputStats"; # Statistics output filename +$command .= " -recipe PPSTATS CHIPSTATS"; +$command .= " -tracedest $traceDest -log $logDest"; + +# Stack the files +unless ($no_op) { + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppMerge: $error_code", $det_id, $iter, $class_id, $error_code); + } + &my_die("Unable to find expected output file: $outputStack\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStack); + &my_die("Unable to find expected output file: $outputStats\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats); + + # Get the statistics on the stacked image + open(my $statsFile, $ipprc->file_resolve("$outputStats")) or + &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + my $contents = do { local $/; <$statsFile> }; # Contents of file + close($statsFile); + + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse($contents) or + &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); + + $stats->parse($metadata) or + &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR); +} + +# Command to update the database +$command = "$dettool -addstacked"; +$command .= " -det_id $det_id -iteration $iter"; +$command .= " -class_id $class_id"; +$command .= " -uri $outputStack"; +$command .= " -recip $recipe"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +# Add the resultant into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform dettool -addstacked: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($det_id and $iter and $class_id and not $no_update) { + my $command = "$dettool -addstacked"; + $command .= " -det_id $det_id"; + $command .= " -iteration $iter"; + $command .= " -class_id $class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system ($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/diff_skycell.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/diff_skycell.pl (revision 16175) @@ -0,0 +1,267 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($diff_id, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'diff_id|d=s' => \$diff_id, # Diff identifier + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --diff_id", + -exitval => 3, + ) unless defined $diff_id; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC DIFFTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $ppSub = can_run('ppSub') or (warn "Can't find ppSub" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for subtraction +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$difftool -inputskyfile -diff_id $diff_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -inputskyfile: $error_code", $diff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $diff_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Subtraction list contains more than two elements", $diff_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files == 2; + +# Identify the input and the template +my ($input, $inputMask, $inputWeight, $inputPath, $inputPSF); # Input files and path +my ($template, $templateMask, $templateWeight, $templatePath, $templateSources); # Template files and path +my $tess_id; # Tesselation identifier +my $skycell_id; # Skycell identifier +my $camera; # Camera +foreach my $file (@$files) { + if (defined $file->{template} and $file->{template}) { + $template = $file->{uri}; + $templatePath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $templateMask = "PPSTACK.OUTPUT.MASK"; + $templateWeight = "PPSTACK.OUTPUT.WEIGHT"; + $templateSources = "PSPHOT.OUTPUT"; + } else { + $templateMask = "PSWARP.OUTPUT.MASK"; + $templateWeight = "PSWARP.OUTPUT.WEIGHT"; + $templateSources = "PSWARP.OUTPUT.SOURCES"; + } + } else { + $input = $file->{uri}; + $inputPath = $file->{path_base}; + if ($file->{warp_id} == 0) { + $inputMask = "PPSTACK.OUTPUT.MASK"; + $inputWeight = "PPSTACK.OUTPUT.WEIGHT"; + $inputPSF = "PSPHOT.PSF.SAVE"; + } else { + $inputMask = "PSWARP.OUTPUT.MASK"; + $inputWeight = "PSWARP.OUTPUT.WEIGHT"; + $inputPSF = "PSPHOT.PSF.SAVE"; + } + } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $diff_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } + +} + +&my_die("Unable to identify template", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $template; +&my_die("Unable to identify input", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $input; +&my_die("Unable to identify camera", $diff_id, $PS_EXIT_SYS_ERROR) unless defined $camera; +$ipprc->define_camera($camera); + +$templateMask = $ipprc->filename($templateMask, $templatePath); +$inputMask = $ipprc->filename($inputMask, $inputPath); +$templateWeight = $ipprc->filename($templateWeight, $templatePath); +$inputWeight = $ipprc->filename($inputWeight, $inputPath); +$inputPSF = $ipprc->filename($inputPSF, $inputPath); +$templateSources = $ipprc->filename($templateSources, $templatePath); + +&my_die("Couldn't find input: $template", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($template); +&my_die("Couldn't find input: $templateMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateMask); +&my_die("Couldn't find input: $templateWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateWeight); +&my_die("Couldn't find input: $input", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputWeight); +&my_die("Couldn't find input: $inputPSF", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputPSF); +&my_die("Couldn't find input: $templateSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateSources); + +# Get the output filenames +$workdir = caturi( $workdir, "tess_" . $tess_id, "sky_" . $skycell_id ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$tess_id.$skycell_id.dif$diff_id", $workdir, $input ); +my $outputName = $ipprc->filename("PPSUB.OUTPUT", $outputRoot); +my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outputRoot); +my $outputWeight = $ipprc->filename("PPSUB.OUTPUT.WEIGHT", $outputRoot); +my $outputSources = $ipprc->filename("PSPHOT.OUTPUT", $outputRoot); +#my $bin1Name = $ipprc->filename("PPSUB.BIN1", $outputRoot); +#my $bin2Name = $ipprc->filename("PPSUB.BIN2", $outputRoot); +my $outputStats = $outputRoot . '.stats'; + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".log"; # Log messages + +# Perform subtraction +unless ($no_op) { + my $command = "$ppSub $input $template $outputRoot"; + $command .= " -inmask $inputMask"; + $command .= " -refmask $templateMask"; + $command .= " -inweight $inputWeight"; + $command .= " -refweight $templateWeight"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -sources $templateSources"; + $command .= " -psf $inputPSF"; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppSub: $error_code", $diff_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $diff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the residual image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $diff_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $diff_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $diff_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the subtraction result + { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -uri $outputName -path_base $outputRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool -adddiffskyfile: $error_code", $diff_id, $error_code); + } + } + + # Register the run as completed + { + my $command = "$difftool -updaterun -diff_id $diff_id -state stop"; # Command to run difftool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform difftool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $diff_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($diff_id and not $no_update) { + my $command = "$difftool -adddiffskyfile -diff_id $diff_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => 1); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ds9_cmf_regions.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ds9_cmf_regions.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ds9_cmf_regions.pl (revision 16175) @@ -0,0 +1,127 @@ +#!/usr/bin/env perl + +use strict; +use Astro::FITS::CFITSIO qw( :constants ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; + +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +my $xpaset = `which xpaset`; +my $xpaget = `which xpaget`; + +die "Unable to find xpaget and xpaset.\n" unless ($xpaset =~ /\S+/ and $xpaget =~ /\S+/); + + +my ( $filename, # Filename containing photometry + $extname, # Extension name containing photometry + $frame, # Frame number in ds9 + $colour, # Region colour + $radius, # Radius for circle + ); + +# Defaults +$colour = "red"; +$radius = 5; + +GetOptions( + 'file=s' => \$filename, + 'ext=s' => \$extname, + 'frame=s' => \$frame, + 'colour=s' => \$colour, + 'radius=f' => \$radius, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --file --ext", + -exitval => 3) + unless defined $filename + and defined $extname; + +my $status; # Status of FITSIO calls +my $fits = Astro::FITS::CFITSIO::open_file( $filename, READONLY, $status ); # FITS file handle +check_fitsio($status); +$fits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status); +my $numRows; # Number of rows in table +$fits->get_num_rows($numRows, $status) and check_fitsio($status); + +my ($xCol, $yCol); # Column numbers for x and y +$fits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); + +my ($xType, $yType); # Types for x and y +$fits->get_coltype($xCol, $xType, undef, undef, $status) and check_fitsio($status); +$fits->get_coltype($yCol, $yType, undef, undef, $status) and check_fitsio($status); + +my ($x, $y); # Coordinates read from table +$fits->read_col($xType, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); +$fits->read_col($yType, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); +$fits->close_file($status); + +my ($coordFile, $coordName) = tempfile( "/tmp/ds9_cmf_regions.XXXX", UNLINK => 1 ); +for (my $i = 0; $i < $numRows; $i++) { + print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . + ",$radius \# color = $colour\n"; +} +close $coordFile; + +my @settings = settings_save("regions format", + "regions system"); # Settings to save + +xpaset("frame $frame") if defined $frame; +xpaset("regions format ds9"); +xpaset("regions system image"); +xpaset("regions load $coordName"); + +xpaset(@settings); + +print "Plotted $numRows sources.\n"; + +### Pau. + + + + +# From Astro::FITS::CFITSIO demo +sub check_fitsio +{ + my $status = shift; # Status of FITSIO calls + + if ($status != 0) { + my $msg; # Message to output + Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg ); + die "CFITSIO error: $msg\n"; + } +} + +# Save specified settings +sub settings_save +{ + my @settings; # Values of settings + foreach my $setting (@_) { + my @values = xpaget($setting); + push @settings, $setting . ' ' . shift @values; + } + return @settings; +} + + +# XPA subroutines courtesy Derek Fox +sub xpaset { + foreach my $cmd (@_) { + system("xpaset -p ds9 $cmd"); + } +} +sub xpaget { + my @out; + foreach my $cmd (@_) { + my $output = `xpaget ds9 $cmd`; + push @out, $output; + } + return @out; +} + + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/flatcorr_init.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/flatcorr_init.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/flatcorr_init.pl (revision 16175) @@ -0,0 +1,221 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_init.pl +## given dbname, dvodb, filter, time range, camera, telescope, etc? +## select matching images and register as a new flatcorr run + +## should this be implemented as a ippTool program? +## this is a lot like dettool -definebyquery +## flatcorr -definebyquery +## this should queue the selected images with a mode that stops at camera +## ?? how do we block this batch from running through warp ?? + +## we also need the test / query tool: +## flatcorr -pendingcorr +## examine the chip & camera stages and check for all images to have completed (successfully or not) + +## allow a certain fraction of failures... + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dvo_id, $catdir, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'dvo_id|i=s' => \$dvo_id, + 'catdir|c=s' => \$catdir, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dvo_id --catdir --region", + -exitval => 3) unless + defined $dvo_id and + defined $catdir and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addcalrun"; +$command .= " -dvo_id $dvo_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -status SUCCESS"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if ($dvo_id && $region && $last_step && $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/flatcorr_proc.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/flatcorr_proc.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/flatcorr_proc.pl (revision 16175) @@ -0,0 +1,221 @@ +#!/usr/bin/env perl + +## USAGE:flatcorr_proc.pl --dbname --corr_id + +## this script does the following steps: + +# extract the details of the flatcorr run: dvodb, filter, camera, etc? + +# relphot -D CATDIR $dvodb -grid (outgrid.fits) (filter) -region 0 360 -90 90 (other parameters?) + +# detselect flat? + +# dvoMakeCorr -file outgrid.fits -ref ref.fits outcorr + +# dettool -register -det_type FLATCORR -filelevel (level) -workdir -inst, etc +# foreach $imfile () +# dettool -register_imfile -uri, etc, etc + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dvo_id, $catdir, $region, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'dvo_id|i=s' => \$dvo_id, + 'catdir|c=s' => \$catdir, + 'region|r=s' => \$region, + 'dbname|d=s' => \$dbname,# Database name + 'workdir|w=s' => \$workdir, # Working directory for output files + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dvo_id --catdir --region", + -exitval => 3) unless + defined $dvo_id and + defined $catdir and + defined $region; + +# Look for programs we need +my $missing_tools; +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $uniphot = can_run('uniphot') or (warn "Can't find uniphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $caltool = can_run('caltool') or (warn "Can't find caltool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# select the primary filters from DVO query? +my (@filters) = `photcodeList -average`; + +# parse the region (RAs,RAe:DECs,DECe) : item = +/-NNN.NNNN +my @coords = split (":", $region); +my ($RAs, $RAe) = split (",", $coords[0]); +my ($DECs, $DECe) = split (",", $coords[1]); + +# Run addstar -resort +{ + my $command = "$addstar -resort"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RESORT", $status, $dbname); + } +} + +# Run relphot (filter) for each filter +{ + foreach my $filter (@filters) { + my $command = "$relphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELPHOT", $status, $dbname); + } + } +} + +# Run uniphot (filter) for each filter +# XXX skip this one? run less frequently? +if (0) { + foreach my $filter (@filters) { + my $command = "$uniphot $filter"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "UNIPHOT", $status, $dbname); + } + } +} + +{ + my $command = "$relastro -objects"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.OBJECTS", $status, $dbname); + } +} + +{ + my $command = "$relastro -images"; + $command .= "-D CATDIR $catdir"; + $command .= "-region $RAs $RAe $DECs $DECe"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform addstar -resort on region $region: $error_code", $dvo_id, $region, "RELASTRO.IMAGES", $status, $dbname); + } +} + +my $command = "$caltool -addcalrun"; +$command .= " -dvo_id $dvo_id"; +$command .= " -region $region"; +$command .= " -last_step RELASTRO.IMAGES"; +$command .= " -status SUCCESS"; +$command .= " -dbname $dbname" if defined $dbname; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $dvo_id = shift; + my $region = shift; + my $last_step = shift; + my $status = shift; + my $dbname = shift; + + carp($msg); + if ($dvo_id && $region && $last_step && $status and not $no_update) { + my $command = "$caltool -addcalrun"; + $command .= " -dvo_id $dvo_id"; + $command .= " -region $region"; + $command .= " -last_step $last_step"; + $command .= " -status $status"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/gpc_seeing.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/gpc_seeing.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/gpc_seeing.pl (revision 16175) @@ -0,0 +1,28 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: gpc_seeing.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OP'; +$RECIPE_PSPHOT = 'PSPHOT.SEEING'; + +# recommend only processing to PSFMODEL +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +# XXX otis can read the output psf model, or we can supply a program to interpret the model + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_darkstats.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_darkstats.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_darkstats.pl (revision 16175) @@ -0,0 +1,198 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; + +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); # tools to parse the IPP configuration information + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_darkstats.pl --dbname (name) --det_id (id) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_datapath.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_datapath.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_datapath.pl (revision 16175) @@ -0,0 +1,15 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +die "No filename specified.\n" if scalar @ARGV != 1; + +print $ipprc->file_resolve(shift @ARGV) . "\n"; + +1; + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_detrend_combine.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_detrend_combine.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_detrend_combine.pl (revision 16175) @@ -0,0 +1,226 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Data::Dumper; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use PS::IPP::Metadata::Stats; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats + +# Parse command-line arguments +my ($det_type, $filelevel, $inst, $telescope, $filter, + $det_id1, $iter1, $det_id2, $iter2, $operation, $mask, + $workdir, $dbname, $no_update, $no_op); +GetOptions( + 'det_type=s' => \$det_type, # Detrend type for new detrend + 'filelevel=s' => \$filelevel, # File level for new detrend + 'inst=s' => \$inst, # Instrument for new detrend + 'telescope=s' => \$telescope, # Telescope for new detrend + 'filter=s' => \$filter, # Filter name for new detrend + 'det_id1=s' => \$det_id1, # Detrend id for detrend 1 + 'iteration1=s' => \$iter1, # Iteration for detrend 1 + 'det_id2=s' => \$det_id2, # Detrend id for detrend 2 + 'iteration2=s' => \$iter2, # Iteration for detrend 2 + 'operation=s' => \$operation, # Operation to perform on files + 'mask' => \$mask, # Operation is on a mask + 'workdir=s' => \$workdir, # Working directory for output files + 'dbname=s' => \$dbname, # Database name + 'no-update' => \$no_update, # Don't update the database + 'no-op' => \$no_op, # Don't do any operations + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options --det_type --filelevel --inst --telescope --det_id1 --iteration1 --det_id2 --iteration2 --workdir", + -exitval => 3, + ) + unless defined $det_type + and defined $filelevel + and defined $inst + and defined $telescope + and defined $det_id1 + and defined $iter1 + and defined $det_id2 + and defined $iter2 + and defined $operation + and defined $workdir; + +$ipprc->define_camera($inst); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; + +# Look for programs we need +my $missing_tools; +my $detselect = can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $ppArith = can_run('ppArith') or (warn "Can't find ppArith" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get the list of inputs +my $files1 = filelist($det_id1, $iter1); # Hash of input files for detrend 1 +my $files2 = filelist($det_id2, $iter2); # Hash of input files for detrend 2 +die("File lists for detrends have differing lengths") unless scalar keys %$files1 == scalar keys %$files2; + +my ($det_id, $iter); # Detrend identifier for the new detrend +{ + my $command = "$dettool -register_detrend -det_type $det_type -filelevel $filelevel -workdir $workdir " . + "-inst $inst -telescope $telescope"; # Command to run + $command .= " -filter $filter" if defined $filter; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend: $error_code"); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or die("Unable to parse metadata config doc\n"); + my $md = parse_md_list($metadata) or die("Unable to parse metadata list\n"); + + $det_id = $$md[0]->{det_id}; + $iter = $$md[0]->{iteration}; + + die("Unable to get det_id and iteration for new detrend.\n") unless defined $det_id and defined $iter; +} + +my $outRoot = caturi($workdir, "$inst.$det_id.$iter"); # Output root name +my $filerule = (defined $mask ? "PPARITH.OUTPUT.MASK" : "PPARITH.OUTPUT.IMAGE"); # File rule for ppArith + +foreach my $class_id ( keys %$files1 ) { + my $md1 = $$files1{$class_id}; + my $md2 = $$files2{$class_id}; + die("Class_id=$class_id not defined for det_id=$det_id2") unless defined $md2; + + my $uri1 = $$md1[0]->{uri}; + my $uri2 = $$md2[0]->{uri}; + + die("Unable to find input file $uri1\n") unless $ipprc->file_exists($uri1); + die("Unable to find input file $uri2\n") unless $ipprc->file_exists($uri2); + + my $outName = $ipprc->filename($filerule, $outRoot, $class_id); + my $outStats = $outRoot . '.stats'; + + my $command = "$ppArith -file1 $uri1 -op \'$operation\' -file2 $uri2 $outRoot"; # Command to run + $command .= " -stats $outStats -recipe PPSTATS $RECIPE_PPSTATS"; + $command .= ' -mask' if defined $mask; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run ppArith: $error_code"); + } + + die("Unable to find ppArith product: $outName\n") unless $ipprc->file_exists($outName); + die("Unable to find ppArith product: $outStats\n") unless $ipprc->file_exists($outStats); + + # Get the statistics on the processed image + my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + { + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outStats) or die("Can't open stats file $outStats: $!"); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + + my $metadata = $mdcParser->parse(join "", @contents) or die("Unable to parse metadata config doc"); + + unless ($stats->parse($metadata)) { + &my_die("Failure extracting metadata from the statistics output file.\n"); + } + } + + # Register the imfile + { + my $command = "$dettool -register_detrend_imfile -det_id $det_id "; # Command to run + $command .= " -class_id $class_id -uri $outName -path_base $outRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run dettool -register_detrend_imfile: $error_code"); + } + } +} + + +### Pau. + + +# Get a list of files for the given detrend +sub filelist +{ + my $det_id = shift; # Detrend identifier + my $iter = shift; # Iteration + + my $command = "$detselect -select -det_id $det_id -iteration $iter"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die("Unable to run detselect: $error_code"); + } + + # Because of the length, need to split into individual metadatas --- it parses SO much quicker! + my %files; + + my $md = $mdcParser->parse( join( "", @$stdout_buf ) ); # Parsed metadata + my $list = parse_md_list( $md ); + + foreach my $item ( @$list ) { + my $class_id = $item->{class_id}; + die("Multiple definitions of class_id=$class_id found for det_id=$det_id, iteration=$iter\n") if + defined $files{$class_id}; + $files{$class_id} = parse_md_list($md); + } + + return \%files; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_filename.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_filename.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_filename.pl (revision 16175) @@ -0,0 +1,48 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# use these to check the apache environment +#print "$ENV{'PATH'}\n"; +#print "$ENV{'PERL5LIB'}\n"; + +use PS::IPP::Config; +my $ipprc = PS::IPP::Config->new(); + +my ($filerule, $class_id, $basename, $camera); + +GetOptions('filerule=s' => \$filerule, + 'class_id=s' => \$class_id, + 'basename=s' => \$basename, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --filerule --class_id --basename --camera", + -exitval => 3, +) unless defined $basename + and defined $filerule + and defined $class_id + and defined $camera; + +$ipprc->define_camera($camera); + +#print "$filerule\n"; +#print "$basename\n"; +#print "$camera\n"; +#print "$class_id\n"; + +my $filename = $ipprc->filename($filerule, $basename, $class_id); +#print "$filename\n"; + +my $realname = $ipprc->file_resolve( $filename ); +print "$realname\n"; + +1; + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_maskscript.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_maskscript.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_maskscript.pl (revision 16175) @@ -0,0 +1,199 @@ +#!/usr/bin/env perl + +# use warnings; +# use strict; +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use IO::Handle; + +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); # tools to parse the IPP configuration information + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, $det_id, $camera); + +GetOptions('dbname=s' => \$dbname, + 'det_id=s' => \$det_id, + 'camera|c=s' => \$camera, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "USAGE: ipp_maskscript.pl --dbname (name) --det_id (id) --iter (iteration) --camera (name)", + -exitval => 3, + ) unless defined $dbname and defined $det_id and defined $camera; + +# I could determine the camera from a query for the detrun +$ipprc->define_camera($camera); + +### Get list of dark imfile results + +# define the dettool command +my $command = "dettool -processedimfile -select_state stop"; # Command to run +$command .= " -det_id $det_id"; +$command .= " -dbname $dbname" if defined $dbname; + +# run the dettool command and catch the output +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", $error_code); +} + +# parse the output into a list +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); +my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + +my @bg_data; +my @bg_stdev_data; +my @bg_name; +my @bg_exptime; +my %components; + +print STDERR "extracted the data from the database\n"; + +# we now have a list of imfiles; we need to extract the background for each cell +# from the stats files for each imfile +foreach my $item (@$list) { + my $path_base = $item->{path_base}; + my $class_id = $item->{class_id}; + my $exp_time = $item->{exp_time}; + + my $rootName = $ipprc->file_resolve ($path_base); + my $statsName = "$rootName.$class_id.stats"; + + # print STDERR "rootName: $rootName : $exp_time\n"; + print STDERR "statsName: $statsName : $exp_time\n"; + + my $statsFile; + open $statsFile, $statsName; + my @contents = <$statsFile>; + close ($statsFile); + + # print STDERR "contents: @contents\n"; + + my $parser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR); + + &parse_stats_table ($exp_time, $class_id, $statsList); +} + +print STDERR "parsed the stats from the data files\n"; + +for (my $i = 0; $i < @bg_data; $i++) { + $nameX = "$bg_name[$i].exp"; + $nameY = "$bg_name[$i].bg"; + push @{$nameX}, $bg_exptime[$i]; + push @{$nameY}, $bg_data[$i]; +} + +if (-e "output.dat") { unlink "output.dat"; } + +print STDERR "dumping stats\n"; +open (MANA, "|mana --norc"); +MANA->autoflush; + +foreach my $component (@components) { + $nameX = "$component.exp"; + $nameY = "$component.bg"; + + print MANA "delete X Y\n"; + + open (DATA, ">$component.dat"); + for (my $i = 0; $i < @{$nameX}; $i++) { + print DATA "${$nameX}[$i] ${$nameY}[$i]\n"; + } + close (DATA); + + print MANA "data $component.dat\n"; + print MANA "read X 1 Y 2\n"; + print MANA "fit X Y 2 -clip 3 3\n"; + print MANA "output output.dat\n"; + print MANA "echo $component METADATA\n"; + print MANA "echo \" NORDER_X S32 2 \"\n"; + print MANA "echo \" VAL_X00 F64 \$C0\"\n"; + print MANA "echo \" VAL_X01 F64 \$C1\"\n"; + print MANA "echo \" VAL_X02 F64 \$C2\"\n"; + print MANA "echo \" NELEMENTS S32 3 \"\n"; + print MANA "echo END\n"; + print MANA "echo\n"; + print MANA "output stdout\n"; + + print MANA "applyfit X Yf\n"; + print MANA "lim X Y\n"; + print MANA "clear\n"; + print MANA "box\n"; + print MANA "plot -x 2 -pt 2 -sz 1.0 -c black X Y\n"; + print MANA "plot -x 2 -pt 7 -sz 1.0 -c red X Yf\n"; + + print STDERR "hit return to continue\n"; + $answer = ; +} + +close (MANA); + +exit 0; + +sub parse_stats_table +{ + my ($exp_time, $tag, $md) = @_; + + # descend through the fpa + foreach my $entry (@$md) { + # print STDERR "name: $entry->{name}, class: $entry->{class}\n"; + # recurse on nested metadata + if ($entry->{class} eq 'metadata') { + my $newtag = $tag . "_" . $entry->{name}; + &parse_stats_table ($exp_time, $newtag, $entry->{value}); + } + + if ($entry->{name} =~ /^(SAMPLE|ROBUST|FITTED|CLIPPED)/) { + # It's a statistic of some sort + if ($entry->{name} =~ /_STDEV$/) { + push @bg_stdev_data, $entry->{value}; + } else { + push @bg_name, $tag; + push @bg_data, $entry->{value}; + push @bg_exptime, $exp_time; + # print STDERR "$tag $exp_time $entry->{value}\n"; + } + if (!$componentsHash{$tag}) { + push @components, $tag; + $componentsHash{$tag} = 1; + } + next; + } + } + return 1; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exit_code = shift; # Exit code to add + + carp($msg); + exit $exit_code; +} + +# - get the exp_time as well from dettool +# - build an array of bg & exptime for each cell +# - fit the trend (in mana? pslib functions?) +# - write the polynomial for each cell Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_camera.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_camera.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_camera.pl (revision 16175) @@ -0,0 +1,74 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $workdir, # Working directory + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'workdir|w=s' => \$workdir, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $camera_exp = can_run('camera_exp.pl') or (warn "Can't find camera_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Camera exposure processing +my $list; +{ + my $command = "$camtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get camera exposure list: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from camtool.\n"; +} + +foreach my $item (@$list) { + my $cam_id = $item->{cam_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + + my $command = "$camera_exp --cam_id $cam_id --exp_tag $exp_tag --camera $camera --dbname $dbname"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do camera processing on $cam_id: $error_code\n" if not $success; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_chip.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_chip.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_chip.pl (revision 16175) @@ -0,0 +1,93 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +my $chip = can_run('chip_imfile.pl') or (warn "Can't find chip_imfile.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 2 imfile processing +my @whole; # The whole list for processing +{ + my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 2 imfile list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); +} + +my @single = (); + +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from chiptool.\n"; + + foreach my $item (@$list) { + my $chip_id = $item->{chip_id}; + my $exp_id = $item->{exp_id}; + my $exp_tag = $item->{exp_tag}; + my $camera = $item->{camera}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $reduction = $item->{reduction}; + my $workdir = $item->{workdir}; + + my $command = "$chip --chip_id $chip_id --exp_id $exp_id --exp_tag $exp_tag --class_id $class_id --uri $uri --dbname $dbname --camera $camera"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir and $workdir ne "NULL"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success; + } + + @single = (); + + } +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_detrend.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_detrend.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_detrend.pl (revision 16175) @@ -0,0 +1,315 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use Getopt::Long; +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +my ($dbname, # Database name to use + ); +GetOptions( + 'dbname|d=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); +my $detrend_process_imfile = can_run('detrend_process_imfile.pl') or (warn "Can't find detrend_process_imfile.pl" and $missing_tools = 1); +my $detrend_process_exp = can_run('detrend_process_exp.pl') or (warn "Can't find detrend_process_exp.pl" and $missing_tools = 1); +my $detrend_stack = can_run('detrend_stack.pl') or (warn "Can't find detrend_stack.pl" and $missing_tools = 1); +my $detrend_norm_calc = can_run('detrend_norm_calc.pl') or (warn "Can't find detrend_norm_calc.pl" and $missing_tools = 1); +my $detrend_norm_apply = can_run('detrend_norm_apply.pl') or (warn "Can't find detrend_norm_apply.pl" and $missing_tools = 1); +my $detrend_norm_exp = can_run('detrend_norm_exp.pl') or (warn "Can't find detrend_norm_exp.pl" and $missing_tools = 1); +my $detrend_resid = can_run('detrend_resid.pl') or (warn "Can't find detrend_resid.pl" and $missing_tools = 1); +my $detrend_reject_imfile = can_run('detrend_reject_imfile.pl') or (warn "Can't find detrend_reject_imfile.pl" and $missing_tools = 1); +my $detrend_reject_exp = can_run('detrend_reject_exp.pl') or (warn "Can't find detrend_reject_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Process raw imfiles +{ + my $command = "$dettool -toprocessedimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $class_id = $item->{class_id}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + my $command = "$detrend_process_imfile --det_id $det_id --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --input_uri $uri --camera $camera --dbname $dbname"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw imfile processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Process raw exposures +{ + my $command = "$dettool -toprocessedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get detrend raw list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + my $command = "$detrend_process_exp --det_id $det_id --det_type $det_type --exp_tag $exp_tag --exp_id $exp_id --camera $camera --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do raw exposure processing on $det_id $exp_tag: $error_code\n" if not $success; + } +} + +# Stack +{ + my $command = "$dettool -tostacked -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get stack list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + my $command = "$detrend_stack --det_id $det_id --iteration $iteration --class_id $class_id --det_type $det_type --camera $camera --dbname $dbname"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to stack detrend $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Calculate normalisation +{ + my $command = "$dettool -tonormalizedstat -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalise calculation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + my $command = "$detrend_norm_calc --det_id $det_id --iteration $iteration --det_type $det_type --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to calculate normalisation for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Apply normalisation +{ + my $command = "$dettool -tonormalize -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalisation list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + my $command = "$detrend_norm_apply --det_id $det_id --iteration $iteration --class_id $class_id --value $value --input_uri $uri --camera $camera --det_type $det_type --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to apply normalisation for $det_id $iteration $class_id: $error_code\n" if not $success; + } +} + +# Examine normalised exposure +{ + my $command = "$dettool -tonormalizedexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get normalised exposures list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $class_id = $item->{class_id}; + my $value = $item->{norm}; + my $uri = $item->{uri}; + my $camera = $item->{camera}; + my $workdir = $item->{workdir}; + + my $command = "$detrend_norm_exp --det_id $det_id --iteration $iteration --camera $camera --det_type $det_type --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to examine normalised exposure for $det_id $iteration: $error_code\n" if not $success; + } +} + +# Get residuals +{ + my $command = "$dettool -toresidimfile -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual processing list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $class_id = $item->{class_id}; + my $det_type = $item->{det_type}; + my $detrend = $item->{det_uri}; + my $uri = $item->{uri}; + my $mode = $item->{mode}; + my $workdir = $item->{workdir}; + my $reduction = $item->{reduction}; + + my $command = "$detrend_resid --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --class_id $class_id --det_type $det_type --camera $camera --input_uri $uri --mode $mode --dbname $dbname"; + $command .= " --reduction $reduction" if defined $reduction; + $command .= " --detrend $detrend" if defined $detrend; + $command .= " --workdir $workdir" if defined $workdir; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do residual processing on $exp_tag $class_id: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + +# Reject based on imfiles +{ + my $command = "$dettool -toresidexp -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual imfile list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{exp_tag}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + my $command = "$detrend_reject_imfile --det_id $det_id --iteration $iteration --exp_tag $exp_tag --exp_id $exp_id --det_type $det_type --camera $camera --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do imfile rejection on $exp_tag $det_id $iteration: $error_code\n" if not $success; + } +} + +# Reject based on exposures +{ + my $command = "$dettool -todetrunsummary -dbname $dbname"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get residual exposure list: $error_code\n" if not $success; + my $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from dettool.\n"; + + foreach my $item (@$list) { + my $camera = $item->{camera}; + my $det_id = $item->{det_id}; + my $iteration = $item->{iteration}; + my $det_type = $item->{det_type}; + my $workdir = $item->{workdir}; + + my $command = "$detrend_reject_exp --det_id $det_id --iteration $iteration --det_type $det_type --camera $camera --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do exposure rejection on $det_id $iteration: $error_code\n" if not $success; + } +} + + +__END__ + + Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_diff.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_diff.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_diff.pl (revision 16175) @@ -0,0 +1,91 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $difftool = can_run('difftool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $diff_skycell = can_run('diff_skycell.pl') or + (warn "Can't find diff_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image differencing +{ + my $list; + my $command = "$difftool -todiffskyfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of diffs: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from difftool.\n"; + + foreach my $item (@$list) { + my $diff_id = $item->{diff_id}; + my $workdir = $item->{workdir}; + + my $command = "$diff_skycell --diff_id $diff_id --dbname $dbname"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); +### die "Unable to do difference for $diff_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject.pl (revision 16175) @@ -0,0 +1,121 @@ +#!/usr/bin/env perl + +# this program injects a list of single-file exposures into the db, +# taking the filename (without .fits) as the exp_tag. the user +# supplies a temporary telescope and camera name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and camera name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($workdir, $dbname, $telescope, $camera, $reduction, $help); +GetOptions( + 'workdir|w=s' => \$workdir, # working directory for output files + 'dbname|d=s' => \$dbname, # Database name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'camera|i=s' => \$camera, # user-supplied camera name + 'reduction=s' => \$reduction, # user-supplied camera name + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless + defined $telescope and + defined $camera; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + my $absfile = File::Spec->rel2abs( $file ); + inject($absfile, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +print "$num files injected.\n"; + +sub inject +{ + my $absfile = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile ); + my ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -dbname $dbname" if defined $dbname; + $command_exp .= " -reduction $reduction" if defined $reduction; + + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id fpa"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + + return 1; +} + + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject_mosaic.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject_mosaic.pl (revision 16175) @@ -0,0 +1,166 @@ +#!/usr/bin/env perl + +# this program injects a set of multi-file exposures into the db. the +# program takes a list of base exposure names and injects all files +# associated with the exposure. It constructs the expected filenames +# from the exposure tag and rules for the camera. This program is for +# the test only since it requires too much information at the inject +# stage. use 'ipp_serial_inject.pl' for single-file images and +# 'ipp_serial_inject_split.pl' for multiple file images in split +# format + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($camera, # Camera used + $telescope, # Telescope used + $dbname, # Database name + $workdir, # Working directory + $path, # Path to data + ); +GetOptions( + 'camera|c=s' => \$camera, + 'telescope|t=s' => \$telescope, + 'workdir=s' => \$workdir, + 'path=s' => \$path, + 'dbname=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --camera --telescope --workdir --path --dbname", + -exitval => 3, + ) unless defined $camera + and defined $telescope + and defined $workdir + and defined $path + and defined $dbname; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +# Look for programs we need +my $missing_tools; +my $pxinject = can_run('pxinject') or (warn "Can't find pxinject" and $missing_tools = 1); + +if (scalar @ARGV == 0) { + die "No exposures provided.\n"; +} + +# Inject new data into the database +my @classes; # Names of the classes +my @files; # What to add to the filename for each class +my $imfiles; +my $add_dir; # Add directory name to get file name? +if ($camera eq "MEGACAM") { + for (my $i = 0; $i < 36; $i++) { + push @classes, sprintf("ccd%02d", $i); + push @files, sprintf(".ccd%02d", $i); + } +} elsif ($camera eq "MCSHORT") { + @classes = ( 'ccd12', 'ccd13', 'ccd14', 'ccd21', 'ccd22', 'ccd23' ); + @files = ( '.ccd12', '.ccd13', '.ccd14', '.ccd21', '.ccd22', '.ccd23' ); +} elsif ($camera eq "CTIO_MOSAIC2") { + @classes = (); + @files = (); +} elsif ($camera eq "TC3") { + @classes = ( 'CCID58-1-06b2', 'CCID45-1-14A', 'CCID45-1-11A', 'CCID45-1-22A', + 'CCID45-1-04C', 'CCID45-1-13A', 'CCID45-1-05A', 'CCID45-1-19A' ); + @files = ( '00', '01', '10', '11', '20', '21', '30', '31' ); + $add_dir = 1; +} elsif ($camera eq "SIMMOSAIC") { + @classes = ( 'Chip00', 'Chip01', 'Chip10', 'Chip11' ); + @files = ( '.Chip00', '.Chip01', '.Chip10', '.Chip11' ); +} elsif ($camera eq "SIMTEST") { + @classes = (); + @files = (); +} elsif ($camera eq "GPC1") { + for (my $i = 0; $i < 8; $i++) { + for (my $j = 0; $j < 8; $j++) { + if (($i == 0 or $i == 7) and ($j == 0 or $j == 7)) { + # Excluding corner chips + next; + } + push @classes, "XY$i$j"; + push @files, "$i$j"; + } + } + $add_dir = 1; +} else { + die "Unrecognised camera name: $camera.\n"; +} + +foreach my $exp_name ( @ARGV ) { + my $command = "$pxinject -newExp -tmp_exp_name $exp_name -tmp_inst $camera -tmp_telescope $telescope -workdir $workdir"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name: $error_code\n" if not $success; + + my @line = split(/\s+/, $$stdout_buf[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + for (my $i = 0; $i < scalar @classes; $i++) { + my $class_id = $classes[$i]; + my $file_id = $files[$i]; + my $filename = $exp_name . $file_id . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id $class_id -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name $class_id: $error_code\n" if not $success; + } + + if (scalar @classes == 0) { + my $filename = $exp_name . '.fits'; + $filename = caturi( $exp_name, $filename ) if defined $add_dir; + $filename = caturi( $path, $filename ); + + die "Unable to find file $filename" unless -f $ipprc->file_resolve( $filename ); + + $filename = $ipprc->convert_filename_relative( $filename ); + my $command = "$pxinject -newImfile -exp_id $exp_id -tmp_class_id fpa -uri $filename"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code\n" if not $success; + } + + # Update the exposure to run + { + my $command = "$pxinject -updatenewExp -exp_id $exp_id -state run"; # Command to run + $command .= " -dbname $dbname" if defined ($dbname); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to activate $exp_name: $error_code\n" if not $success; + } + +} + +END { + my $status = $?; +system("sync") == 0 + or die "failed to execute sync: $!" ; +$? = $status; +} + + +__END__ + + Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject_split.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject_split.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_inject_split.pl (revision 16175) @@ -0,0 +1,133 @@ +#!/usr/bin/env perl + +# this program injects a list of multi-file exposures into the db. +# the program takes a list of directory names (dir), and injects all +# dir/*.fits. It takes the directory name as the exp_tag. the user +# supplies a temporary telescope and instrument name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and instrument name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($workdir, $dbname, $telescope, $camera, $reduction, $help); +GetOptions( + 'workdir|w=s' => \$workdir, # working directory for output files + 'dbname|d=s' => \$dbname, # Database name + 'telescope|t=s' => \$telescope, # Telescope used + 'camera|c=s' => \$camera, # Camera used + 'reduction=s' => \$reduction, # user-supplied camera name + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many image exposures (directories) into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( + -msg => "Required options: --camera --telescope --workdir --path --dbname", + -exitval => 3) unless + defined $telescope and + defined $camera; + +# Look for programs we need +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $filedir ( @ARGV ) { + # check for filedir existence + if (! -e $filedir) { die "file dir $filedir not found\n"; } + my $absfiledir = File::Spec->rel2abs( $filedir ); + inject($absfiledir, $workdir, $dbname, $telescope, $camera); + $num ++; +} + +sub inject +{ + my $absfiledir = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my @files = <$absfiledir/*.fits>; + my $Nfiles = scalar @files; + + print "absfiledir: $absfiledir\n"; + print "Nfiles: $Nfiles\n"; + + # the absfiledir is of the form path://PATH/data/foo/expname + my ( $vol, $path, $exp_name ) = File::Spec->splitpath( $absfiledir ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -dbname $dbname" if defined $dbname; + $command_exp .= " -reduction $reduction" if defined $reduction; + + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + my $exp_id = $line[2]; # The exposure tag + + foreach my $absfile (@files) { + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + my ( $tmpvol, $tmppath, $filename ) = File::Spec->splitpath( $absfile ); + my ( $class_name ) = $filename =~ /(.*)\.fits/; + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id $class_name"; + $command_imfile .= " -uri $relfile"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + + return 1; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_register.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_register.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_register.pl (revision 16175) @@ -0,0 +1,122 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + ); +GetOptions( + 'dbname|d=s' => \$dbname, +) or pod2usage( 2 ); + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $register_imfile = can_run('register_imfile.pl') or (warn "Can't find register_imfile.pl" and $missing_tools = 1); +my $register_exp = can_run('register_exp.pl') or (warn "Can't find register_exp.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Phase 0 imfile processing +{ + my $command = "$regtool -pendingimfile -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 imfile list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_id = $item->{exp_id}; + my $exp_name = $item->{tmp_exp_name}; + my $class_id = $item->{tmp_class_id}; + my $workdir = $item->{workdir}; + my $uri = $item->{uri}; + + my $command = "$register_imfile --exp_id $exp_id --tmp_class_id $class_id --tmp_exp_name $exp_name --uri $uri --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 imfile processing on $exp_name $class_id: $error_code\n" if not $success; + } + + + @single = (); + + } + } +} + +# Phase 0 exposure processing +{ + my $command = "$regtool -pendingexp -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get phase 0 exposure list: $error_code\n" if not $success; + + my @whole = split /\n/, join( '', @$stdout_buf ); + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from regtool.\n"; + + foreach my $item (@$list) { + my $exp_tag = $item->{tmp_exp_name} . '.' . $item->{exp_id}; + my $exp_id = $item->{exp_id}; + my $workdir = $item->{workdir}; + + my $command = "$register_exp --exp_tag $exp_tag --exp_id $exp_id --dbname $dbname"; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do phase 0 exposure processing on $exp_tag: $error_code\n" if not $success; + } + + @single = (); + + } + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_stack.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_stack.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_stack.pl (revision 16175) @@ -0,0 +1,94 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $no_op, # No operations? + $no_update, # No updating? + $save_temps, # Save temporary files? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'no-op' => \$no_op, + 'no-update' => \$no_update, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or + (warn "Can't find difftool" and $missing_tools = 1); +my $stack_skycell = can_run('stack_skycell.pl') or + (warn "Can't find stack_skycell.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Image stacking +{ + my $list; + my $command = "$stacktool -tosum -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get list of stacks: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from stacktool.\n"; + + foreach my $item (@$list) { + my $stack_id = $item->{stack_id}; + my $workdir = $item->{workdir}; + + my $command = "$stack_skycell --stack_id $stack_id --dbname $dbname"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --save-temps" if defined $save_temps; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); +### die "Unable to do stack for $stack_id: $error_code\n" if not $success; + } +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_warp.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_warp.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_serial_warp.pl (revision 16175) @@ -0,0 +1,125 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($dbname, # Database name to use + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( + 'dbname|d=s' => \$dbname, + 'no-op' => \$no_op, + 'no-update' => \$no_update, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --dbname", + -exitval => 3, + ) unless defined $dbname; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or + (warn "Can't find warptool" and $missing_tools = 1); +my $warp_skycell = can_run('warp_skycell.pl') or + (warn "Can't find warp_skycell.pl" and $missing_tools = 1); +my $warp_overlap = can_run('warp_overlap.pl') or + (warn "Can't find warp_overlap.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + +# Calculate overlaps +{ + my $list; + my $command = "$warptool -tooverlap -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for which to calculate overlaps: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + + my $command = "$warp_overlap --warp_id $warp_id --camera $camera --dbname $dbname"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warp overlaps on $warp_id: $error_code\n" if not $success; + } +} + + +# Warping proper +{ + my $list; + my $command = "$warptool -towarped -dbname $dbname"; # Command to run + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to get warps for warping: $error_code\n" if not $success; + $list = parse_md_list( $mdcParser->parse( join( '', @$stdout_buf ) ) ) or + die "Unable to parse output from warptool.\n"; + + foreach my $item (@$list) { + my $warp_id = $item->{warp_id}; + my $skycell_id = $item->{skycell_id}; + my $tess_id = $item->{tess_id}; + my $cam_id = $item->{cam_id}; + my $workdir = $item->{workdir}; + my $camera = $item->{camera}; + + my $command = "$warp_skycell --warp_id $warp_id --skycell_id $skycell_id --tess_id $tess_id --camera $camera --dbname $dbname"; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + $command .= " --workdir $workdir" if defined $workdir and $workdir ne "NULL"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => 1 ); + die "Unable to do warp processing on $warp_id,$skycell_id: $error_code\n" if not $success; + } +} + + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/ipp_simulation_data.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/ipp_simulation_data.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/ipp_simulation_data.pl (revision 16175) @@ -0,0 +1,220 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Math::Trig; +use File::Spec; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($name, # Base name for output images + $camera, # Name of camera to use + $telescope, # Telesceope name + $dbname, # Database name + $path, # Path to data + $workdir, # Working directory for data + $no_cal, # Don't produce calibration files + $no_update # Don't update the database + ); + +GetOptions( + 'name=s' => \$name, + 'camera=s' => \$camera, + 'telescope=s' => \$telescope, + 'dbname=s' => \$dbname, + 'path=s' => \$path, + 'workdir=s' => \$workdir, + 'no-cal' => \$no_cal, + 'no-update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --name --path --camera --telescope --dbname", + -exitval => 3, + ) unless + defined $name and + defined $path and + defined $camera and + defined $telescope and + defined $dbname; + +$workdir = $path if not defined $workdir; + +# Look for programs we need +my $missing_tools; +my $ppSim = can_run('ppSim') + or (warn "Can't find ppSim" and $missing_tools = 1); +my $inject = can_run('ipp_serial_inject_mosaic.pl') + or (warn "Can't find ipp_serial_inject_mosaic.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Number of bias images +use constant BIAS => 20; +# Dark exposure times +use constant DARK => [ 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300 ]; +# Flat-field image characteristics +use constant FLAT => [ + { + filter => 'r', + exptime => [ 0.1, 0.1, 0.1, 0.1, 0.1, 0.5, 1, 2, 5, 10, 20, 20, 20, 20, 20 ], + }, + { + filter => 'i', + exptime => [ 20, 20, 20, 20, 20 ], + } + ]; +# Object image characteristics +use constant OBJECT => [ + { + filter => 'r', + exptime => [ 5, 10, 10, 10, 10, 10, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.15, + sky => 20.86, + dither => 40, + }, + { + filter => 'i', + exptime => [ 5, 30, 30, 30, 30, 30, 240 ], + seeing => [ 0.8, 0.4, 0.6, 0.8, 1.2, 1.5, 0.7 ], + ra => 150.119167, + dec => 2.205833, + pa => 0, + zp => 25.00, + sky => 20.15, + dither => 40, + }, + ]; +use constant SCALE => 0.2; # Plate scale + +############################################################################################################# +### Now do the work +############################################################################################################# + +my $counter = 0; + +# Generate bias images +for (my $i = 0; $i < BIAS; $i++) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type BIAS $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate dark images +foreach my $exptime ( @{DARK()} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run ( command => "$ppSim -camera $camera -type DARK -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + +# Generate flat images +foreach my $set ( @{FLAT()} ) { + my $filter = $set->{filter}; # Name of filter + foreach my $exptime ( @{$set->{exptime}} ) { + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + unless ($no_cal) { + run( command => "$ppSim -camera $camera -type FLAT -filter $filter -exptime $exptime $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } + } +} + +# Generate object images +foreach my $set ( @{OBJECT()} ) { + my $filter = $set->{filter}; # Name of filter + my $ra0 = $set->{ra}; # Base Right Ascension (deg) + my $dec0 = $set->{dec}; # Base Declination (deg) + my $pa = $set->{pa}; # Position angle (deg) + my $zp = $set->{zp}; # Zero point + my $scale = SCALE(); # Plate scale (arcsec/pix) + my $sky = 10**( -0.4 * ( $set->{sky} - $zp ) ) * $scale**2; # Sky background (counts/s) + my $dither = $set->{dither} / 3600; # Dither size (deg) + + for (my $i = 0; $i < scalar @{$set->{exptime}}; $i++) { + my $exptime = ${$set->{exptime}}[$i]; # Exposure time + my $seeing = ${$set->{seeing}}[$i]; # Seeing (pix) + my $ra = $ra0 + (2*rand() - 1) * $dither * cos(deg2rad($dec0)); # RA with dither + my $dec = $dec0 + (2*rand() - 1) * $dither; # Dec with dither + + my $basename; # Output base filename + ( $basename, $counter ) = filename( $name, $counter ); + my $filename = caturi( $path, $basename ); + run( command => "$ppSim -camera $camera -type OBJECT -filter $filter -exptime $exptime " . + "-skyrate $sky -ra $ra -dec $dec -pa $pa -scale $scale -zp $zp -seeing $seeing $filename", + verbose => 1 ) or die "Unable to run ppSim"; + unless ($no_update) { + run( command => "$inject --camera $camera --telescope $telescope --path $path " . + "--workdir $workdir --dbname $dbname $basename", + verbose => 1 ) or die "Unable to inject file."; + } + } +} + + +### Pau. + + +# Generate a filename from the base name and counter +sub filename +{ + my $base = shift; # Base name + my $num = shift; # Number + my $workdir = shift; # Working directory + my $name = sprintf("$base%04d", $num); + $num++; + return ( $name, $num ); +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/isp_trans.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/isp_trans.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/isp_trans.pl (revision 16175) @@ -0,0 +1,30 @@ +#!/usr/bin/env perl +# basic ISP transmission analysis: + +if (@ARGV != 1) { die "USAGE: isp_trans.pl (input.fits)\n"; } +$input = $ARGV[0]; + +# for input file /path/foo.fits, use /path/foo for output + +@words = split ('\.', $input); +if (@words > 1) { pop @words; } +$output = join (".", @words); + +# use constant RECIPE => 'PPIMAGE_OBDSFRA'; # Recipe to use +$RECIPE_PPIMAGE = 'PPIMAGE_OA'; # Recipe to use (switch to OBDSFRA when detrend images are ready) +$RECIPE_PSPHOT = 'PSPHOT.SUMMIT'; +$CALDIR = '/data/alala.0/ipp/ippRefs/catdir.synth.bright'; # source of photometric calibration data +$IMTABLE = 'images.dat'; # source of photometric calibration data + +vsystem ("ppImage -file $input $output -recipe PPIMAGE $RECIPE_PPIMAGE -recipe PSPHOT $RECIPE_PSPHOT"); +if ($status) { die "failure running ppImage\n"; } + +vsystem ("addstar -incal -image -D CAMERA isp -D IMAGE_TABLE $IMTABLE -D CATDIR $CALDIR $output.smf"); +if ($status) { die "failure getting calibration from addstar\n"; } + +sub vsystem { + print STDERR "@_\n"; + my $status = system ("@_"); + $status; +} + Index: /branches/eam_branch_20080121/ippScripts/scripts/magic_tree.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/magic_tree.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/magic_tree.pl (revision 16175) @@ -0,0 +1,344 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Operations qw( skycell_file + generate_skycells + ); + +use Astro::FITS::CFITSIO qw( :constants ); +Astro::FITS::CFITSIO::PerlyUnpacking(1); + +use Math::Trig; +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use constant MAX_FIELDS => 4; # Maximum number of fields to be in a node + +# Parse the command-line arguments +my ($magic_id, $tess_id, $camera, $ra0, $dec0, $dbname, $workdir, $save_temps, $no_update, $no_op); +GetOptions( + 'magic_id=s' => \$magic_id, # Magic identifier + 'tess_id=s' => \$tess_id, # Tessellation identifier + 'camera=s' => \$camera, # Camera name + 'ra=f' => \$ra0, # Boresight right ascension, radians + 'dec=f' => \$dec0, # Boresight declination, radians + 'dbname=s' => \$dbname, # Database name + 'workdir=s' => \$workdir, # Working directory, for output files + 'save-temps' => \$save_temps, # Save temporary files? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_id --camera --workdir", + -exitval => 3) unless + defined $magic_id and + defined $tess_id and + defined $ra0 and + defined $dec0 and + defined $camera and + defined $workdir; + +$ipprc->define_camera($camera); + + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +### Get a list of skycells +my @skycells; # List of skycells +{ + my $command = "$magictool -inputskyfile -magic_id $magic_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magictool -inputfile: $error_code", $magic_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); + + my $inputs = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); + + foreach my $input ( @$inputs ) { + push @skycells, $input->{skycell_id}; + } +} + +generate_skycells($ipprc, $tess_id, \@skycells, $workdir) or + &my_die("Unable to generate skycells", $magic_id, $PS_EXIT_PROG_ERROR); + +### For each skycell, project centre of skycell onto tangent plane of boresight +my @fields; +foreach my $skycell_id ( @skycells ) { + my $skyfile = skycell_file($tess_id, $skycell_id, $workdir) or + &my_die("Unable to get name of skycell file", $magic_id, $PS_EXIT_PROG_ERROR); + &my_die("Unable to find skycell file", $magic_id, $PS_EXIT_SYS_ERROR) unless + $ipprc->file_exists( $skyfile ); + + my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfile ); + &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status; + + # Get the useful header keywords + my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR); + my $naxis2 = $$header{'NAXIS2'} or &my_die("Can't find NAXIS2", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR); + my $crota1 = $$header{'CROTA1'}; + my $crota2 = $$header{'CROTA2'}; + + &my_die("Unexpected projection: $ctype1 and $ctype2.", $magic_id, $PS_EXIT_SYS_ERROR) unless + $ctype1 =~ /^\'RA---TAN\s*\'$/ and $ctype2 =~ /^\'DEC--TAN\s*\'$/; + &my_die("Can't determine size of skycell ($naxis1,$naxis2)", $magic_id, $PS_EXIT_SYS_ERROR) unless + $naxis1 > 0 and $naxis2 > 0; + &my_die("Can't determine scale of skycell ($cdelt1,$cdelt2)", $magic_id, $PS_EXIT_SYS_ERROR) if + not defined $cdelt1 or $cdelt1 == 0 or not defined $cdelt2 or $cdelt2 == 0; + &my_die("We don't know how to handle rotations ($crota1,$crota2)", $magic_id, $PS_EXIT_SYS_ERROR) + if defined $crota1 or defined $crota2; + + # Relative coordinates of centre of the field + my $x = $naxis1 - $crpix1; + my $y = $naxis2 - $crpix2; + + # Coordinates on tangent plane + my $xi = $pc11 * ($x) + $pc12 * ($y); + my $eta = $pc21 * ($x) + $pc22 * ($y); + $xi *= $cdelt1; + $eta *= $cdelt2; + + # Coordinates on rotated celestial sphere + my $phi = atan2($eta,$xi) + pi/2; + my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2)); + + # Coordinates on celestial sphere + $crval1 = deg2rad($crval1); + $crval2 = deg2rad($crval2); + my $ra = $crval1 + atan2(cos($theta) * sin($phi), + sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi)); + my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi)); + + # Rotate to boresight + my $phi_new = atan2(cos($dec) * sin($ra - $ra0), + sin($dec) * cos($dec0) + cos($dec) * sin($dec0) * cos($ra - $ra0)); + my $theta_new = asin(sin($dec) * sin($dec0) - cos($dec) * cos($dec0) * cos($ra - $ra0)); + + # Project + my $rad = 180 / pi * cot($theta_new); + my $xi_new = $rad * sin($phi); + my $eta_new = - $rad * cos($phi); + + my $field = { id => $skycell_id, + xi => $xi_new, + eta => $eta_new, + }; + + push @fields, $field; +} + +### Subdivide list of positions into kd-tree +my $root = { # Root node of tree + contents => \@fields, # Contents of node + position => 'root', # Position in tree + children => {}, # Children of node +}; +my @tasks = ( $root ); +while (scalar @tasks > 0) { + my $node = shift @tasks; + divide_node($node, \@tasks); +} + +### Format tree for magictool +my $mdcTree = print_node($root); # The tree in MDC format +my ($treeFile, $treeName) = tempfile( "magictree.${magic_id}.XXXX", UNLINK => !$save_temps ); +print $treeFile, $mdcTree; +close $treeFile; + +### Input tree into database +{ + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -dep_file $treeName"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform magictool -inputtree: $error_code"); + exit($error_code); + } + } else { + print "Skipping command: $command\n"; + } +} + +### Pau. + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_id = shift; # Magic identifier + my $exit_code = shift; # Exit code to add + + carp($msg); + if ($magic_id and not $no_update) { + my $command = "$magictool -inputtree"; + $command .= " -magic_id $magic_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +# Divide a list into two, returning the lower and upper parts +sub divide_list +{ + my $list = shift; # List to divide + my $index = shift; # Name of index for sorting + + my @sorted = sort { $$a{$index} <=> $$b{$index} } @$list; # Sorted list + my $median = int(scalar @sorted / 2); # Median point of list + my @upper = splice(@sorted, $median); # Upper part of the sorted list + + return (\@sorted, \@upper); +} + +# Create a new node, add it to the parent, and add it to the task list if required +sub new_node +{ + my $parent = shift; # The parent node + my $contents = shift; # Contents of the new node + my $position = shift; # Position description + my $tasks = shift; # Tasks to do + + my $node = { + contents => $contents, + position => $parent->{position} . '_' . $position, + children => {}, + }; + + $parent->{children}->{$position} = $node; + + push @$tasks, $node if scalar @$contents > 4; + + return $node; +} + +# Divide a node +sub divide_node +{ + my $node = shift; # Node to divide + my $tasks = shift; # Tasks to do + + my $position = $node->{position}; + + my $contents = $node->{contents} or die "Can't find contents of node."; # Contents of node + + my ($lower, $upper) = divide_list($contents, 'xi'); + + if (scalar @$lower > 4) { + my ($ll, $lr) = divide_list($lower, 'eta'); + new_node($node, $ll, 'll', $tasks); + new_node($node, $lr, 'lr', $tasks); + } else { + new_node($node, $lower, 'L', $tasks); + } + + if (scalar @$upper > 4) { + my ($ul, $ur) = divide_list($upper, 'eta'); + new_node($node, $ul, 'ul', $tasks); + new_node($node, $ur, 'ur', $tasks); + } else { + new_node($node, $upper, 'U', $tasks); + } + + $node->{contents} = undef; + + return $node; +} + +# Print the contents of a node +sub print_node +{ + my $node = shift; # Node to print + + my $position = $node->{position}; # Position of node + + my $output = "$position\t\tMULTI\n"; # Output text + + if (defined $node->{contents}) { + foreach my $field ( @{$node->{contents}} ) { + my $skycell_id = $field->{id}; # Skycell name + $output .= "$position\t\tSTR\t$skycell_id\n"; + $output .= "$skycell_id\t\tSTR\tNULL\t\# $field->{xi},$field->{eta}\n"; + } + } else { + foreach my $div ( keys %{$node->{children}} ) { + $output .= "$position\t\tSTR\t${position}_$div\n"; + $output .= print_node($node->{children}->{$div}); + } + } + + return $output; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/mdc2list.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/mdc2list.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/mdc2list.pl (revision 16175) @@ -0,0 +1,77 @@ +#!/usr/bin/env perl + +# Program to convert a PS metadata config syntax (usually from one of +# the IPP tools) into a flat tab-delimited list suitable for parsing +# using "split". + +# Copyright (C) 2006 Joshua Hoblitt, Paul A. Price. + +use strict; +use warnings; + +use PS::IPP::Metadata::Config; # Supplies the metadata config parser. + +die "Program to convert a PS metadata config file (usually from one of the IPP\n" . + "tools) into a flat tab-delimited list suitable for parsing using \'split\'.\n\n" . + "Usage: $0 [INFILE [OUTFILE]]\n" if ((join "", @ARGV) =~ /--help/ or scalar @ARGV > 2); + +my $inFile; # Input file +my $outFile; # Output file +if (scalar @ARGV >= 1) { + my $inName = shift @ARGV; + open $inFile, $inName or die "Can't open $inName: $!\n"; + if (scalar @ARGV == 1) { + my $outName = shift @ARGV; + open $outFile, ">", $outName or die "Can't open $outName: $!\n"; + } else { + $outFile = *STDOUT; + } +} else { + $inFile = *STDIN; + $outFile = *STDOUT; +} + +my @input = <$inFile>; # Contents of the metadata config file +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $md = $mdcParser->parse(join "", @input) + or die "unable to parse metadata config doc"; +my $hashes = mds2hashes($md); # An array of hashes +foreach my $pending (@$hashes) { + foreach my $key (keys %$pending) { + print $outFile ( $pending->{$key} . "\t"); + } + print $outFile "\n"; +} + +### Pau. + + +# Given an array of MDs, return an array of hashes +sub mds2hashes +{ + my $mds = shift; # Reference to the metadatas + my @array; # The array of hashes, to be returned + foreach my $md (@$mds) { + my $values = md2hash($md->{value}); + push @array, $values; + } + return \@array; +} + +# Convert the metadata to a hash; in effect, strips out the comment, type and class fields. +sub md2hash +{ + my $values = shift; # Reference to the metadata + my %hash; # Hash, to be returned + foreach my $data (@$values) { + $hash{$data->{name}} = $data->{value}; + } + return \%hash; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} Index: /branches/eam_branch_20080121/ippScripts/scripts/register_exp.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/register_exp.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/register_exp.pl (revision 16175) @@ -0,0 +1,224 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw( freeze thaw ); +use File::Basename qw( basename ); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my ($cache, $exp_id, $exp_tag, $workdir, $dbname, $no_update, $no_op); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'exp_tag|t=s' => \$exp_tag, + 'workdir|w=s' => \$workdir, # Working directory for output files + 'dbname|d=s' => \$dbname, # Database name + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --exp_tag", + -exitval => 3) unless + defined $exp_id and + defined $exp_tag; + +# add -detrend UNLESS type is one of SCIENCE listed below (eg, OBJECT) +my @SCIENCE = ( "object", "science" ); # Observation types to NOT mark as detrend +my $DETREND_FLAG = "-end_stage reg"; # Flag to use to mark detrend exposure + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ # register imfile + # label STATISTIC CHIPTOOL FLAG + { name => "exp_name", type => "constant", flag => "-exp_name", dtype => "string" }, # File level + { name => "telescope", type => "constant", flag => "-telescope", dtype => "string" }, # File level + { name => "camera", type => "constant", flag => "-inst", dtype => "string" }, # File level + { name => "filelevel", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "object", type => "constant", flag => "-object", dtype => "string" }, + { name => "exp_type", type => "constant", flag => "-exp_type", dtype => "string" }, # File level + { name => "filter", type => "constant", flag => "-filter", dtype => "string" }, # File level + { name => "comment", type => "constant", flag => "-comment", dtype => "string" }, # ObsComment + { name => "dateobs", type => "constant", flag => "-dateobs", dtype => "string" }, # File level + { name => "ccd_temp", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "exp_time", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "sat_pixel_frac", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # Fraction of saturated pixels + { name => "airmass", type => "mean", flag => "-airmass", dtype => "float" }, # Airmass + { name => "ra", type => "mean", flag => "-ra", dtype => "float" }, # Right ascension + { name => "decl", type => "mean", flag => "-decl", dtype => "float" }, # Declination + { name => "posang", type => "mean", flag => "-posang", dtype => "float" }, # Position angle + { name => "alt", type => "mean", flag => "-alt", dtype => "float" }, # Altitude + { name => "az", type => "mean", flag => "-az", dtype => "float" }, # Azimuth + { name => "m1_x", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "m1_y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "m1_z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "m1_tip", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "m1_tilt", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "m2_x", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "m2_y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "m2_z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "m2_tip", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "m2_tilt", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "env_temperature", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "env_humidity", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "env_wind_speed", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "env_wind_dir", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "-teltemp_m1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "-teltemp_m1cell", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "-teltemp_m2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "-teltemp_spider", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "-teltemp_truss", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "-teltemp_extra", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "pon_time", type => "mean", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "bg", type => "mean", flag => "-bg", dtype => "float" }, # background + { name => "bg", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, # Azimuth + { name => "bg_stdev", type => "rms", flag => "-bg_stdev", dtype => "float" }, # Azimuth + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for commands we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "can't find regtool" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +# Get the list of imfiles & their stats +{ + my $command = "$regtool -processedimfile -exp_id $exp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -processedimfile on exposure id $exp_id: $error_code"); + exit ($error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $PS_EXIT_CONFIG_ERROR); + } +} + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_name")) eq "NULL") { &my_die ("exp_name not found", $exp_id, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedexp"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_tag $exp_tag"; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +my $exp_type = $stats->value_for_flag ("-exp_type"); + +# Add the detrend flag, if needed +{ + my $object = 0; # Is it an object exposure? + foreach my $scienceType (@SCIENCE) { + if (lc($exp_type) =~ /$scienceType/) { + $object = 1; + last; + } + } + $command .= " $DETREND_FLAG" unless $object; +} + +# Output results to the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to run regtool -addprocessedexp for $exp_id: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +### Pau. + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exit_code = shift; + + carp($msg); + if ($exp_id and not $no_update) { + my $command = "$regtool -addprocessedexp -exp_id $exp_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; +### system($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} Index: /branches/eam_branch_20080121/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/register_imfile.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/register_imfile.pl (revision 16175) @@ -0,0 +1,246 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Cache::File; +use Storable qw(freeze thaw); +use File::Basename qw( basename); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; + +use PS::IPP::Config qw( + $PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration +use File::Spec; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $workdir, $dbname, $no_update, $no_op); +GetOptions( + 'caches' => \$cache, + 'exp_id|e=s' => \$exp_id, + 'tmp_class_id|i=s' => \$tmp_class_id, + 'tmp_exp_name|n=s' => \$tmp_exp_name, + 'uri|u=s' => \$uri, + 'workdir|w=s' => \$workdir, # Working directory for output files + 'dbname|d=s' => \$dbname,# Database name + 'no-update' => \$no_update, + 'no-op' => \$no_op, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --tmp_class_id --tmp_exp_name --uri", + -exitval => 3) unless + defined $exp_id and + defined $tmp_class_id and + defined $tmp_exp_name and + defined $uri; + +my $RECIPE = "REGISTER"; # Recipe to use for ppStats + +# values to extract from output metadata and the stats to calculate +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC CHIPTOOL FLAG + { name => "FILE.LEVEL", type => "constant", flag => "-filelevel", dtype => "string" }, # File level + { name => "CLASS.ID", type => "constant", flag => "-class_id", dtype => "string" }, # Real Class ID + { name => "FPA.OBJECT", type => "constant", flag => "-object", dtype => "string" }, # Object + { name => "FPA.OBSTYPE", type => "constant", flag => "-exp_type", dtype => "string" }, # Exposure type + { name => "FPA.FILTER", type => "constant", flag => "-filter", dtype => "string" }, # Filter used + { name => "FPA.COMMENT", type => "constant", flag => "-comment", dtype => "string" }, # Obs Comment + { name => "FPA.AIRMASS", type => "constant", flag => "-airmass", dtype => "float" }, # Airmass + { name => "FPA.RA", type => "constant", flag => "-ra", dtype => "float" }, # Right ascension + { name => "FPA.DEC", type => "constant", flag => "-decl", dtype => "float" }, # Declination + { name => "FPA.ALT", type => "constant", flag => "-alt", dtype => "float" }, # Altitude + { name => "FPA.AZ", type => "constant", flag => "-az", dtype => "float" }, # Azimuth + { name => "FPA.POSANGLE", type => "constant", flag => "-posang", dtype => "float" }, # Position angle + { name => "FPA.TIME", type => "constant", flag => "-dateobs", dtype => "string" }, # Date of observation (UTC) + { name => "FPA.TELESCOPE", type => "constant", flag => "-telescope", dtype => "string" }, # Telescope + { name => "FPA.INSTRUMENT", type => "constant", flag => "-inst", dtype => "string" }, # Instrument + { name => "FPA.M1X", type => "constant", flag => "-m1_x", dtype => "float" }, # M1X + { name => "FPA.M1Y", type => "constant", flag => "-m1_y", dtype => "float" }, # M1Y + { name => "FPA.M1Z", type => "constant", flag => "-m1_z", dtype => "float" }, # M1Z + { name => "FPA.M1TIP", type => "constant", flag => "-m1_tip", dtype => "float" }, # M1TIP + { name => "FPA.M1TILT", type => "constant", flag => "-m1_tilt", dtype => "float" }, # M1TILT + { name => "FPA.M2X", type => "constant", flag => "-m2_x", dtype => "float" }, # M2X + { name => "FPA.M2Y", type => "constant", flag => "-m2_y", dtype => "float" }, # M2Y + { name => "FPA.M2Z", type => "constant", flag => "-m2_z", dtype => "float" }, # M2Z + { name => "FPA.M2TIP", type => "constant", flag => "-m2_tip", dtype => "float" }, # M2TIP + { name => "FPA.M2TILT", type => "constant", flag => "-m2_tilt", dtype => "float" }, # M2TILT + { name => "FPA.ENV.TEMP", type => "constant", flag => "-env_temperature", dtype => "float" }, # external temp + { name => "FPA.ENV.HUMID", type => "constant", flag => "-env_humidity", dtype => "float" }, # external humidity + { name => "FPA.ENV.WIND", type => "constant", flag => "-env_wind_speed", dtype => "float" }, # external wind speed + { name => "FPA.ENV.DIR", type => "constant", flag => "-env_wind_dir", dtype => "float" }, # external wind direction + + { name => "FPA.TELTEMP.M1", type => "constant", flag => "-teltemp_m1", dtype => "float" }, # Primary mirror temps (C) + { name => "FPA.TELTEMP.M1CELL", type => "constant", flag => "-teltemp_m1cell", dtype => "float" }, # Primary mirror support temps (C) + { name => "FPA.TELTEMP.M2", type => "constant", flag => "-teltemp_m2", dtype => "float" }, # Secondary mirror temps (C + { name => "FPA.TELTEMP.SPIDER", type => "constant", flag => "-teltemp_spider", dtype => "float" }, # Spider temperatures (C) + { name => "FPA.TELTEMP.TRUSS", type => "constant", flag => "-teltemp_truss", dtype => "float" }, # Mid truss temperatures (C + { name => "FPA.TELTEMP.EXTRA", type => "constant", flag => "-teltemp_extra", dtype => "float" }, # Miscellaneous temperatures (C) + + { name => "FPA.PON.TIME", type => "constant", flag => "-pon_time", dtype => "float" }, # time since last power on + { name => "CHIP.TEMP", type => "mean", flag => "-ccd_temp", dtype => "float" }, # CCD temperature + { name => "CELL.EXPOSURE", type => "mean", flag => "-exp_time", dtype => "float" }, # Exposure time + { name => "SAT_PIXEL_FRAC", type => "mean", flag => "-sat_pixel_frac", dtype => "float" }, # fraction of saturated pixels + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_MEDIAN", type => "stdev", flag => "-bg_mean_stdev", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $regtool = can_run('regtool') + or (warn "Can't find regtool" and $missing_tools = 1); +my $ppStats = can_run('ppStats') + or (warn "Can't find ppStats" and $missing_tools = 1); + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + +# setup cache interface +# XXX why is this being cached? +my $c = Cache::File->new( + cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)), + default_expires => '7200 sec', +); + +my $now_time = localtime(); +printf STDERR "\nstarting ppStats: %s\n", $now_time; + +# Run ppStats on the input file +{ + # extract the data from the image header; -level is used to get FILE.LEVEL and CLASS.ID + my $command = "$ppStats $uri -recipe PPSTATS $RECIPE -level"; # Command to run ppStats + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + cache_run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $error_code); + } + + # Parse the output + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary? + unless ($metadata) { + &my_die ("Unable to parse metadata config doc", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_PROG_ERROR); + } + + # extract the stats from the metadata + unless ($stats->parse($metadata)) { + &my_die ("Unable to find all values", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); + } +} + +$now_time = localtime(); +printf STDERR "\ndone with ppStats: %s\n", $now_time; + +# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type +if (uc($stats->value_for_flag ("-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-inst")) eq "NULL") { &my_die ("inst not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-class_id")) eq "NULL") { &my_die ("class_id not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } +if (uc($stats->value_for_flag ("-exp_type")) eq "NULL") { &my_die ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } + +my $command = "$regtool -addprocessedimfile"; +$command .= " -exp_id $exp_id"; +$command .= " -exp_name $tmp_exp_name"; # keep the supplied exp_name (could be derived from the file) +$command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID +$command .= " -uri $uri "; +$command .= " -dbname $dbname" if defined $dbname; +$command .= $stats->cmdflags(); + +$now_time = localtime(); +printf STDERR "\nrunning regtool update: %s\n", $now_time; + +# Push the results into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn ("Unable to perform regtool -addprocessedimfile: $error_code"); + exit($error_code); + } +} else { + print "skipping command: $command\n"; +} + +$now_time = localtime(); +printf STDERR "\ndone with regtool update: %s\n", $now_time; + +sub cache_run +{ + my %p = @_; + + my $cmd_output = $c->get($p{command}) if $cache; + if (defined $cmd_output) { + return @{thaw $cmd_output}; + } else { + my @output = run(%p); + $c->set($p{command}, freeze \@output) if $cache; + return @output; + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $exp_id = shift; + my $exp_name = shift; + my $tmp_class_id = shift; + my $uri = shift; + my $exit_code = shift; + + # for failed imfiles, we insert UNKNOWN for inst, telescope, class_id + + carp($msg); + if ($exp_id && $tmp_class_id and not $no_update) { + my $command = "$regtool -addprocessedimfile"; + $command .= " -exp_id $exp_id"; + $command .= " -exp_name $exp_name"; + $command .= " -tmp_class_id $tmp_class_id"; + $command .= " -uri $uri "; + $command .= " -telescope UNKNOWN"; + $command .= " -inst UNKNOWN"; + $command .= " -class_id $tmp_class_id"; + $command .= " -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +# Pau. + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/stack_skycell.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/stack_skycell.pl (revision 16175) @@ -0,0 +1,262 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + caturi + ); +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($stack_id, $dbname, $workdir, $no_update, $no_op, $save_temps); +GetOptions( + 'stack_id|d=s' => \$stack_id, # Stack identifier + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --stack_id", + -exitval => 3, + ) unless defined $stack_id; + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC STACKTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $ppStack = can_run('ppStack') or (warn "Can't find ppStack" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of components for stacking +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$stacktool -inputskyfile -stack_id $stack_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -inputskyfile: $error_code", $stack_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $stack_id, $PS_EXIT_PROG_ERROR); +} + +&my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless + scalar @$files >= 2; + +# Parse the list of input files to get the tesselation, skycell identifiers and camera +my $skycell_id; # Skycell identifier +my $tess_id; # Tesselation identifier +my $camera; # Camera +foreach my $file (@$files) { + # skip warps which are speicified as 'ignored' + if ($file->{ignored}) { next; } + if (defined $tess_id) { + &my_die("Tesselation identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{tess_id} eq $tess_id; + } else { + $tess_id = $file->{tess_id}; + } + if (defined $skycell_id) { + &my_die("Skycell identifiers don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless + $file->{skycell_id} eq $skycell_id; + } else { + $skycell_id = $file->{skycell_id}; + } + if (defined $camera) { + &my_die("Cameras don't match", $stack_id, $PS_EXIT_SYS_ERROR) unless $file->{camera} eq $camera; + } else { + $camera = $file->{camera}; + } +} + +&my_die("Can't find camera", $stack_id, $PS_EXIT_SYS_ERROR) unless $camera; +$ipprc->define_camera($camera); + +# Generate MDC file with the inputs +my ($listFile, $listName) = tempfile( "$tess_id.$skycell_id.stk$stack_id.list.XXXX", + UNLINK => !$save_temps ); +my $num = 0; +# my $inputSources; # Sources to use as stamps +foreach my $file (@$files) { + if ($file->{ignored}) { next; } + + print $listFile "INPUT$num\tMETADATA\n"; + $num++; + + my $image = $file->{uri}; # Image name + my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name + my $weight = $ipprc->filename( "PSWARP.OUTPUT.WEIGHT", $file->{path_base} ); # Weight name + + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); + &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); + + print $listFile "\tIMAGE\tSTR\t" . $image . "\n"; + print $listFile "\tMASK\tSTR\t" . $mask . "\n"; + print $listFile "\tWEIGHT\tSTR\t" . $weight . "\n"; + + ### XXX NEED TO UPDATE THESE appropriately + print $listFile "\tSEEING\tF32\t" . 1.0 . "\n"; + print $listFile "\tWEIGHTING\tF32\t" . 1.0 . "\n"; + print $listFile "\tSCALE\tF32\t" . 1.0 . "\n"; + + print $listFile "END\n\n"; + + # XXX details about supplying sources, etc, need to be specified in the libraries + #unless (defined $inputSources) { + #$inputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); + #&my_die("Source file $inputSources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $inputSources ); + #} + +} + +# Get the output filenames +$workdir = caturi( $workdir, "tess_" . $tess_id, "sky_" . $skycell_id ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$tess_id.$skycell_id.stk$stack_id", $workdir, $$files[0]->{uri} ); +my $outputName = $ipprc->filename("PPSTACK.OUTPUT", $outputRoot); +my $outputMask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $outputRoot); +my $outputWeight = $ipprc->filename("PPSTACK.OUTPUT.WEIGHT", $outputRoot); +#my $outputSources = $ipprc->filename("PSPHOT.OUTPUT", $outputRoot, "none"); +#my $bin1Name = $ipprc->filename("PPSTACK.BIN1", $outputRoot); +#my $bin2Name = $ipprc->filename("PPSTACK.BIN2", $outputRoot); +my $outputStats = $outputRoot . '.stats'; + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".log"; # Log messages + +# Perform stacking +unless ($no_op) { + my $command = "$ppStack $listName $outputRoot"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + # $command .= " -sources $inputSources"; XXX : see above re recipes + # $command .= " -target 5.0"; + # $command .= " -photometry"; + $command .= " -tracedest $traceDest -log $logDest"; + + # print "\n\nWARNING: Convolving to 1 arcsec FWHM seeing!\n\n"; + print "\n\nWARNING: NOT Convolving!\n\n"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStack: $error_code", $stack_id, $error_code); + } + &my_die("Couldn't find expected output file: $outputName", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + &my_die("Couldn't find expected output file: $outputMask", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); +# &my_die("Couldn't find expected output file: $outputSources", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); +# &my_die("Couldn't find expected output file: $bin1Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name); +# &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); + &my_die("Couldn't find expected output file: $outputStats", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + + # Get the statistics on the stacked image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $stack_id, $PS_EXIT_SYS_ERROR); + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $metadata = $mdcParser->parse(join "", @contents) or + &my_die("Unable to parse metadata config doc", $stack_id, $PS_EXIT_PROG_ERROR); + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $stack_id, $PS_EXIT_PROG_ERROR); +} + +unless ($no_update) { + + # Add the stack result + { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -uri $outputName -path_base $outputRoot"; + $command .= $stats->cmdflags(); + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -addsumskyfile: $error_code", $stack_id, $error_code); + } + } + + # Register the run as completed + { + my $command = "$stacktool -updaterun -stack_id $stack_id -state stop"; # Command to run stacktool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform stacktool -updaterun: $error_code\n"); + exit($error_code); + } + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $stack_id = shift; # Stack identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($stack_id and not $no_update) { + my $command = "$stacktool -addsumskyfile -stack_id $stack_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/warp_overlap.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/warp_overlap.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/warp_overlap.pl (revision 16175) @@ -0,0 +1,329 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); +use File::Temp qw( tempfile ); +use File::Spec; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + file_scheme + ); +use PS::IPP::Operations qw( generate_skycells ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $camera, $dbname, $workdir, $no_update, $no_op); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --camera --workdir", + -exitval => 3, +) unless defined $warp_id + and defined $camera + and define $workdir; + +$ipprc->define_camera($camera); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get list of component imfiles for exposure +my $imfiles; +{ + my $command = "$warptool -imfile -warp_id $warp_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -imfile: $error_code", $warp_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +my $astromAccept; # Accept the astrometry unconditionally? +my $astromDepth; # File level of the astrometry source (SPLIT or MEF)? +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + $astromAccept = metadataLookupBool($metadata, 'ASTROM.ACCEPT'); + $astromDepth = metadataLookupStr($metadata, 'ASTROM.DEPTH'); +} + +# Determine the imfile/skycell overlaps +my @overlaps = (); + +unless ($no_op) { + # Calculate the overlaps between imfiles and skycells + + my $tessellations = $ipprc->tessellations(); # Hash of defined tessellations + foreach my $tess_id ( keys %$tessellations ) { + ### Because DVO doesn't use psModules, it doesn't understand Nebulous --- check + my $scheme = file_scheme($$tessellations{$tess_id}); # The scheme, e.g., file, path, neb + if (defined $scheme and lc($scheme) eq 'neb') { + &my_die("Tessellation $tess_id refers to a Nebulous path: $$tessellations{$tess_id}", $warp_id, $PS_EXIT_CONFIG_ERROR); + } + + my $tess_dir = $ipprc->convert_filename_absolute( $$tessellations{$tess_id} ); # Catdir for DVO + print STDERR "tessellation: $tess_id, $tess_dir\n"; + + my %unique_skycells = (); # Identified skycells (all unique by virtue of hash property) + + # XXX this is a bit too hard wired: the concept is that astrometry comes from the MOSAIC vs CHIP output + if ($astromSource eq 'PSASTRO.OUTPUT.MEF') { + # We have a MEF astrometry file from psastro + my $imfile = $imfiles->[0]; + my $camRoot = $imfile->{cam_path_base}; + my $astromFile = $ipprc->filename($astromSource, $camRoot); # Astrometry file + + my @matchlist = get_overlaps($astromFile, $tess_dir, $astromAccept); # List of overlaps + if (! @matchlist) { + &my_die("Unable to perform dvoImageOverlaps: missing astrometry", $warp_id, $PS_EXIT_DATA_ERROR); + } + # Match each of the imfiles to this list + foreach my $imfile (@$imfiles) { + extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_id, \@overlaps, \%unique_skycells); + } + } else { + # We have per-imfile astrometry + foreach my $imfile (@$imfiles) { + my $astromFile; + if ($astromSource eq 'PSASTRO.OUTPUT') { + my $chipRoot = $imfile->{chip_path_base}; + my $classID = $imfile->{class_id}; + $astromFile = $ipprc->filename($astromSource, $chipRoot, $classID); # Astrometry file + } else { + $astromFile = $imfile->{chip_uri}; # Astrometry file + } + my @matchlist = get_overlaps($astromFile, $tess_dir, $astromAccept); # List of overlaps + if (! @matchlist) { + print "skipping $astromFile\n"; + } + + extract_overlaps(\@matchlist, $imfile, $astromFile, $tess_id, \@overlaps, \%unique_skycells); + } + } + + # Extract the skycells to images, used as warp templates. + my @skycells = keys %unique_skycells; + generate_skycells($ipprc, $tess_id, \@skycells, $workdir) or + &my_die("Unable to generate skycells for tessellation $tess_id", $warp_id, $PS_EXIT_SYS_ERROR); + } +} else { + # create an overlap with an entry for each skycell:imfile match + foreach my $imfile (@$imfiles) { + my %overlap = (); + $overlap{skycell_id} = 'default'; + $overlap{tess_id} = 'default'; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @overlaps, \%overlap; + } +} + +# XXX this file needs some additional error checking: if no overlaps are found, we +# keep running this step over and over (a successful warptool -addoverlap prevents +# successive warptime -imfile from running. + +# Generate a MDC file with the overlaps +my ($overlapFile, $overlapName) = tempfile( 'overlaps.wrp' . $warp_id . '.mdc.XXXX', UNLINK => 1 ); +print $overlapFile "warpSkyCellMap MULTI\n\n"; +foreach my $overlap (@overlaps) { + print $overlapFile "warpSkyCellMap METADATA\n"; + print $overlapFile " warp_id S32 $warp_id\n"; + print $overlapFile " skycell_id STR $overlap->{skycell_id}\n"; + print $overlapFile " tess_id STR $overlap->{tess_id}\n"; + print $overlapFile " cam_id S32 $overlap->{cam_id}\n"; + print $overlapFile " class_id STR $overlap->{class_id}\n"; + print $overlapFile " fault S16 $overlap->{fault}\n"; + print $overlapFile "END\n\n"; +} +close $overlapFile; + +system "cat $overlapName"; + +# Add the processed file to the database +unless ($no_update) { + my $command = "$warptool -addoverlap -mapfile $overlapName"; # Command to run warptool + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addoverlap: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + + # No failure option on warptool for this mode, so just die. + exit $exit_code; +} + +# Run dvoImageOverlaps to get the overlaps; return the output +sub get_overlaps +{ + my $filename = shift; # Filename on which to run dvoImageOverlaps + my $tess_dir = shift; # Tessellation directory + my $accept = shift; # Do we use the -accept-astrom flag? + + my $command = "$dvoImageOverlaps -D CATDIR $tess_dir " . $ipprc->file_resolve($filename); + $command .= ' -accept-astrom' if $accept; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + if (!$success) { + print "missing astrometry for $filename\n"; + return 0; + } + return split ('\n', (join "", @$stdout_buf)); +} + +# Extract a list of overlaps for an imfile +# +# The command "dvoImageOverlaps -accept-astrom -D CATDIR tessellation (megacam)" returns: +# 729534pa.cmf[ccd00.hdr] : skycell.051.fits +# PSASTRO.OUTPUT[CMF.HEAD] : SKYCELL +# [CMF.HEAD] is optionally used for MEF files (SIMPLE or MOSAIC) +sub extract_overlaps +{ + my $matches = shift; # Reference to list of skycells from dvoImageOverlaps + my $imfile = shift; # Imfile information + my $filename = shift; # Filename used with dvoImageOverlaps + my $tess_id = shift; # Tessellation identifier + my $overlaps = shift; # Reference to list of overlaps + my $unique_skycells = shift; # Reference to hash of found skycells + + # Get rid of the path + my @dirlist = File::Spec->splitdir( $filename ); # The elements of the full path + $filename = pop @dirlist; + + # Work out how to identify this imfile in the output + my $fileLevel = $imfile->{filelevel}; + my $entry; # How to identify this imfile in the dvoImageOverlaps output + if ((lc($fileLevel) eq "chip") && (lc($astromDepth) eq "MEF")) { + my $class_id = $imfile->{class_id}; + my $chipRoot = $ipprc->file_resolve( $imfile->{chip_path_base} ); + my $extname = $ipprc->extname_rule("CMF.HEAD", $class_id); # MEF psastro output + + + $entry = $filename . '\[' . $extname . '\]'; + print STDERR "entry: $entry, class: $class_id, extname: $extname, chiproot: $chipRoot\n"; + } else { + $entry = $filename; + print STDERR "entry: $entry\n"; + } + + my @skycells = &select_skycells($entry, @$matches); # Matching skycells + my $Nskycells = @skycells; + printf STDERR "Nskycells: $Nskycells\n"; + foreach my $skycell (@skycells) { + my %overlap = (); # Overlap information for warptool + $overlap{skycell_id} = $skycell; + $overlap{tess_id} = $tess_id; + $overlap{cam_id} = $imfile->{cam_id}; + $overlap{class_id} = $imfile->{class_id}; + $overlap{fault} = $imfile->{fault}; + push @$overlaps, \%overlap; + + printf STDERR "overlap: %s : %s , %s\n", $skycell, $imfile->{cam_id}, $imfile->{class_id}; + + $unique_skycells->{$skycell} = 1; + } + + return; +} + +# Find skycells in the list that come from a particular entry +sub select_skycells +{ + my $entry = shift; # File+Ext to search for + my @list = @_; # List of "File+Ext : skycell" + + my @skycells = (); + my %unique = (); # Ensure we only return unique skycells for this entry + + foreach my $line (@list) { + if ($line =~ m|$entry|) { + my ($skycell) = $line =~ m|$entry\S*\s+:\s+(\S+)|; + if (not defined $unique{$skycell}) { + push @skycells, $skycell; + $unique{$skycell} = 1; + } + } + } + return @skycells; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branch_20080121/ippScripts/scripts/warp_skycell.pl (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/scripts/warp_skycell.pl (revision 16175) @@ -0,0 +1,262 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); +use PS::IPP::Operations qw( skycell_file ); + +my $ipprc = PS::IPP::Config->new(); # IPP configuration + +my ($warp_id, $skycell_id, $tess_id, $camera, $dbname, $workdir, $no_update, $no_op, $save_temps); +GetOptions( + 'warp_id|i=s' => \$warp_id, # Warp identifier + 'skycell_id|s=s' => \$skycell_id, # Skycell identifier + 'tess_id|s=s' => \$tess_id, # Tesselation identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'workdir|w=s' => \$workdir, # Working directory, for output files + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_id --skycell_id --tess_id --camera", + -exitval => 3, +) unless defined $warp_id + and defined $skycell_id + and defined $tess_id + and defined $camera; + +$ipprc->define_camera($camera); + +my $STATS = + [ + # PPSTATS KEYWORD STATISTIC WARPTOOL FLAG + { name => "ROBUST_MEDIAN", type => "mean", flag => "-bg", dtype => "float" }, + { name => "ROBUST_STDEV", type => "rms", flag => "-bg_stdev", dtype => "float" }, + { name => "GOOD_PIXEL_FRAC", type => "mean", flag => "-good_frac", dtype => "float" }, + ]; +my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Get list of component imfiles for exposure +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $imfiles; +{ + my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -scmap: $error_code", $warp_id, $skycell_id, $tess_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); +} + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +{ + my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); +} + +my $skyFile = skycell_file($tess_id, $skycell_id, $workdir) or + &my_die("Unable to get name of skycell file", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR); +&my_die("Unable to find skycell file: $skyFile", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless + $ipprc->file_exists( $skyFile ); + +$workdir = caturi( $workdir, "tess_" . $tess_id, "sky_" . $skycell_id ) if defined $workdir; +my $outputRoot = $ipprc->file_prepare( "$tess_id.$skycell_id.wrp$warp_id", $workdir, $$imfiles[0]->{uri} ); +my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outputRoot, $skycell_id ); +my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outputRoot, $skycell_id); +my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.WEIGHT", $outputRoot, $skycell_id); +my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outputRoot, $skycell_id); +my $outputPSF = $ipprc->filename("PSPHOT.PSF.SAVE", $outputRoot, $skycell_id); +my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outputRoot, $skycell_id ); +my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outputRoot, $skycell_id ); +my $outputStats = $outputRoot . '.stats'; + +my $traceDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".trace"; # Trace messages +my $logDest = 'file:' . $ipprc->file_resolve($outputRoot) . ".log"; # Log messages + +# Get list of filenames +my ($imageFile, $imageName) = tempfile( "$tess_id.$skycell_id.wrp$warp_id.image.list.XXXX", UNLINK => !$save_temps); +my ($maskFile, $maskName) = tempfile( "$tess_id.$skycell_id.wrp$warp_id.mask.list.XXXX", UNLINK => !$save_temps); +my ($weightFile, $weightName) = tempfile( "$tess_id.$skycell_id.wrp$warp_id.weight.list.XXXX", UNLINK => !$save_temps); +my ($astromFile, $astromName) = tempfile( "$tess_id.$skycell_id.wrp$warp_id.astrom.list.XXXX", UNLINK => !$save_temps); + +# XXXX this is inconsistent: what is the goal of inputAstrom? +my $inputAstrom = 0; +foreach my $imfile (@$imfiles) { + my $image = $imfile->{uri}; # Image name + my $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + my $weight = $ipprc->filename("PPIMAGE.CHIP.WEIGHT", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name + + &my_die("Couldn't find input file: $image", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image); + &my_die("Couldn't find input file: $mask", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($mask); + + my $astrom; # Astrometry file + if ($astromSource eq 'PSASTRO.OUTPUT.MEF') { + $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base}); + &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($astrom); + $inputAstrom = 1; + } + if ($astromSource eq 'PSASTRO.OUTPUT') { + my $chipRoot = $imfile->{chip_path_base}; + my $classID = $imfile->{class_id}; + $astrom = $ipprc->filename($astromSource, $chipRoot, $classID); # Astrometry file + $inputAstrom = 1; + } + + if ($inputAstrom) { + print $imageFile "$image\n"; + print $maskFile "$mask\n"; + print $weightFile "$weight\n"; + print $astromFile "$astrom\n"; + } +} +close $imageFile; +close $maskFile; +close $weightFile; +close $astromFile; + +# Run pswarp +my $accept = 1; # Accept the skycell? +unless ($no_op) { + my $command = "$pswarp"; + $command .= " -list $imageName"; + $command .= " -masklist $maskName"; + $command .= " -weightlist $weightName"; + $command .= " -astromlist $astromName" if $inputAstrom; + $command .= " $outputRoot $skyFile"; + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS WARPSTATS"; + # $command .= " -psf"; # Turn on PSF determination + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $warp_id, $skycell_id, $tess_id, $error_code); + } + + # Check first for the stats file, and if the ACCEPT flag is set. + &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStats); + # Get the statistics on the warped image + my $statsFile; # File handle + open $statsFile, $ipprc->file_resolve($outputStats) or die "Can't open statistics file $outputStats: $!\n"; + my @contents = <$statsFile>; # Contents of file + close $statsFile; + my $contents = join "", @contents; + + my $metadata = $mdcParser->parse($contents) + or &my_die("Unable to parse metadata config", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); + $accept = metadataLookupBool($metadata, "ACCEPT"); + if ($accept) { + $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $warp_id, $skycell_id, $tess_id, $PS_EXIT_PROG_ERROR); + + &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage); + &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight); + &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + # &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); +# &my_die("Couldn't find expected output file: $outputBin1", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1); +# &my_die("Couldn't find expected output file: $outputBin2", $warp_id, $skycell_id, $tess_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2); + } +} + +unless ($no_update) { + my $command = "$warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id"; + $command .= " -ignore" if not $accept; # Completed succesfully, but can't produce product + $command .= " -uri $outputImage -path_base $outputRoot" if $accept; + $command .= $stats->cmdflags() if $accept; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 1); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_id = shift; # Warp identifier + my $skycell_id = shift; # Skycell identifier + my $tess_id = shift; # Tesselation identifier + my $exit_code = shift; # Exit code to add + + warn($msg); + if ($warp_id and $skycell_id and $tess_id and not $no_update) { + my $command = "$warptool -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id -code $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => 1); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /branches/eam_branch_20080121/ippScripts/t/00_distribution.t =================================================================== --- /branches/eam_branch_20080121/ippScripts/t/00_distribution.t (revision 16175) +++ /branches/eam_branch_20080121/ippScripts/t/00_distribution.t (revision 16175) @@ -0,0 +1,25 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005 Joshua Hoblitt +# +# $Id: 00_distribution.t,v 1.1 2007-01-30 01:52:16 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use lib qw( ./lib ./t ); + +use Test::More; + +# example taken from Test::Distribution Pod + +BEGIN { + eval { + require Test::Distribution; + }; + if($@) { + plan skip_all => 'Test::Distribution not installed'; + } else { + import Test::Distribution; # not => qw( podcover ); + } +} Index: /branches/eam_branch_20080121/ippTasks/.cvsignore =================================================================== --- /branches/eam_branch_20080121/ippTasks/.cvsignore (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/.cvsignore (revision 16175) @@ -0,0 +1,9 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +config.log +config.status +configure +install-sh +missing Index: /branches/eam_branch_20080121/ippTasks/Makefile.am =================================================================== --- /branches/eam_branch_20080121/ippTasks/Makefile.am (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/Makefile.am (revision 16175) @@ -0,0 +1,33 @@ +task_files = \ + detrend.mkruns.pro \ + detrend.norm.pro \ + detrend.process.pro \ + detrend.reject.pro \ + detrend.resid.pro \ + detrend.stack.pro \ + mkdetrend.pro \ + pantasks.pro \ + register.pro \ + chip.pro \ + camera.pro \ + warp.pro \ + magic.pro \ + diff.pro \ + stack.pro \ + summit.copy.pro \ + testscript.pro + +detrend_files = \ + simtest.det \ + isp.det + +pantasksdir = $(datadir)/pantasks +ipptasksdir = $(pantasksdir)/modules +ipptasks_DATA = $(task_files) $(detrend_files) + +install-data-hook: + chmod 0755 $(panstasks) $(ipptasksdir) + +EXTRA_DIST = $(task_files) $(detrend_files) + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/eam_branch_20080121/ippTasks/autogen.sh =================================================================== --- /branches/eam_branch_20080121/ippTasks/autogen.sh (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/autogen.sh (revision 16175) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ippTasks +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +#($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $LIBTOOlIZE installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" +# DIE=1 +#} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +#($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { +# echo +# echo "You must have $AUTOHEADER installed to compile $PROJECT." +# echo "Download the appropriate package for your distribution," +# echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" +# DIE=1 +#} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +#$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +#$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branch_20080121/ippTasks/calibration.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/calibration.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/calibration.pro (revision 16175) @@ -0,0 +1,181 @@ +## this file contains the tasks for running the calibration stage +## these tasks use the books XXX + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/calibration +exec mkdir -p $LOGSUBDIR + +book init calBook + +macro calibration.reset + book init calBook +end + +macro calibration.status + book listbook calBook +end + +macro calibration.on + task calibration.init + active true + end +end + +macro calibration.off + task calibration.init + active false + end +end + +# these variables will cycle through the known database names +$calInit_DB = 0 + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.init + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 60 + trange Hourly@00:00 Hourly@00:10 + # every hour on the hour + + if ($DB:n == 0) + npending $DB:n + else + npending 1 + end + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + ## XXX have caltool define the regions, or have the task.exit below do it? + + # generate queues for all of the known (and active) DVO databases + task.exec + # get the current ST: + $RAs = 15*($ST - 1) + $RAe = 15*($ST - 2) + $DECs = -90.0 + $DECe = +90.0 + + $REGION = "$RAs,$RAe:$DECs,$DECe" + + if ($DB:n == 0) + option DEFAULT + command caltool -databases -active true + else + # save the DB name for the exit tasks + option $DB:$calInit_DB + option $REGION + command caltool -databases -dbname $DB:$calInit_DB -region $REGION + $calInit_DB ++ + if ($calInit_DB >= $DB:n) set calInit_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout calBook -key dbname:dvo_id -uniq -setword dbname $options:0 -setword region $options:1 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook calBook + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup calBook + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# create new calibration entries for the currently known DVO databases +# run this multiple times once an hour - on pass for each db +task calibration.resort + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 60 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/calibration.log + + task.exec + book npages calBook -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in calBook + # the sequencing in this task set is by the pantasksState (see notes.txt) + book getpage calBook 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword calBook $pageName pantaskState RUN + + # XXX probably need to set the output / log based on WORKDIR... + book getword calBook $pageName catdir -var CATDIR + book getword calBook $pageName region -var REGION + book getword calBook $pageName dbname -var DBNAME + + # specify choice of remote host + if ($parallel) + host anyhost + else + host local + end + + ## XXX generate output log based on filerule? + ## XXX put this in a function? + + # save the pageName for future reference below + options $pageName + + # XXX do we modify relphot to loop over the filters? + $run = calibrate_dvo.pl --catdir $CATDIR --region $REGION + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit calBook $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword calBook $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/camera.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/camera.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/camera.pro (revision 16175) @@ -0,0 +1,183 @@ +## this file contains the tasks for running the phase 3 stage +## these tasks use the book camPendingExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/camera +exec mkdir -p $LOGSUBDIR + +book init camPendingExp + +macro camera.status + book listbook camPendingExp +end + +macro camera.reset + book init camPendingExp +end + +macro camera.on + task camera.exp.load + active true + end + task camera.exp.run + active true + end +end + +macro camera.off + task camera.exp.load + active false + end + task camera.exp.run + active false + end +end + +# this variable will cycle through the known database names +$camera_DB = 0 + +# select images ready for camera analysis +# new entries are added to camPendingExp +# skip already-present entries +task camera.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout $LOGSUBDIR/camera.exp.log + stderr $LOGSUBDIR/camera.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command camtool -pendingexp -limit 20 + # echo command camtool -pendingexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$camera_DB + command camtool -pendingexp -limit 20 -dbname $DB:$camera_DB + # echo command camtool -pendingexp -limit 20 -dbname $DB:$camera_DB + $camera_DB ++ + if ($camera_DB >= $DB:n) set camera_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout camPendingExp -key cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook camPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup camPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the cameraexposure script on pending images +task camera.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 10 + + task.exec + book npages camPendingExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in camPendingExp (pantaskState == INIT) + book getpage camPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword camPendingExp $pageName pantaskState RUN + book getword camPendingExp $pageName camera -var CAMERA + book getword camPendingExp $pageName exp_tag -var EXP_TAG + book getword camPendingExp $pageName cam_id -var CAM_ID + book getword camPendingExp $pageName workdir -var WORKDIR + book getword camPendingExp $pageName dvodb -var DVODB + book getword camPendingExp $pageName dbname -var DBNAME + book getword camPendingExp $pageName reduction -var REDUCTION + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # output log from filerule + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s" $outpath $EXP_TAG + sprintf outroot "%s/%s.cm.%s" $outpath $EXP_TAG $CAM_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = camera_exp.pl --exp_tag $EXP_TAG --cam_id $CAM_ID --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit camPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword camPendingExp $options:0 pantaskState TIMEOUT + end +end + +## XXX add a global path to output files Index: /branches/eam_branch_20080121/ippTasks/chip.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/chip.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/chip.pro (revision 16175) @@ -0,0 +1,178 @@ +## this file contains the tasks for running the phase 0 stage +## these tasks use the book chipPendingImfile + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/chip +exec mkdir -p $LOGSUBDIR + +book init chipPendingImfile + +macro chip.status + book listbook chipPendingImfile +end + +macro chip.reset + book init chipPendingImfile +end + +macro chip.on + task chip.imfile.load + active true + end + task chip.imfile.run + active true + end +end + +macro chip.off + task chip.imfile.load + active false + end + task chip.imfile.run + active false + end +end + +# this variable will cycle through the known database names +$chip_DB = 0 + +# select images ready for chip analysis +# new entries are added to chipPendingImfile +# skip already-present entries +task chip.imfile.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/chip.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command chiptool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$chip_DB + command chiptool -pendingimfile -limit 20 -dbname $DB:$chip_DB + $chip_DB ++ + if ($chip_DB >= $DB:n) set chip_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipPendingImfile -key chip_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the chip_imfile.pl script on pending images +task chip.imfile.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages chipPendingImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in chipPendingImfile (pantaskState == INIT) + book getpage chipPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipPendingImfile $pageName pantaskState RUN + book getword chipPendingImfile $pageName camera -var CAMERA + book getword chipPendingImfile $pageName exp_id -var EXP_ID + book getword chipPendingImfile $pageName exp_tag -var EXP_TAG + book getword chipPendingImfile $pageName chip_id -var CHIP_ID + book getword chipPendingImfile $pageName workdir -var WORKDIR + book getword chipPendingImfile $pageName class_id -var CLASS_ID + book getword chipPendingImfile $pageName uri -var URI + book getword chipPendingImfile $pageName dbname -var DBNAME + book getword chipPendingImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera $class_id + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s" $outpath $EXP_TAG + sprintf outroot "%s/%s.ch.%s" $outpath $EXP_TAG $CHIP_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = chip_imfile.pl --exp_id $EXP_ID --chip_id $CHIP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --uri $URI --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit chipPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipPendingImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/chiphosts.config =================================================================== --- /branches/eam_branch_20080121/ippTasks/chiphosts.config (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/chiphosts.config (revision 16175) @@ -0,0 +1,98 @@ + +chiphosts MULTI + +# these camera names must match those listed in ipprc.config +chiphosts METADATA + camera STR MEGACAM + ccd00 STR po00 + ccd01 STR po01 + ccd02 STR po02 + ccd03 STR po03 + ccd04 STR po04 + ccd05 STR po05 + ccd06 STR po06 + ccd07 STR po07 + ccd08 STR po08 + ccd09 STR po09 +END + +chiphosts METADATA + camera STR CFH12K + ccd00 STR po10 + ccd01 STR po11 + ccd02 STR po12 +END + +$gpcHost00 = ipp005 +$gpcHost01 = ipp008 +$gpcHost02 = ipp019 +$gpcHost03 = ipp020 +$gpcHost04 = ipp021 + +chiphosts METADATA + camera GPC1 + XY01 STR gpcHost00 + XY02 STR gpcHost00 + XY03 STR gpcHost00 + XY04 STR gpcHost00 + XY05 STR gpcHost00 + XY06 STR gpcHost00 + XY10 STR gpcHost00 + XY11 STR gpcHost00 + XY12 STR gpcHost00 + XY13 STR gpcHost00 + XY14 STR gpcHost00 + XY15 STR gpcHost00 + + XY16 STR gpcHost01 + XY17 STR gpcHost01 + XY20 STR gpcHost01 + XY21 STR gpcHost01 + XY22 STR gpcHost01 + XY23 STR gpcHost01 + XY24 STR gpcHost01 + XY25 STR gpcHost01 + XY26 STR gpcHost01 + XY27 STR gpcHost01 + XY30 STR gpcHost01 + XY31 STR gpcHost01 + + XY32 STR gpcHost02 + XY33 STR gpcHost02 + XY34 STR gpcHost02 + XY35 STR gpcHost02 + XY36 STR gpcHost02 + XY37 STR gpcHost02 + XY40 STR gpcHost02 + XY41 STR gpcHost02 + XY42 STR gpcHost02 + XY43 STR gpcHost02 + XY44 STR gpcHost02 + XY45 STR gpcHost02 + + XY46 STR gpcHost03 + XY47 STR gpcHost03 + XY50 STR gpcHost03 + XY51 STR gpcHost03 + XY52 STR gpcHost03 + XY53 STR gpcHost03 + XY54 STR gpcHost03 + XY55 STR gpcHost03 + XY56 STR gpcHost03 + XY57 STR gpcHost03 + XY60 STR gpcHost03 + XY61 STR gpcHost03 + + XY62 STR gpcHost04 + XY63 STR gpcHost04 + XY64 STR gpcHost04 + XY65 STR gpcHost04 + XY66 STR gpcHost04 + XY67 STR gpcHost04 + XY71 STR gpcHost04 + XY72 STR gpcHost04 + XY73 STR gpcHost04 + XY74 STR gpcHost04 + XY75 STR gpcHost04 + XY76 STR gpcHost04 +END Index: /branches/eam_branch_20080121/ippTasks/configure.ac =================================================================== --- /branches/eam_branch_20080121/ippTasks/configure.ac (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/configure.ac (revision 16175) @@ -0,0 +1,14 @@ +AC_PREREQ(2.59) + +AC_INIT([ippTasks], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([pantasks.pro]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_MAINTAINER_MODE + +AC_PROG_INSTALL + +AC_CONFIG_FILES([ + Makefile +]) +AC_OUTPUT Index: /branches/eam_branch_20080121/ippTasks/detrend.correct.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.correct.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.correct.pro (revision 16175) @@ -0,0 +1,323 @@ +## this file contains the tasks for running the detrend correction stage +## these tasks use the books detPendingCorrectedImfile and detPendingCorrectedExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingCorrectedImfile +book init detPendingCorrectedExp + +macro detcorr.reset + book init detPendingCorrectedImfile + book init detPendingCorrectedExp +end + +macro detcorr.status + echo detPendingCorrectedImfile + book listbook detPendingCorrectedImfile + echo detPendingCorrectedExp + book listbook detPendingCorrectedExp +end + +macro detcorr.on + task detrend.correct.load + active true + end + task detrend.correct.run + active true + end + task detrend.correctexp.load + active true + end + task detrend.correctexp.run + active true + end +end + +macro detcorr.off + task detrend.correct.load + active false + end + task detrend.correct.run + active false + end + task detrend.correctexp.load + active false + end + task detrend.correctexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingCorrectedImfile_DB = 0 +$detPendingCorrectedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingCorrectedImfile +# compare the new list with the ones already selected +task detrend.correct.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detcorr.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedImfile_DB + command dettool -tocorrectedimfile -limit 20 -dbname $DB:$detPendingCorrectedImfile_DB + $detPendingCorrectedImfile_DB ++ + if ($detPendingCorrectedImfile_DB >= $DB:n) set detPendingCorrectedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_correct_imfile.pl on pending images +task detrend.correct.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingCorrectedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingCorrectedImfile + book getpage detPendingCorrectedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedImfile $pageName pantaskState RUN + book getword detPendingCorrectedImfile $pageName det_id -var DET_ID + book getword detPendingCorrectedImfile $pageName exp_id -var EXP_ID + book getword detPendingCorrectedImfile $pageName class_id -var CLASS_ID + book getword detPendingCorrectedImfile $pageName exp_tag -var EXP_TAG XXX where does this come from + book getword detPendingCorrectedImfile $pageName det_type -var DET_TYPE XXX where does this come from + book getword detPendingCorrectedImfile $pageName camera -var CAMERA XXX where does this come from + book getword detPendingCorrectedImfile $pageName uri -var URI + book getword detPendingCorrectedImfile $pageName corr_id -var CORR_ID + book getword detPendingCorrectedImfile $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedImfile $pageName workdir -var WORKDIR + book getword detPendingCorrectedImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera $class_id + + ## output log filename + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detcorr.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_correct_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingCorrectedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingCorrectedExp +# compare the new list with the ones already selected +task detrend.correctexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detcorr.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tocorrectedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingCorrectedExp_DB + command dettool -tocorrectedexp -limit 20 -dbname $DB:$detPendingCorrectedExp_DB + $detPendingCorrectedExp_DB ++ + if ($detPendingCorrectedExp_DB >= $DB:n) set detPendingCorrectedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingCorrectedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingCorrectedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingCorrectedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.correctexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingCorrectedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingCorrectedExp + book getpage detPendingCorrectedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingCorrectedExp $pageName pantaskState RUN + book getword detPendingCorrectedExp $pageName det_id -var DET_ID + book getword detPendingCorrectedExp $pageName exp_id -var EXP_ID + book getword detPendingCorrectedExp $pageName det_type -var DET_TYPE + book getword detPendingCorrectedExp $pageName camera -var CAMERA + book getword detPendingCorrectedExp $pageName exp_tag -var EXP_TAG + book getword detPendingCorrectedExp $pageName corr_id -var CORR_ID + book getword detPendingCorrectedExp $pageName corr_type -var CORR_TYPE + book getword detPendingCorrectedExp $pageName workdir -var WORKDIR + book getword detPendingCorrectedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # XXX push this into the filerules? + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detcorr.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_correct_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --corr_id $CORR_ID --corr_type $CORR_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingCorrectedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingCorrectedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/detrend.mkruns.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.mkruns.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.mkruns.pro (revision 16175) @@ -0,0 +1,97 @@ +## XXX this script needs to be updated to use books, not queues +## XXX this script needs to be reconsidered... + +## This script defines tasks which regularly initiate the detrend +## runs. Each detrend type is given its own task, which are run once +## (or twice) per day to build the relevant detrend data + +## when one of these tasks is run, it creates a new detrend run ID in +## the database, and selects the exposures for that run on the basis +## of the specified filters + +# create a new detrend bias-creation run +task mkdetruns.megacam + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.megacam + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new MEGACAM detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.isp + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.isp + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new ISP detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end + +# create a new detrend bias-creation run +task mkdetruns.gpc + # command dettool -select -time NOW -24h -type bias -camera megacam + command mkdetruns.gpc + host local + + periods -poll 1 + periods -exec 60 + periods -timeout 30 + trange 18:00 19:00 -nmax 1 + + # success: a new detrend run was created + task.exit 0 + echo "new GPC detrend runs" + queueprint stdout + end + + # default exit status + task.exit default + echo "unable to define a new detrend run" + end + + # operation timed out? is the database down? + task.exit timeout + # send someone email? + echo "unable to define a new detrend run" + end +end Index: /branches/eam_branch_20080121/ippTasks/detrend.norm.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.norm.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.norm.pro (revision 16175) @@ -0,0 +1,433 @@ +## this file contains the tasks for running the detrend normalization stages +## these tasks use the books detPendingNormStatImfile detPendingNormImfile detPendingNormExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingNormStatImfile +book init detPendingNormImfile +book init detPendingNormExp + +macro detnorm.reset + book init detPendingNormStatImfile + book init detPendingNormImfile + book init detPendingNormExp +end + +macro detnorm.status + book listbook detPendingNormStatImfile + book listbook detPendingNormImfile + book listbook detPendingNormExp +end + +macro detnorm.on + task detrend.norm.load + active true + end + task detrend.norm.run + active true + end + task detrend.normexp.load + active true + end + task detrend.normexp.run + active true + end + task detrend.normstat.load + active true + end + task detrend.normstat.run + active true + end +end + +macro detnorm.off + task detrend.norm.load + active false + end + task detrend.norm.run + active false + end + task detrend.normexp.load + active false + end + task detrend.normexp.run + active false + end + task detrend.normstat.load + active false + end + task detrend.normstat.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingNormStatImfile_DB = 0 +$detPendingNormImfile_DB = 0 +$detPendingNormExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingNormStatImfile +# compare the new list with the ones already selected +task detrend.normstat.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.normstat.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalizedstat -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormStatImfile_DB + command dettool -tonormalizedstat -limit 20 -dbname $DB:$detPendingNormStatImfile_DB + $detPendingNormStatImfile_DB ++ + if ($detPendingNormStatImfile_DB >= $DB:n) set detPendingNormStatImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormStatImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormStatImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normstat.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingNormStatImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingNormStatImfile + book getpage detPendingNormStatImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormStatImfile $pageName pantaskState RUN + book getword detPendingNormStatImfile $pageName det_id -var DET_ID + book getword detPendingNormStatImfile $pageName det_type -var DET_TYPE + book getword detPendingNormStatImfile $pageName iteration -var ITERATION + book getword detPendingNormStatImfile $pageName camera -var CAMERA + book getword detPendingNormStatImfile $pageName workdir -var WORKDIR + book getword detPendingNormStatImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID + $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_norm_calc.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormStatImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormStatImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormImfile +# compare the new list with the ones already selected +task detrend.norm.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.norm.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalize -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormImfile_DB + command dettool -tonormalize -limit 20 -dbname $DB:$detPendingNormImfile_DB + $detPendingNormImfile_DB ++ + if ($detPendingNormImfile_DB >= $DB:n) set detPendingNormImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.norm.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingNormImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingNormImfile + book getpage detPendingNormImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormImfile $pageName pantaskState RUN + book getword detPendingNormImfile $pageName det_type -var DET_TYPE + book getword detPendingNormImfile $pageName camera -var CAMERA + book getword detPendingNormImfile $pageName uri -var URI + book getword detPendingNormImfile $pageName det_id -var DET_ID + book getword detPendingNormImfile $pageName iteration -var ITERATION + book getword detPendingNormImfile $pageName class_id -var CLASS_ID + book getword detPendingNormImfile $pageName norm -var NORM + book getword detPendingNormImfile $pageName workdir -var WORKDIR + book getword detPendingNormImfile $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera $class_id + + # XXX use ipp_filename.pl to lookup output file names + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID + $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.$CLASS_ID.log + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_norm_apply.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --value $NORM --input_uri $URI --camera $CAMERA --det_type $DET_TYPE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingNormExp +# compare the new list with the ones already selected +task detrend.normexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.normexp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tonormalizedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingNormExp_DB + command dettool -tonormalizedexp -limit 20 -dbname $DB:$detPendingNormExp_DB + $detPendingNormExp_DB ++ + if ($detPendingNormExp_DB >= $DB:n) set detPendingNormExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingNormExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingNormExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.normexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingNormExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingNormExp + book getpage detPendingNormExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingNormExp $pageName pantaskState RUN + book getword detPendingNormExp $pageName det_id -var DET_ID + book getword detPendingNormExp $pageName iteration -var ITERATION + book getword detPendingNormExp $pageName det_type -var DET_TYPE + book getword detPendingNormExp $pageName camera -var CAMERA + book getword detPendingNormExp $pageName workdir -var WORKDIR + book getword detPendingNormExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # XXX add $WORKDIR/$LOG_DIR + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + $outpath = $outpath/$CAMERA.$DET_TYPE.$DET_ID + $logfile = $outpath/$CAMERA.$DET_TYPE.norm.$DET_ID.$ITERATION.log + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_norm_exp.pl --det_id $DET_ID --iteration $ITERATION --camera $CAMERA --det_type $DET_TYPE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingNormExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingNormExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/detrend.process.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.process.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.process.pro (revision 16175) @@ -0,0 +1,320 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingProcessedImfile and detPendingProcessedExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingProcessedImfile +book init detPendingProcessedExp + +macro detproc.reset + book init detPendingProcessedImfile + book init detPendingProcessedExp +end + +macro detproc.status + echo detPendingProcessedImfile + book listbook detPendingProcessedImfile + echo detPendingProcessedExp + book listbook detPendingProcessedExp +end + +macro detproc.on + task detrend.process.load + active true + end + task detrend.process.run + active true + end + task detrend.processexp.load + active true + end + task detrend.processexp.run + active true + end +end + +macro detproc.off + task detrend.process.load + active false + end + task detrend.process.run + active false + end + task detrend.processexp.load + active false + end + task detrend.processexp.run + active false + end +end + + +# these variables will cycle through the known database names +$detPendingProcessedImfile_DB = 0 +$detPendingProcessedExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingProcessedImfile +# compare the new list with the ones already selected +task detrend.process.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detproc.imfile.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toprocessedimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedImfile_DB + command dettool -toprocessedimfile -limit 20 -dbname $DB:$detPendingProcessedImfile_DB + $detPendingProcessedImfile_DB ++ + if ($detPendingProcessedImfile_DB >= $DB:n) set detPendingProcessedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedImfile + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand + end +end + +# run detrend_process_imfile.ps on pending images +task detrend.process.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingProcessedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingProcessedImfile + book getpage detPendingProcessedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedImfile $pageName pantaskState RUN + book getword detPendingProcessedImfile $pageName det_id -var DET_ID + book getword detPendingProcessedImfile $pageName det_type -var DET_TYPE + book getword detPendingProcessedImfile $pageName exp_id -var EXP_ID + book getword detPendingProcessedImfile $pageName class_id -var CLASS_ID + book getword detPendingProcessedImfile $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedImfile $pageName uri -var URI + book getword detPendingProcessedImfile $pageName camera -var CAMERA + book getword detPendingProcessedImfile $pageName workdir -var WORKDIR + book getword detPendingProcessedImfile $pageName dbname -var DBNAME + book getword detPendingProcessedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera $class_id + + ## output log filename + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_process_imfile.pl --det_id $DET_ID --exp_id $EXP_ID --det_type $DET_TYPE --class_id $CLASS_ID --exp_tag $EXP_TAG --input_uri $URI --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingProcessedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingProcessedExp +# compare the new list with the ones already selected +task detrend.processexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detproc.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toprocessedexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingProcessedExp_DB + command dettool -toprocessedexp -limit 20 -dbname $DB:$detPendingProcessedExp_DB + $detPendingProcessedExp_DB ++ + if ($detPendingProcessedExp_DB >= $DB:n) set detPendingProcessedExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingProcessedExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingProcessedExp + end + + # error + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.processexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingProcessedExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new exposures in detPendingProcessedExp + book getpage detPendingProcessedExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingProcessedExp $pageName pantaskState RUN + book getword detPendingProcessedExp $pageName det_id -var DET_ID + book getword detPendingProcessedExp $pageName exp_id -var EXP_ID + book getword detPendingProcessedExp $pageName det_type -var DET_TYPE + book getword detPendingProcessedExp $pageName camera -var CAMERA + book getword detPendingProcessedExp $pageName exp_tag -var EXP_TAG + book getword detPendingProcessedExp $pageName workdir -var WORKDIR + book getword detPendingProcessedExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # XXX push this into the filerules? + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detproc.%s" $outpath $EXP_TAG $DET_ID + $logfile = `ipp_filename.pl --filerule LOG.EXP --camera $CAMERA --class_id NONE --basename $outroot` + if ("$logfile" == "") + $logfile = $outroot.log + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_process_exp.pl --det_id $DET_ID --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit detPendingProcessedExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingProcessedExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/detrend.reject.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.reject.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.reject.pro (revision 16175) @@ -0,0 +1,167 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the book detRejectExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detRejectExp + +macro detreject.reset + book init detRejectExp +end + +macro detreject.status + book listbook detRejectExp +end + +macro detreject.on + task detrend.reject.load + active true + end + task detrend.reject.run + active true + end +end + +macro detreject.off + task detrend.reject.load + active false + end + task detrend.reject.run + active false + end +end + +# this variable will cycle through the known database names +$detRejectExp_DB = 0 + +# select images ready for copy +# new entries are added to detRejectExp +# compare the new list with the ones already selected +task detrend.reject.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detrend.reject.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -todetrunsummary -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detRejectExp_DB + command dettool -todetrunsummary -limit 20 -dbname $DB:$detRejectExp_DB + $detRejectExp_DB ++ + if ($detRejectExp_DB >= $DB:n) set detRejectExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detRejectExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detRejectExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detRejectExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand failure + end +end + +# copy new images, sending job to desired host +task detrend.reject.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detRejectExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detRejectExp + book getpage detRejectExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detRejectExp $pageName pantaskState RUN + book getword detRejectExp $pageName det_id -var DET_ID + book getword detRejectExp $pageName iteration -var ITERATION + book getword detRejectExp $pageName det_type -var DET_TYPE + book getword detRejectExp $pageName mode -var MODE + book getword detRejectExp $pageName camera -var CAMERA + book getword detRejectExp $pageName workdir -var WORKDIR + book getword detRejectExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/%s.%s.%s" $outroot $CAMERA $DET_TYPE $DET_ID + sprintf logfile "%s/%s.%s.%s.%s.detreject.log" $outroot $CAMERA $DET_TYPE $DET_ID $ITERATION + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = detrend_reject_exp.pl --det_id $DET_ID --iteration $ITERATION --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detRejectExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand failure + book setword detRejectExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/detrend.resid.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.resid.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.resid.pro (revision 16175) @@ -0,0 +1,320 @@ +## this file contains the tasks for running the detrend processing stage +## these tasks use the books detPendingResidImfile and detPendingResidExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingResidImfile +book init detPendingResidExp + +macro detresid.reset + book init detPendingResidImfile + book init detPendingResidExp +end + +macro detresid.status + book listbook detPendingResidImfile + book listbook detPendingResidExp +end + +macro detresid.on + task detrend.resid.load + active true + end + task detrend.resid.run + active true + end + task detrend.residexp.load + active true + end + task detrend.residexp.run + active true + end +end + +macro detresid.off + task detrend.resid.load + active false + end + task detrend.resid.run + active false + end + task detrend.residexp.load + active false + end + task detrend.residexp.run + active false + end +end + +# these variables will cycle through the known database names +$detPendingResidImfile_DB = 0 +$detPendingResidExp_DB = 0 + +# select images ready for copy +# new entries are added to detPendingResidImfile +# compare the new list with the ones already selected +task detrend.resid.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detresid.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toresidimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingResidImfile_DB + command dettool -toresidimfile -limit 20 -dbname $DB:$detPendingResidImfile_DB + $detPendingResidImfile_DB ++ + if ($detPendingResidImfile_DB >= $DB:n) set detPendingResidImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidImfile + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.resid.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingResidImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingResidImfile + book getpage detPendingResidImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidImfile $pageName pantaskState RUN + book getword detPendingResidImfile $pageName det_id -var DET_ID + book getword detPendingResidImfile $pageName exp_id -var EXP_ID + book getword detPendingResidImfile $pageName iteration -var ITERATION + book getword detPendingResidImfile $pageName det_type -var DET_TYPE + book getword detPendingResidImfile $pageName mode -var MODE + book getword detPendingResidImfile $pageName exp_tag -var EXP_TAG + book getword detPendingResidImfile $pageName class_id -var CLASS_ID + book getword detPendingResidImfile $pageName uri -var URI + book getword detPendingResidImfile $pageName det_uri -var DET_URI + book getword detPendingResidImfile $pageName camera -var CAMERA + book getword detPendingResidImfile $pageName workdir -var WORKDIR + book getword detPendingResidImfile $pageName dbname -var DBNAME + book getword detPendingResidImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera $class_id + + # XXX use ipp_filename.pl to lookup output file names + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_resid.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --class_id $CLASS_ID --det_type $DET_TYPE --detrend $DET_URI --input_uri $URI --camera $CAMERA --mode $MODE + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create command + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidImfile $options:0 pantaskState TIMEOUT + end +end + +# select images ready for copy +# new entries are added to detPendingResidExp +# compare the new list with the ones already selected +task detrend.residexp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detresid.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -toresidexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingResidExp_DB + command dettool -toresidexp -limit 20 -dbname $DB:$detPendingResidExp_DB + $detPendingResidExp_DB ++ + if ($detPendingResidExp_DB >= $DB:n) set detPendingResidExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingResidExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingResidExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.residexp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages detPendingResidExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingResidExp + book getpage detPendingResidExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingResidExp $pageName pantaskState RUN + book getword detPendingResidExp $pageName det_id -var DET_ID + book getword detPendingResidExp $pageName exp_id -var EXP_ID + book getword detPendingResidExp $pageName iteration -var ITERATION + book getword detPendingResidExp $pageName det_type -var DET_TYPE + book getword detPendingResidExp $pageName mode -var MODE + book getword detPendingResidExp $pageName exp_tag -var EXP_TAG + book getword detPendingResidExp $pageName include -var INCLUDE + book getword detPendingResidExp $pageName camera -var CAMERA + book getword detPendingResidExp $pageName workdir -var WORKDIR + book getword detPendingResidExp $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera FPA + + # XXX use ipp_filename.pl to lookup output file names + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s/%s" $outpath $CAMERA $DET_TYPE $DET_ID $EXP_TAG + sprintf outroot "%s/%s.detresid.%s.%s" $outpath $EXP_TAG $DET_ID $ITERATION + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id NONE --basename $outroot` + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + $run = detrend_reject_imfile.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create example job options as a demonstration + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingResidExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingResidExp $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/detrend.stack.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/detrend.stack.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/detrend.stack.pro (revision 16175) @@ -0,0 +1,177 @@ +## this file contains the tasks for running the detrend stacking stage +## these tasks use the book detPendingStackedImfile + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/detrend +exec mkdir -p $LOGSUBDIR + +book init detPendingStackedImfile + +macro detstack.reset + book init detPendingStackedImfile +end + +macro detstack.status + echo detPendingStackedImfile + book listbook detPendingStackedImfile +end + +macro detstack.on + task detrend.stack.load + active true + end + task detrend.stack.run + active true + end +end + +macro detstack.off + task detrend.stack.load + active false + end + task detrend.stack.run + active false + end +end + +# this variable will cycle through the known database names +$detPendingStackedImfile_DB = 0 + +# select images ready for detrend_stack.pl +# new entries are added to detPendingStackedImfile +# compare the new list with the ones already selected +task detrend.stack.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/detstack.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command dettool -tostacked -limit 20 + else + # save the DB name for the exit tasks + option $DB:$detPendingStackedImfile_DB + command dettool -tostacked -limit 20 -dbname $DB:$detPendingStackedImfile_DB + $detPendingStackedImfile_DB ++ + if ($detPendingStackedImfile_DB >= $DB:n) set detPendingStackedImfile_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout detPendingStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook detPendingStackedImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup detPendingStackedImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# copy new images, sending job to desired host +task detrend.stack.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 120 + + task.exec + book npages detPendingStackedImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in detPendingStackedImfile + book getpage detPendingStackedImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword detPendingStackedImfile $pageName pantaskState RUN + book getword detPendingStackedImfile $pageName det_id -var DET_ID + book getword detPendingStackedImfile $pageName iteration -var ITERATION + book getword detPendingStackedImfile $pageName det_type -var DET_TYPE + book getword detPendingStackedImfile $pageName class_id -var CLASS_ID + book getword detPendingStackedImfile $pageName camera -var CAMERA + book getword detPendingStackedImfile $pageName workdir -var WORKDIR + book getword detPendingStackedImfile $pageName dbname -var DBNAME + book getword detPendingStackedImfile $pageName reduction -var REDUCTION + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $camera $class_id + + if ("$WORKDIR" == "NULL") + $outpath = `pwd` + else + $outpath = `ipp_datapath.pl $WORKDIR` + end + sprintf outpath "%s/%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID + sprintf outroot "%s/%s.%s.%s.%s" $outpath $CAMERA $DET_TYPE $DET_ID $ITERATION + $logfile = `ipp_filename.pl --filerule LOG.IMFILE --camera $CAMERA --class_id $CLASS_ID --basename $outroot` + if ("$logfile" == "") + sprintf logfile "%s.%s.log" $outroot $CLASS_ID + end + + stdout $logfile + stderr $logfile + exec mkdir -p $outpath + + # save the pageName for future reference below + options $pageName + + $run = detrend_stack.pl --det_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --det_type $DET_TYPE --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit detPendingStackedImfile $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword detPendingStackedImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/diff.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/diff.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/diff.pro (revision 16175) @@ -0,0 +1,180 @@ +### This file contains panTasks definitions for performing the image differencing. + +### After a difference (with associated diff_id) is defined, the difference is performed +### (tasks in diffSkyfile). + + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/diff +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init diffSkyfile + +### Database lists +$diffSkycell_DB = 0 + +### Check status of diffing tasks +macro diff.status + book listbook diffSkyfile +end + +### Reset diffing tasks +macro diff.reset + book init diffSkyfile +end + +### Turn diffing tasks on +macro diff.on + task diff.skycell.load + active true + end + task diff.skycell.run + active true + end +end + +### Turn diffing tasks off +macro diff.off + task diff.skycell.load + active false + end + task diff.skycell.run + active false + end +end + + +### Load tasks for doing the differences +### Tasks are loaded into diffSkyfile. +task diff.skycell.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/diff.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command difftool -todiffskyfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$diffSkycell_DB + command difftool -todiffskyfile -limit 20 -dbname $DB:$diffSkycell_DB + $diffSkycell_DB ++ + if ($diffSkycell_DB >= $DB:n) set diffSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout diffSkyfile -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook diffSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup diffSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the diff overlaps +### Tasks are taken from diffSkyfile. +task diff.skycell.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages diffSkyfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in diffSkyfile (pantaskState == INIT) + book getpage diffSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword diffSkyfile $pageName pantaskState RUN + book getword diffSkyfile $pageName diff_id -var DIFF_ID + book getword diffSkyfile $pageName workdir -var WORKDIR + book getword diffSkyfile $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on diffs + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/diff.%s" $outroot $DIFF_ID + sprintf logfile "%s/skycell.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = diff_skycell.pl --diff_id $DIFF_ID + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit diffSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword diffSkyfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branch_20080121/ippTasks/isp.det =================================================================== --- /branches/eam_branch_20080121/ippTasks/isp.det (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/isp.det (revision 16175) @@ -0,0 +1,91 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/bias -det_type bias -select_exp_type bias -select_dateobs_begin 2007-01-13T00:00:00" + dbname STR isp + pantaskState STR DONE + detRunID STR 1 +END + +detRunDef METADATA + detRunName STR DARK-warm + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_min -27.5 -select_ccd_temp_max -24.0 -ccd_temp_min -27.5 -ccd_temp_max -24.0" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK-cool + prereq STR BIAS + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/dark -det_type dark -select_exp_type dark -select_exp_time_min 0.5 -select_ccd_temp_max -27.5 -ccd_temp_max -27.5" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR "DARK-warm DARK-cool" + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/shutter -det_type shutter -select_exp_type skyflat -select_exp_time_min 0.05 -select_filter g -select_dateobs_begin 2007-01-14T04:00:00 -select_dateobs_end 2007-01-14T05:00:00 -select_sat_pixel_frac_max 0.1" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-g + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/g -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter g -filter g -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/r -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter r -filter r -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/i -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter i -filter i -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-z + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/z -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter z -filter z -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-y + prereq STR SHUTTER + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/flat/y -det_type flat -select_exp_type skyflat -select_exp_time_min 0.05 -select_sat_pixel_frac_max 0.1 -select_filter y -filter y -select_dateobs_begin 2007-01-14T00:00:00 -select_dateobs_end 2007-01-15T00:00:00" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-z + prereq STR FLAT-z + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/z -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter z -filter z -select_dateobs_begin 2007-01-14T09:19:00 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FRINGE-y + prereq STR FLAT-y + options STR "-definebyquery -inst ISP-Apogee -workdir path://ISP/detrend/fringe/y -det_type fringe -select_exp_type object -select_exp_time_min 0.05 -select_filter y -filter y -select_dateobs_begin 2007-01-13 -select_dateobs_end 2007-01-18" + dbname STR isp + pantaskState STR NULL +END Index: /branches/eam_branch_20080121/ippTasks/magic.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/magic.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/magic.pro (revision 16175) @@ -0,0 +1,229 @@ + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/magic +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init magicToProcess +book init magicToMask +book init magicToSkyfileMask + +### Database lists +$magic_DB = 0 + +### Check status of warping tasks +#macro warp.status +# book listbook warpInputExp +# book listbook warpPendingSkyCell +#end + +### Reset warping tasks +#macro warp.reset +# book init warpInputExp +# book init warpPendingSkyCell +#end + +### Turn warping tasks on +# macro warp.on +# task warp.exp.load +# active true +# end +# task warp.exp.run +# active true +# end +# task warp.skycell.load +# active true +# end +# task warp.skycell.run +# active true +# end +# end +# +# ### Turn warping tasks off +# macro warp.off +# task warp.exp.load +# active false +# end +# task warp.exp.run +# active false +# end +# task warp.skycell.load +# active false +# end +# task warp.skycell.run +# active false +# end +# end + +# populate magicToProcess +task magic.toprocess + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/magic.toprocess.log + + task.exec + if ($DB:n == 0) + option DEFAULT + run = command magictool -toprocess -limit 20 + if ($VERBOSE > 1) + echo command $run + end + command $run + else + # save the DB name for the exit tasks + option $DB:$magic_DB + command magictool -toprocess -limit 20 -dbname $DB:$magic_DB + $magic_DB ++ + if ($magic_DB >= $DB:n) set magic_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToProcess -key magic_id:node -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToProcess + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# populate magicToMask +task magic.tomask + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/magic.tomask.log + + task.exec + if ($DB:n == 0) + option DEFAULT + run = command magictool -tomask -limit 20 + if ($VERBOSE > 1) + echo command $run + end + command $run + else + # save the DB name for the exit tasks + option $DB:$magic_DB + command magictool -tomask -limit 20 -dbname $DB:$magic_DB + $magic_DB ++ + if ($magic_DB >= $DB:n) set magic_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToMask + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + +# populate magicToMask +task magic.toskyfilemask + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/magic.toskyfilemask.log + + task.exec + if ($DB:n == 0) + option DEFAULT + run = command magictool -toskyfilemask -limit 20 + if ($VERBOSE > 1) + echo command $run + end + command $run + else + # save the DB name for the exit tasks + option $DB:$magic_DB + command magictool -toskyfilemask-limit 20 -dbname $DB:$magic_DB + $magic_DB ++ + if ($magic_DB >= $DB:n) set magic_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicToSkyfileMask -key magic_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicToProcess + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicToSkyfileMask + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branch_20080121/ippTasks/mkdetrend.config =================================================================== --- /branches/eam_branch_20080121/ippTasks/mkdetrend.config (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/mkdetrend.config (revision 16175) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branch_20080121/ippTasks/mkdetrend.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/mkdetrend.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/mkdetrend.pro (revision 16175) @@ -0,0 +1,217 @@ + +# example output from a dettool -definebyquery +# detRun METADATA +# det_id STR 3 +# iteration S32 0 +# det_type STR SHUTTER +# mode STR master +# state STR run +# filelevel STR FPA +# END + +macro load.detRunDefs + if ($0 != 2) + echo "USAGE: load.detRunDefs (filename)" + break + end + queueload detruns -x "cat $MODULES:0/$1" + ipptool2book detruns detRunDef -key detRunName -uniq -setword pantaskState INIT + queuedelete detruns +end + +task detrun.launch + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i j N found missed detRunName prereq prereqList prereqState cmdOptions dbname + + # how many detRunDefs are waiting to be started? + book npages detRunDef -var N -key pantaskState INIT + # echo "detrun.launch N: $N" + if ($N == 0) break + + $found = 0 + + # search the detRunDef book for an entry which is unstarted (state NULL) + # and for which the dependencies are done + for i 0 $N + book getpage detRunDef $i -var detRunName -key pantaskState INIT + # echo "detrun.launch detRunName: $detRunName" + if ("$detRunName" == "NULL") break + + book getword detRunDef $detRunName prereq -var prereqList + if (("$prereqList" == "NULL") || ("$prereqList" == "NONE")) + $found = 1 + last + end + + list prereq -split $prereqList + $missed = 0 + for j 0 $prereq:n + # prereq must exist and be in state of DONE + book getword detRunDef $prereq:$j pantaskState -var prereqState + + # echo "detrun.launch prereqState: $prereqState" + # echo "detrun.launch detRunName: $detRunName" + if ("$prereqState" != "DONE") + $missed = 1 + last + end + end + if ($missed == 0) + $found = 1 + last + end + + if ($found == 0) + if ($VERBOSE >= 2) + echo "$detRunName not ready" + end + else + if ($VERBOSE >= 2) + echo "$detRunName ready" + end + end + end + + if ($found == 0) break + + # echo "detrun.launch detRunName: $detRunName" + + book getword detRunDef $detRunName options -var cmdOptions + book getword detRunDef $detRunName dbname -var dbname + book setword detRunDef $detRunName pantaskState INIT + + if ($VERBOSE >= 1) + echo "starting detrend $detRunName :" + echo "command dettool $cmdOptions -dbname $dbname" + end + + options $dbname $detRunName + command dettool $cmdOptions -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # we need to track the detID generated + book setword detRunDef $detRunName detRunID $detRunID + book setword detRunDef $detRunName pantaskState RUN + + # drop the detRun book after we've grabbed the state + book delete detRun + end +end + +$detRunNcheck = 0 + +# loop over the outstanding detRuns and update state +task detrun.check + host local + + periods -poll 1 + periods -exec 5 + periods -timeout 30 + active true + npending 1 + + # each outstanding detrend run blocks the next one. only launch the next + # if the current one is done + task.exec + local i N found detRunName detRunID dbname + + book npages detRunDef -var N -key pantaskState RUN + if ($N == 0) break + if ($detRunNcheck >= $N) + $detRunNcheck = 0 + end + + # try the next detRun entry which is in state RUN + book getpage detRunDef $detRunNcheck -var detRunName -key pantaskState RUN + $detRunNcheck ++ + + if ("$detRunName" == "NULL") + if ($VERBOSE >= 3) + echo "no more active detruns (Ncheck = $detRunNcheck)" + end + break + end + + # info needed to check on current state + book getword detRunDef $detRunName detRunID -var detRunID + book getword detRunDef $detRunName dbname -var dbname + + if ($VERBOSE >= 2) + echo " checking on $detRunName" + echo "command dettool -runs -det_id $detRunID -dbname $dbname" + end + + # we need a dettool command which takes a specific det_id and returns + # the current state + options $dbname $detRunName + command dettool -runs -det_id $detRunID -dbname $dbname + end + + task.exit $EXIT_SUCCESS + local detRunID state + + $dbname = $options:0 + $detRunName = $options:1 + + # convert 'stdout' to book format + ipptool2book stdout detRun -key det_id -uniq + if ($VERBOSE > 2) + book listbook detRun + end + + # we should have launched only a single detRun + book getpage detRun 0 -var detRunID + if ("$detRunID" == "NULL") + echo "ERROR:detRun not defined" + break + end + + # what is the current state of the detRun? + book getword detRun $detRunID state -var state + + # use any other state? + if ("$state" == "stop") + # update the state for this detRunName, drop detRun page + book setword detRunDef $detRunName pantaskState DONE + if ($VERBOSE > 1) + echo "$detRunName is done" + end + else + if ($VERBOSE > 1) + echo "$detRunName is still running" + end + end + + # drop this book when completed + book delete detRun + end +end Index: /branches/eam_branch_20080121/ippTasks/notes.txt =================================================================== --- /branches/eam_branch_20080121/ippTasks/notes.txt (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/notes.txt (revision 16175) @@ -0,0 +1,265 @@ + +2008.01.18 + + I'm working on the summit.copy.pro interaction with nebulous (and + those of the other ippTasks) and the assignment of jobs and data + locations to hosts on the basis of the chip/host relationship. I + have defined a table which relates the chip (class_id) to the + desired host name. There are a few places, and a few slightly + different ways, in which this gets used: + + * choice of host for command execution: + + every job which operates on a specific chip must be told the chip + host so it can run the job on that machine. the information is + supplied in pantasks to the job with the 'host' command. I've + made a pantasks macro 'set.host.by.camera' which takes the camera + and class_id, looks up the match in the chip/host table, and calls + the host command with the desired host. If there is no match + (camera not listed in chip/host table or chip not listed in + table), the host is set to 'anyhost', which randomly chooses one. + I have used this command for all tasks which generate a pcontrol + command. For the tasks which are performed at the camera level, + these are set to anyhost. For the tasks which operate on stacks, + we can use the same structure to associate the skycell with a + host. + + * dsget output target: + + dsget is called by pantasks in the task 'dsget'. currently, dsget + accepts a --filename argument which specifies the output filename + to use. we have to tell t + + + * choice of output directory. + + the + + +2007.10.11 + + The calibration suite uses a database table of known/registered DVO + databases to be calibrated. The suite operates on a segment of the + sky in each run, nominally a section ~1h west of the current ST. + + caltool -pendingdbs returns a list of entries (mdc format), one for + each DVO database in the table. + + caltool -databases [-active] : returns the list of dvo databases + caltool -addcalrun : inserts the stats for a single calibration run + caltool -setstate (active/inactive) : modify the state of a dvo + database + caltool -adddatabase : insert a new database + + XXX a potential source of collisions. the books are marked by + unique keys in the db tables, but a single pantasks run may be + addressing multiple databases. IDs are not unique across + databases. we can solve this by including in the key the dbname + + the calibration suite runs the following tasks, in sequences on each + of the databases: + + * resort, relphot, uniphot, relastro.objects, relastro.images + + ** detrend correction: any detrend images which need to be corrected + get inserted in the detCorrectedExp table. + + dettool commands needed: + * add a detrend exposure (det_id, exp_id) to the detCorrectedExp table + * add a detrend imfile (det_id, exp_id, class_id) to the + detCorrectedImfile table + * select pending detCorrectedExp + * select pending detCorrectedImfiles + + +2007.01.08 + + I have updated pantasks to used named tags in named groups to make + life much easier when changing the schema of the db tables. The + function ipptool2book convernts the output from an ippTool query in + the form of a set of metadata structures (saved in stdout) into a + book format. + + dettool -tonormalizedstat DetrendNormStat + dettool -tonormalize DetrendNorm + dettool -tonormalizedexp DetrendNormExp + dettool -toprocessedimfile DetrendProcessImfiles + dettool -toprocessedexp DetrendProcessExposures + dettool -residdetrun DetrendRejectExp + dettool -toresidimfile DetrendResidImfiles + dettool -toresidexp DetrendResidExposures + dettool -tostacked DetrendStackClass + p0tool -pendingimfile Phase0Imfiles + p0tool -pendingexp Phase0Exposures + p2tool -pendingimfile Phase2Imfiles + p3tool -pendingexp Phase3Exposures + +2006.08.28 + +- ipptools need to use expID:camera:telescope as unique key (ie, not expID alone) +- p0tool -pendingimfile -simple needs to report the expKey (ie, not expID) +- + + +This directory contains the PanTasks scripts which define the steps +performed by the IPP. These scripts use the ippTools to decide what +images need to be processed, and to update the Metadata Database with +the current state of the analysis. The analysis is performed using +IPP programs which are distributed to the cluster computers as needed. + +1. The Detrend Creation Scripts + +The detrend creation process is among the more complex elements of the +IPP. There are N major steps which must be performed: + +1.1. Define New Detrend Runs + +The script 'detrend.mkruns.pro' defines tasks which initiate new +Detrend Runs. There is a separate task for each camera automatically +processed by the system (mkdetruns.CAMERA). These tasks are executed +on a regular, but infrequent basis. The tasks call external helper +programs which actually run the command dettools -define. The helper +programs are thus responsible for identifying all of the types of +detrend images to be run for a given camera, as well as the correct +set of filters to apply. These tasks are run locally and only affect +the database. PanTasks does not track the detrend runs which are +created. + +- we may need to define these within the PanTasks scripts +- how frequently are these tasks run (1/day?) +- if no raw images are selected by the query, no new detrun should be + created. +- we may need different schedules for the dark (bias/dark) and the + light (flat/fringe) data. And yet another schedule for sky data + (skyflat/skyfringe). + +1.2. Process new raw imfiles + +The script 'detrend.process.pro' defines tasks which processes the new +raw imfiles for any detrend run. This script defines two tasks, +'dettool.raw.load' and 'dettool.raw.process'. The first script +occasionally polls the Metadata Database to select raw imfiles which +have been identified as part of a detrend run. The identified raw +imfiles are added to an internal queue, DetrendImfilesToProcess. Any +raw imfile already in the list is ignored. The imfiles are added to +the queue with a state of NEW. The second task examines the queue for +imfiles with state NEW and defines a ppImage analysis job for that +imfile. The job is sent to pcontrol for processing on the distributed +cluster. At this point, the corresponding queue entry is set to the +state RUN. Upon success, the queue entry is set to DONE and the +database is notified of the successful operation. The DONE entries +are occasionally pruned from the queue; this must be done in the same +task as the initial query and the update of the list based on the +database results. + +- we need to define the columns in the output of dettool -raw +- we need a method to associate an imfile with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs +- we need a method to identify the correct recipe (function of type?) + +1.3. Stack new processed imfiles + +The script 'detrend.stack.pro' defines tasks which stack the newly +processed imfiles for any detrend run iteration. This script defines +two tasks, 'dettool.stack.load and 'dettool.stack.process. The first +script occasionally polls the Metadata Database to select class IDs +for detrend runs for which the input imfiles have all been processed. +The identified detRun/classID combination is added to an internal +queue, DetrendClassIDtoStack. The entries are added to the queue with +a state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppMerge analysis job for that +detID/classID. The helper script selects the corresponding input +imfiles and passes them to ppMerge. The job is sent to pcontrol for +processing on the distributed cluster. At this point, the +corresponding queue entry is set to the state RUN. Upon success, the +queue entry is set to DONE and the database is notified of the +successful operation. The DONE entries are occasionally pruned from +the queue; this must be done in the same task as the initial query and +the update of the list based on the database results. + +- we need to define the columns in the output of dettool -chip -processed +- we need a method to associate a classID with a node in the cluster +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) + + +1.4. Normalize stacked masters + +The script 'detrend.norm.pro' defines tasks which normalize the newly +processed stacks for any detrend run iteration. This script defines +two tasks, 'dettool.norm.load and 'dettool.norm.process. The first +script occasionally polls the Metadata Database to select detrun IDs +for which the classes have all been stacked. The identified +detRunID/iteration combination is added to an internal queue, +DetrendRunToNormalize. The entries are added to the queue with a +state of NEW. The second task examines the queue for entries with a +state of NEW. It defines a ppNorm analysis job for that +detID/iteration. The job is sent to pcontrol for processing on the +distributed cluster. At this point, the corresponding queue entry is +set to the state RUN. Upon success, the queue entry is set to DONE and +the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need to define the ppNorm operation inputs and outputs better + + +1.5. Generate Resid Imfiles + +The script 'detrend.resid.pro' defines tasks which apply the newly +generated detrend image to the corresponding input images to generate +the residual images. This script defines two tasks, +'dettool.resid.load and 'dettool.resid.process. The first script +occasionally polls the Metadata Database to select imfiles for which +the master detrend images has been normalized. The identified imfiles +are added to an internal queue, DetrendResidImfiles. The entries are +added to the queue with a state of NEW. The second task examines the +queue for entries with a state of NEW. It defines a ppImage analysis +job for that imfile. The job is sent to pcontrol for processing on +the distributed cluster. At this point, the corresponding queue entry +is set to the state RUN. Upon success, the queue entry is set to DONE +and the database is notified of the successful operation. The DONE +entries are occasionally pruned from the queue; this must be done in +the same task as the initial query and the update of the list based on +the database results. + +- we need to define the columns in the output of dettool +- can we select the imfiles here without apriori knowledge that the + master has been normalized? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) +- we need a method to identify the correct recipe (function of type?) +- we need a method to pass the matching detrend image to ppImage in + the correct context (ie, a bias needs to be supplied to the -bias element...) + +1.6. Assess Detrend Results + +The script 'detrend.assess.pro' defines tasks which assess the +residual statistics for completed detrend runs. This script defines +two tasks, 'dettool.assess.load and 'dettool.assess.process. The +first script occasionally polls the Metadata Database to select +detrunIDs/iterations for which the resid imfiles have all been +generated. The identified detruns are added to an internal queue, +DetrendRunToAssess. The entries are added to the queue with a state +of NEW. The second task examines the queue for entries with a state +of NEW. It defines an analysis job for that collection. The job is +sent to pcontrol for processing on the distributed cluster. At this +point, the corresponding queue entry is set to the state RUN. Upon +success, the queue entry is set to DONE and the database is notified +of the successful operation. The DONE entries are occasionally pruned +from the queue; this must be done in the same task as the initial +query and the update of the list based on the database results. + +- the assess tool needs to trigger a new run, if needed. +- what is the command to run on the resid statistics? +- we need to define the columns in the output of dettool +- can we select the detrend runs here? +- we need to perform some appropriate action when the queries fail. +- we need a method to generate the output URIs (not a function of the imfiles...) Index: /branches/eam_branch_20080121/ippTasks/pantasks.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/pantasks.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/pantasks.pro (revision 16175) @@ -0,0 +1,389 @@ + +# XXX make these upper-case +$NEBULOUS = 0 +$network = 1 +$parallel = 1 +$VERBOSE = 1 +$LOGDIR = `pwd` +$scripts = . + +$loadpoll = 0.25 +$loadexec = 5 +$runpoll = 0.5 +$runexec = 1 + +$EXIT_SUCCESS = 0 +$EXIT_UNKNOWN_ERR = 1 +$EXIT_SYS_ERR = 2 +$EXIT_CONFIG_ERR = 3 +$EXIT_PROG_ERR = 4 +$EXIT_DATA_ERR = 5 + +# DB lists the database names in use; the default ipprc database +# is only used if DB:n is 0 +$DB:n = 0 + +# user function to add databases by name (avoids duplicates) +macro add.database + if ($0 != 2) + echo "USAGE: add.database (db)" + break + end + if ($?DB:n == 0) + list DB -add $1 + return + end + + $found = 0 + for i 0 $DB:n + if ($DB:$i == $1) + $found = 1 + echo "$DB:$i set" + last + end + end + + if ($found == 0) + list DB -add $1 + end +end + +macro del.database + if ($0 != 2) + echo "USAGE: del.database (db)" + break + end + if ($?DB:n == 0) + return + end + +### XXX There is a bug in the below "list DB -del $1" line. + echo "This function doesn't work" +# list DB -del $1 +end + +macro init.cluster.po + $parallel = 1 + controller exit true + + # po05 -- using for pantasks + # po23 -- broken perl + + controller host add po02 + controller host add po03 + controller host add po04 +# controller host add po05 + controller host add po06 + controller host add po07 +# controller host add po08 +# controller host add po09 +# controller host add po10 +# controller host add po11 +# controller host add po12 +# controller host add po13 +# controller host add po14 +# controller host add po15 +# controller host add po16 +# controller host add po17 +# controller host add po18 +# controller host add po19 +# controller host add po20 +# controller host add po21 +# controller host add po22 +# controller host add po23 +# controller host add po24 +end + +macro init.cluster.sn + $parallel = 1 + controller exit true + controller host add sn2 + controller host add sn3 + controller host add sn4 + controller host add sn5 +end + +macro init.isp + list DB -add isp +end + +macro init.essence + list DB -add essence_v2 +end + +macro init.simtest.local + list DB -add simtest + $parallel = 0 + controller exit true +end + +macro init.simtest.default.db + $host = `hostname` + if ($parallel) + controller exit true + controller host add $host + end +end + +macro init.simtest + list DB -add simtest + + $host = `hostname` + if ($parallel) + controller exit true + controller host add $host + end +end + +# need the ability to activate or deactivate specific tasks... + +macro detrend.on + detproc.on + detstack.on + detnorm.on + detresid.on + detreject.on +end + +macro detrend.off + detproc.off + detstack.off + detnorm.off + detresid.off + detreject.off +end + +macro detrend.reset + detproc.reset + detstack.reset + detnorm.reset + detresid.reset + detreject.reset +end + +macro all.on + register.on + detrend.on + chip.on + camera.on + warp.on + diff.on + stack.on +end + +macro all.off + register.off + detrend.off + chip.off + camera.off + warp.off + diff.off + stack.off +end + +macro module.tasks + module register.pro + module chip.pro + module camera.pro + module detrend.process.pro + module detrend.stack.pro + module detrend.norm.pro + module detrend.resid.pro + module detrend.reject.pro + module warp.pro + module diff.pro + module stack.pro +end + +macro showcommand + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + else + echo "command: $command" + end + end +end + +macro add_standard_args + if ($0 != 2) + echo "Must pass in the command of interest, and this function will supplement" + stop + end + + local command + $command = $$1 + + if ($?DBNAME && "$DBNAME" != "DEFAULT") + $command = $command --dbname $DBNAME + end + if ($?NOOP != 0) + if ($NOOP != 0) + $command = $command --no-op + end + end + if ($?NOUPDATE != 0) + if ($NOUPDATE != 0) + $command = $command --no-update + end + end + $$1 = $command +end + + +macro process_exit + if ($0 != 4) + echo "USAGE: process_exit (bookName) (pageName) (exitCode)" + break + end + + $bookName = $1 + $pageName = $2 + $exitCode = $3 + + if ($VERBOSE > 4) + echo "*** exit status ***" + echo "JOB_STATUS: $JOB_STATUS" + echo "JOB_HOSTNAME: $JOB_HOSTNAME" + echo "JOB_DTIME: $JOB_DTIME" + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # success + if ($exitCode == $EXIT_SUCCESS) + # the handler scripts update DB the tables; here we just update the page + book setword $bookName $pageName pantaskState DONE + return + end + + # failure related to the data files + # jobs which result in DATAERR must have db pantaskState updated + if ($exitCode == $EXIT_DATA_ERR) + showcommand failure + book setword $bookName $pageName pantaskState DATA_ERR + return + end + + if ($VERBOSE) + echo "*** stdout ***" + queueprint stdout + echo "*** stderr ***" + queueprint stderr + end + + # failure related to the data files + if ($exitCode == $EXIT_SYS_ERR) + # stop + showcommand "system failure" + book setword $bookName $pageName pantaskState SYS_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_CONFIG_ERR) + # stop + showcommand "config error" + book setword $bookName $pageName pantaskState CONFIG_ERR + return + end + + # failure related to the data files + if ($exitCode == $EXIT_PROG_ERR) + # stop + showcommand "programming error" + book setword $bookName $pageName pantaskState PROG_ERR + return + end + + # any other exit status + showcommand failure + book setword $bookName $pageName pantaskState UNKNOWN_ERR +end + +## XXX for the moment, remove all errors +macro process_cleanup + if ($0 != 2) + echo "USAGE: process_cleanup (bookname)" + break + end + + book delpage $1 -key pantaskState DONE + book delpage $1 -key pantaskState SYS_ERR + book delpage $1 -key pantaskState DATA_ERR + book delpage $1 -key pantaskState PROG_ERR + book delpage $1 -key pantaskState CONFIG_ERR + book delpage $1 -key pantaskState UNKNOWN_ERR +end + +macro set.verbosity + if ($0 != 2) + echo "USAGE: set.verbosity (level)" + break + end + $VERBOSE = $1 +end + +macro show.variables + ?? +end + +macro set.workdir.by.camera + if ($0 != 3) + echo "USAGE: set.workdir.by.camera (camera) (class_id)" + break + end + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $1 $2 -var $host + + if ($NEBULOUS) + if ("$host" == "NULL") + $workdir = $workdir_default + $volume = "NULL" + return + end + strsub $workdir_template @HOST@ $host -var workdir + strsub $volume_template @HOST@ $host -var volume + return + end + + if ("$host" == "NULL") + $workdir = $workdir_default + return + end + strsub $workdir_template @HOST@ $host -var workdir +end + +macro set.host.by.camera + if ($0 != 3) + echo "USAGE: set.host.by.camera (camera) (class_id)" + break + end + + if (not($parallel)) + host local + return + end + + # missing camera and/or chiphosts table results in host = NULL + book getword chiphosts $1 $2 -var $host + + if ("$host" == "NULL") + host anyhost + else + host $host + end +end Index: /branches/eam_branch_20080121/ippTasks/register.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/register.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/register.pro (revision 16175) @@ -0,0 +1,304 @@ +## this file contains the tasks for running the registration stage +## these tasks use the books regPendingImfile and regPendingExp + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/register +exec mkdir -p $LOGSUBDIR + +book init regPendingImfile +book init regPendingExp + +macro register.reset + book init regPendingImfile + book init regPendingExp +end + +macro register.status + book listbook regPendingImfile + book listbook regPendingExp +end + +macro register.on + task register.imfile.load + active true + end + task register.imfile.run + active true + end + task register.exp.load + active true + end + task register.exp.run + active true + end +end + +macro register.off + task register.imfile.load + active false + end + task register.imfile.run + active false + end + task register.exp.load + active false + end + task register.exp.run + active false + end +end + +# these variables will cycle through the known database names +$regPendingImfile_DB = 0 +$regPendingExp_DB = 0 + +# select images ready for register analysis +# new entries are added to regPendingImfile +# compare the new list with the ones already selected +task register.imfile.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + # silently drop stdout + stdout NULL + stderr $LOGSUBDIR/register.imfile.log + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command regtool -pendingimfile -limit 20 + else + # save the DB name for the exit tasks + option $DB:$regPendingImfile_DB + command regtool -pendingimfile -limit 20 -dbname $DB:$regPendingImfile_DB + $regPendingImfile_DB ++ + if ($regPendingImfile_DB >= $DB:n) set regPendingImfile_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingImfile -key exp_id:tmp_class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingImfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_imfile.pl script on pending images +task register.imfile.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages regPendingImfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in regPendingImfile + book getpage regPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingImfile $pageName pantaskState RUN + + book getword regPendingImfile $pageName exp_id -var EXP_ID + book getword regPendingImfile $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword regPendingImfile $pageName uri -var URI + book getword regPendingImfile $pageName workdir -var WORKDIR_TEMPLATE + book getword regPendingImfile $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host + set.host.for.camera $camera $class_id + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, workdir_default, volume_template, volume_default, + # it sets workdir and volume + ### this function needs to be improved: it is using fixed input and output var names + ### do we want these types of small outputs to go to a common filesystem? + set.workdir.by.camera $CAMERA $TMP_CLASS_ID $WORKDIR_TEMPLATE + + ## generate output log based on filerule XXX put this in a function? + $outroot = `ipp_datapath.pl $WORKDIR` + sprintf outroot "%s/%s" $outroot $EXP_TAG + sprintf logfile "%s/%s.reg.log" $outroot $TMP_CLASS_ID + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + # save the pageName for future reference below + options $pageName + + $run = register_imfile.pl --exp_id $EXP_ID --tmp_class_id $TMP_CLASS_ID --tmp_exp_name $TMP_EXP_NAME --uri $URI + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit regPendingImfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword regPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for register_exp.pl +task register.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/register.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command regtool -pendingexp -limit 20 + else + # save the DB name for the exit tasks + option $DB:$regPendingExp_DB + command regtool -pendingexp -limit 20 -dbname $DB:$regPendingExp_DB + $regPendingExp_DB ++ + if ($regPendingExp_DB >= $DB:n) set regPendingExp_DB = 0 + end + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout regPendingExp -key exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook regPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup regPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the register_exp.pl script on pending exposures +task register.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 30 + + task.exec + book npages regPendingExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in regPendingExp (pantaskState == INIT) + book getpage regPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword regPendingExp $pageName pantaskState RUN + book getword regPendingExp $pageName exp_id -var EXP_ID + book getword regPendingExp $pageName tmp_exp_name -var TMP_EXP_NAME + book getword regPendingExp $pageName workdir -var WORKDIR + book getword regPendingExp $pageName dbname -var DBNAME + + # EXP_TAG is used to generate the unique, but human-readable, filenames + sprintf EXP_TAG "%s.%s" $TMP_EXP_NAME $EXP_ID + + # specify choice of remote host:(need to choose based on chips) + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + $outroot = `ipp_datapath.pl $WORKDIR` + sprintf outroot "%s/%s" $outroot $EXP_TAG + sprintf logfile "%s/%s.reg.log" $outroot $EXP_TAG + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + # save the pageName for future reference below + options $pageName + + $run = register_exp.pl --exp_id $EXP_ID --exp_tag $EXP_TAG + add_standard_args run + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit regPendingExp $options:0 $JOB_STATUS + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword regPendingExp $options:0 pantaskState TIMEOUT + end +end + +## XXX add a global path to output files Index: /branches/eam_branch_20080121/ippTasks/simtest.det =================================================================== --- /branches/eam_branch_20080121/ippTasks/simtest.det (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/simtest.det (revision 16175) @@ -0,0 +1,42 @@ + +detRunDef MULTI + +detRunDef METADATA + detRunName STR BIAS + prereq STR NONE + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type BIAS -select_exp_type BIAS" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR DARK + prereq STR BIAS + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type DARK -select_exp_type DARK" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR SHUTTER + prereq STR DARK + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-r + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r" + dbname STR eamtest + pantaskState STR NULL +END + +detRunDef METADATA + detRunName STR FLAT-i + prereq STR SHUTTER + options STR "-definebyquery -workdir path://EAMWORK -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i" + dbname STR eamtest + pantaskState STR NULL +END Index: /branches/eam_branch_20080121/ippTasks/stack.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/stack.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/stack.pro (revision 16175) @@ -0,0 +1,180 @@ +### This file contains panTasks definitions for performing the image stacking. + +### After a stack (with associated stack_id) is defined, the stack is performed +### (tasks in stackSumSkyfile). + + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/stack +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init stackSumSkyfile + +### Database lists +$stackSkycell_DB = 0 + +### Check status of stacking tasks +macro stack.status + book listbook stackSumSkyfile +end + +### Reset stacking tasks +macro stack.reset + book init stackSumSkyfile +end + +### Turn stacking tasks on +macro stack.on + task stack.skycell.load + active true + end + task stack.skycell.run + active true + end +end + +### Turn stacking tasks off +macro stack.off + task stack.skycell.load + active false + end + task stack.skycell.run + active false + end +end + + +### Load tasks for doing the stack +### Tasks are loaded into stackSumSkyfile. +task stack.skycell.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/stack.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command stacktool -tosum -limit 20 + else + # save the DB name for the exit tasks + option $DB:$stackSkycell_DB + command stacktool -tosum -limit 20 -dbname $DB:$stackSkycell_DB + $stackSkycell_DB ++ + if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout stackSumSkyfile -key stack_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook stackSumSkyfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup stackSumSkyfile + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the stack overlaps +### Tasks are taken from stackSumSkyfile. +task stack.skycell.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages stackSumSkyfile -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in stackSumSkyfile (pantaskState == INIT) + book getpage stackSumSkyfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword stackSumSkyfile $pageName pantaskState RUN + book getword stackSumSkyfile $pageName stack_id -var STACK_ID + book getword stackSumSkyfile $pageName workdir -var WORKDIR + book getword stackSumSkyfile $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on stacks + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/stack.%s" $outroot $STACK_ID + sprintf logfile "%s/skycell.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = stack_skycell.pl --stack_id $STACK_ID + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit stackSumSkyfile $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword stackSumSkyfile $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branch_20080121/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/summit.copy.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/summit.copy.pro (revision 16175) @@ -0,0 +1,355 @@ +## PanTasks scripts for Summit Copy + +module pantasks.pro + +# pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt +# pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + +# NOTE: workdir / volume mangling and nebulous. these tasks copy the +# imfiles from the summit, placing the output files in a directory +# which is based on the chip/host relationship. If nebulous is being +# used, the volume name is set based on the host; if nebulous is not +# being used, the workdir is set to include the host name. The copy +# operation is targetted to the same host by pcontrol. The value of +# workdir is set to be a template into which the appropriate value of +# @HOST@ may be substituted in later scripts. + +# the templates are used if we have a class_id/host relationship; if none is found, the default values are used +# XXX not sure how to handle the .N value if we need to use more than one +if ($NEBULOUS) + $workdir_default = neb:// + $workdir_template = neb:// + $volume_default = ipp004-v0 + $volume_template = @HOST@-v0 +else + $workdir_default = /data/ipp004.0/ + $workdir_template = /data/@HOST@.0/ +end + +$VERBOSE = 1 + +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end + +# list of DataStores to pull data from +book init pzDataStore +# list of summit exps that need to be queried +book init pzPendingExp +# list of summit imfiles that need to be downloaded +book init pzPendingImfile + +# build a book of datastores to poll for data +task pztool.datastore + command pztool -datastore + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 15 + periods -poll 1 + periods -timeout 15 + npending 1 + + # success + task.exit 0 + # flush pzDataStore book + book init pzDataStore + # convert 'stdout' to book format + ipptool2book stdout pzDataStore -key camera:telescope -uniq + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +$datastore_index = 0 + +# run pzgetexp periodically to populate pzPendingExp in the database (no I/O) +task pzgetexp + # timeout shorter than exec so jobs do not build up + periods -exec 5 + periods -poll 1 + periods -timeout 5 + host local + + task.exec + # find an exp that needs imfiles fetched + book getpage pzDataStore $datastore_index -var pageName + if ("$pageName" == "NULL") break + + # increment our pzDataStore index and loop back to 0 and the end of the + # book + $datastore_index ++ + book npages pzDataStore -var npages + if ($datastore_index == $npages ) + $datastore_index = 0 + end + + book getword pzDataStore $pageName camera -var CAMERA + book getword pzDataStore $pageName telescope -var TELESCOPE + book getword pzDataStore $pageName uri -var URI + + # store the current page + options $pageName + + $run = pzgetexp -uri $URI -inst $CAMERA -telescope $TELESCOPE + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + + command $run + end + + task.exit 0 + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# build a book of exps/filesetids that need to be queried +task pztool.pendingexp + command pztool -pendingexp -limit 25 + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 15 + periods -poll 1 + periods -timeout 15 + + # success + task.exit 0 + # convert 'stdout' to book format +# ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + +# XXX we probably want the database to be user-defined... + ipptool2book stdout pzPendingExp -key exp_name:camera:telescope -uniq -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingExp + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + +# run pzgetimfiles on pending exps +task pzgetimfile + periods -exec 0.05 + periods -poll 0.025 + periods -timeout 15 + host local + npending 20 + + task.exec + if ($network == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingExp -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingExp $pageName pantaskState RUN + + book getword pzPendingExp $pageName exp_name -var EXP_NAME + book getword pzPendingExp $pageName camera -var CAMERA + book getword pzPendingExp $pageName telescope -var TELESCOPE + book getword pzPendingExp $pageName dateobs -var DATEOBS + book getword pzPendingExp $pageName exp_type -var EXP_TYPE + book getword pzPendingExp $pageName uri -var URI + book getword pzPendingExp $pageName imfiles -var IMFILES + + # store the current page + options $pageName + + $batman = $EXP_NAME + $run = pzgetimfiles -uri $URI -filesetid $batman -inst $CAMERA -telescope $TELESCOPE + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingExp $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingExp $options:0 pantaskState TIMEOUT + end +end + + +# build a book of imfiles/files that need to be downloaded +task pztool.pendingimfile + command pztool -pendingimfile -limit 120 + host local + + # timeout shorter than exec so jobs do not build up + periods -exec 5 + periods -poll 1 + periods -timeout 5 + + # success + task.exit 0 + # convert 'stdout' to book format +# ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + + ipptool2book stdout pzPendingImfile -key exp_name:camera:telescope:class:class_id -uniq -setword pantaskState INIT + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingImfile + end + + task.exit default + showcommand failure + end + task.exit timeout + showcommand timeout + end +end + + +# get dsget to retreive an imfile and then call pztool -copydone +task dsget + periods -exec 5 + periods -poll 0.05 + periods -timeout 120 + + task.exec + if ($network == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingImfile -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingImfile $pageName pantaskState RUN + + book getword pzPendingImfile $pageName uri -var URI + book getword pzPendingImfile $pageName exp_name -var EXP_NAME + book getword pzPendingImfile $pageName class_id -var CLASS_ID + book getword pzPendingImfile $pageName bytes -var BYTES + book getword pzPendingImfile $pageName md5sum -var MD5SUM + book getword pzPendingImfile $pageName dateobs -var DATEOBS + book getword pzPendingImfile $pageName camera -var $CAMERA + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, workdir_default, volume_template, volume_default, + # it sets workdir and volume + set.workdir.by.camera $CAMERA $CLASS_ID + + # figure out filename + # XXXX this filename needs to use nebulous names + $FILENAME = $workdir/$CAMERA/$YEAR/$MONTH/$DAY/$EXP_NAME/$EXP_NAME.$CLASS_ID.fits + $FILEROOT = $workdir_template/$CAMERA/$YEAR/$MONTH/$DAY + + book setword pzPendingImfile $pageName filename $FILENAME + + set.host.by.camera $CAMERA $CLASS_ID + host anyhost + + # store the pageName with this job + options $pageName + options $FILEROOT + + $run = dsget --uri $URI --filename $FILENAME --bytes $BYTES + if ($MD5SUM) + $run = $run --md5 $MD5SUM + end + if ($NEBULOUS && ("$volname" != "NULL")) + $run = $run --volume $volume + end + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit 0 + # get related info from the associated page + book getword pzPendingImfile $options:0 exp_name -var EXP_NAME + book getword pzPendingImfile $options:0 class -var CLASS + book getword pzPendingImfile $options:0 camera -var CAMERA + book getword pzPendingImfile $options:0 telescope -var TELESCOPE + book getword pzPendingImfile $options:0 class_id -var CLASS_ID + book getword pzPendingImfile $options:0 filename -var FILENAME + + # we need to set the values of tess_id, dvodb, end_stage, workdir here + # these are functions of (at least) EXP_TYPE and SURVEY_ID, both of which should come from the data store + + # update the database with the success + $run = pztool -copydone -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -class $CLASS -class_id $CLASS_ID -uri $FILENAME -workdir $options:1 + # tess_id, dvodb, workdir, end_stage + + # create the command line + if ($VERBOSE > 1) + echo exec $run + end + exec $run + + # update the pantasks process tables + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + task.exit default + showcommand failure + process_exit pzPendingImfile $options:0 $JOB_STATUS + end + + task.exit timeout + showcommand timeout + book setword pzPendingImfile $options:0 pantaskState TIMEOUT + end +end Index: /branches/eam_branch_20080121/ippTasks/testscript.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/testscript.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/testscript.pro (revision 16175) @@ -0,0 +1,80 @@ +# test for pcontrol +macro test.pcontrol.simple + stop + for i 0 100 + job ls -R /etc + end + + status + host add alala + + status + run +end + +# test for pcontrol +macro test.pcontrol + exec p0tool -pendingimfile -simple | head -n 100 > list.dat + + stop + list line -x "cat list.dat" + echo $line:0 + for i 0 $line:n + list word -split $line:$i + job phase0_imfile.pl --exp_tag $word:0 --class_id fpa --uri $word:3 + end + + status + host add sn2 + host add sn3 + host add sn4 + host add sn5 + + status + run +end + +# test for pclient +macro test.pclient + exec p0tool -pendingimfile -simple | head -n 100 > list.dat + + list line -x "cat list.dat" + echo $line:0 + for i 0 $line:n + list word -split $line:$i + job phase0_imfile.pl --exp_tag $word:0 --class_id fpa --uri $word:3 + + for i 0 50 + check + output /dev/null + status + output stdout + if ($JOBSTATUS == EXIT) + $i = 50 + reset + echo "DONE" + else + usleep 200000 + end + end + if ($JOBSTATUS != EXIT) + echo TIMEOUT*** + end + end +end + +macro simple.test + for j 0 100 + job ls + for i 0 50 + check + status + if ($JOBSTATUS == EXIT) + $i = 50 + reset + else + usleep 10000 + end + end + end +end Index: /branches/eam_branch_20080121/ippTasks/warp.pro =================================================================== --- /branches/eam_branch_20080121/ippTasks/warp.pro (revision 16175) +++ /branches/eam_branch_20080121/ippTasks/warp.pro (revision 16175) @@ -0,0 +1,319 @@ +### This file contains panTasks definitions for performing the image warping. + +### This is done in two (main) steps. After a warp (with associated warp_id) is defined, +### overlaps between the exposure being warped and skycells are calculated (tasks in warpInputExp). +### Then for each skycell, the warp is made (tasks in warpPendingSkycell). + + +### Setups +if ($?network == 0) + $network = 1 +end +if ($?parallel == 0) + $parallel = 0 +end +if ($?VERBOSE == 0) + echo "VERBOSE not defined: load pantasks.pro first" + break +end +if ($?LOGDIR == 0) + echo "LOGDIR not defined: load pantasks.pro first" + break +end + +$LOGSUBDIR = $LOGDIR/warp +exec mkdir -p $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init warpInputExp +book init warpPendingSkyCell + +### Database lists +$warpExp_DB = 0 +$warpSkycell_DB = 0 + +### Check status of warping tasks +macro warp.status + book listbook warpInputExp + book listbook warpPendingSkyCell +end + +### Reset warping tasks +macro warp.reset + book init warpInputExp + book init warpPendingSkyCell +end + +### Turn warping tasks on +macro warp.on + task warp.exp.load + active true + end + task warp.exp.run + active true + end + task warp.skycell.load + active true + end + task warp.skycell.run + active true + end +end + +### Turn warping tasks off +macro warp.off + task warp.exp.load + active false + end + task warp.exp.run + active false + end + task warp.skycell.load + active false + end + task warp.skycell.run + active false + end +end + +### Load tasks for calculating the warp overlaps +### Tasks are loaded into warpInputExp. +task warp.exp.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/warp.exp.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -tooverlap -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpExp_DB + command warptool -tooverlap -limit 20 -dbname $DB:$warpExp_DB + $warpExp_DB ++ + if ($warpExp_DB >= $DB:n) set warpExp_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpInputExp -key warp_id:cam_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpInputExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpInputExp + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpInputExp. +task warp.exp.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages warpInputExp -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in warpInputExp (pantaskState == INIT) + book getpage warpInputExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpInputExp $pageName pantaskState RUN + book getword warpInputExp $pageName warp_id -var WARP_ID + book getword warpInputExp $pageName camera -var CAMERA + book getword warpInputExp $pageName workdir -var WORKDIR + book getword warpInputExp $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on warps + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/warp.%s" $outroot $WARP_ID + sprintf logfile "%s/overlap.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = warp_overlap.pl --warp_id $WARP_ID --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpInputExp $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpInputExp $options:0 pantaskState TIMEOUT + end +end + + +### Load tasks for doing the warps +### Tasks are loaded into warpPendingSkyCell. +task warp.skycell.load + host local + + periods -poll $loadpoll + periods -exec $loadexec + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/warp.skycell.log + + task.exec + if ($DB:n == 0) + option DEFAULT + command warptool -towarped -limit 20 + else + # save the DB name for the exit tasks + option $DB:$warpSkycell_DB + command warptool -towarped -limit 20 -dbname $DB:$warpSkycell_DB + $warpSkycell_DB ++ + if ($warpSkycell_DB >= $DB:n) set warpSkycell_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpPendingSkyCell -key warp_id:skycell_id:tess_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpPendingSkyCell + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpPendingSkyCell + end + + # locked list + task.exit default + showcommand failure + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for calculating the warp overlaps +### Tasks are taken from warpPendingSkyCell. +task warp.skycell.run + periods -poll $runpoll + periods -exec $runexec + periods -timeout 60 + + task.exec + book npages warpPendingSkyCell -var N + if ($N == 0) break + if ($network == 0) break + + # look for new images in warpPendingSkyCell (pantaskState == INIT) + book getpage warpPendingSkyCell 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpPendingSkyCell $pageName pantaskState RUN + book getword warpPendingSkyCell $pageName warp_id -var WARP_ID + book getword warpPendingSkyCell $pageName skycell_id -var SKYCELL_ID + book getword warpPendingSkyCell $pageName tess_id -var TESS_ID + book getword warpPendingSkyCell $pageName camera -var CAMERA + book getword warpPendingSkyCell $pageName workdir -var WORKDIR + book getword warpPendingSkyCell $pageName dbname -var DBNAME + + # specify choice of remote host + # XXX need to choose based on warps + if ($parallel) + host anyhost + else + host local + end + + ## generate output log based on filerule + if ("$WORKDIR" == "NULL") + $outroot = `pwd` + else + $outroot = `ipp_datapath.pl $WORKDIR` + end + sprintf outroot "%s/warp.%s" $outroot $WARP_ID + sprintf logfile "%s/skycell.log" $outroot + stdout $logfile + stderr $logfile + exec mkdir -p $outroot + + $run = warp_skycell.pl --warp_id $WARP_ID --skycell_id $SKYCELL_ID --tess_id $TESS_ID --camera $CAMERA + if ("$WORKDIR" != "NULL") + $run = $run --workdir $WORKDIR + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit warpPendingSkyCell $options:0 $JOB_STATUS + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpPendingSkyCell $options:0 pantaskState TIMEOUT + end +end +