Index: trunk/ippScripts/Build.PL =================================================================== --- trunk/ippScripts/Build.PL (revision 42730) +++ trunk/ippScripts/Build.PL (revision 42835) @@ -156,4 +156,9 @@ scripts/remote_md5sum.pl scripts/fpcamera_exp.pl + scripts/xcstack_skycell.pl + scripts/xcsky.pl + scripts/xccal.pl + scripts/xcff_warp.pl + scripts/xcff_summary.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- trunk/ippScripts/scripts/addstar_run.pl (revision 42730) +++ trunk/ippScripts/scripts/addstar_run.pl (revision 42835) @@ -30,10 +30,6 @@ my $missing_tools; my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); -my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); - -# these are only used by addstar_run, not addstar_multi -my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1); my $loadgalphot = can_run('loadgalphot') or (warn "Can't find loadgalphot" and $missing_tools = 1); @@ -44,5 +40,5 @@ my $minidvodb_path; -my ( $add_id, $camera, $stage, $stage_id, $stage_extra1, $outroot, $stageroot, $dbname, $reduction, +my ( $add_id, $camera, $stage, $stage_id, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps, $addrun_host); @@ -52,5 +48,4 @@ 'stage|s=s' => \$stage, # Camera 'stage_id|w=s' => \$stage_id, - 'stage_extra1|w=s' => \$stage_extra1, # the number for a staticskymulti (for finding cmf), or stack_id 'dbname|d=s' => \$dbname, # Database name 'outroot|w=s' => \$outroot, # output file base name @@ -83,14 +78,16 @@ defined $addrun_host and defined $camera; -if ($stage =~ /cam/ && !defined $stage_id) { - my_die("cam stage requires -stage_id", $add_id, 3); - -} -if ($minidvodb && !defined($minidvodb_group)) { - my_die( "missing minidvodb_group", $add_id, 3 ); - } + +my $dtime_addstar = 0; +my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + +# XXX EAM : 20240204 : not sure this is needed anymore: +if ($stage =~ /cam/ && !defined $stage_id) { my_die("cam stage requires -stage_id", $add_id, 3); } +if ($minidvodb && !defined($minidvodb_group)) { my_die( "missing minidvodb_group", $add_id, 3 ); } + my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); if ($redirect) { @@ -106,345 +103,178 @@ $ipprc->outroot_prepare($outroot); - -# # Step 1: sort out reductions / recipes -# - # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; -#if ($stage =~/diff/) { -# $reduction = 'ADDSTAR'; -#} -#if ($stage =~/fullforce/) { -# $reduction = 'ADDSTAR'; -#} - -if ($stage eq 'fullforce') { - $reduction='DEFAULT'; - #hardwired because why not -} -elsif ($stage eq 'fullforce_summary') { - $reduction = 'DEFAULT'; -} -elsif ($stage eq 'diff') { - $reduction='DEFAULT'; - #hardwired because why not -} - -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); -# XXX This is now not used: do we still need it? -if ($stage eq 'cam') { - $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -} -elsif ($stage eq 'stack') { - $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STACK'); # Recipe to use -} -elsif ($stage eq 'skycal') { - $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -} -#if ($stage =~/staticsky/) { -# $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STATICSKY'); # Recipe to use -#} - - + +my $recipe_addstar; + +# choose the recipe based on the stage: +# this probably should have different recipes for each possible stage +if ($stage eq 'cam') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } +if ($stage eq 'skycal') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } # Recipe to use +if ($stage eq 'diff') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } # Recipe to use +if ($stage eq 'fullforce') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } # Recipe to use +if ($stage eq 'fullforce_summary') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } # Recipe to use +if ($stage eq 'xcff') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } # Recipe to use +if ($stage eq 'xcff_summary') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); } # Recipe to use + +# XXX stack and staticsky are no longer allowed stages: need to use 'skycal' outputs +# if ($stage eq 'stack') { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STACK'); } # Recipe to use +# if ($stage =~/staticsky/) { $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STATICSKY'); } &my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; - # the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs -# # Step 2: Determine where sources should come from, modify them as needed. -# - # it was PSASTRO.OUTPUT my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); -my $fpaObjectsAlt = $fpaObjects; - -if ($stage eq 'skycal') { - #should be ok for skycal? - print "using $fpaObjects for $stage\n"; -} -elsif ($stage eq 'diff') { - print "using $fpaObjects for $stage\n"; -} -elsif ($stage eq 'fullforce') { - print "using $fpaObjects for $stage\n"; -} - -if ($stage eq 'cam') { - # if it is cam stage we need to be careful when grabbing the filename. - # This breaks down into a few steps: - - #get info about the cam_id - my $magicked; - { - my $mdcParser = PS::IPP::Metadata::Config->new; - my $command = "$camtool -processedexp -cam_id " . $stage_id; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success; - if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first - &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR); - print "listing nothing\n"; - } else { - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); - #this fails if there is nothing listed. I checked. - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - my $mparsed = $comp->{magicked}; - - if (!defined($mparsed)) { - &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR); - } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. - print "found a value for magicked:$mparsed\n"; - $magicked = $mparsed; - } - - #is this cam_id magicked or not? - if ($magicked) { - $stage_extra1 = $magicked; - $fpaObjects = $ipprc->destreaked_filename("$fpaObjects") or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - print "cam_id is magicked, using $fpaObjects for the cam smf\n"; - } else { - print "cam_id is NOT magicked, using $fpaObjects for the cam smf\n"; - } - } -} - -if (($stage eq 'stack') || - ($stage eq 'skycal') || - ($stage eq 'diff') || - ($stage eq 'fullforce')|| - ($stage eq 'fullforce_summary')) { - $fpaObjects =~ s/smf$/cmf/; - $fpaObjectsAlt =~ s/smf$/cmf/; -} - -my $fpaObjects1; -my $fpaObjects2; -my $checkalt = 0; -if ($stage eq 'staticsky') { - $checkalt = 1; - my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $stageroot); #this is mostly rigtht except the .cmf needs either - # .000.cmf or .stk.xxxxx.cmf - print "$sources\n\n\n"; - &my_die( "can't find the filter_num for staticsky_multi, giving up.", $add_id, $PS_EXIT_SYS_ERROR) unless (defined $stage_extra1); - - - $fpaObjects1 = $sources; - $fpaObjects2 = $sources; - $fpaObjects = $sources; - - my $nice_num = sprintf ("%03d", $stage_extra1); - - $fpaObjects1 =~ s/cmf$/stk.$stage_extra1.cmf/; - $fpaObjects2 =~ s/cmf$/$nice_num.cmf/; #this make it look for .001.cmf, etc - # we have 3 of them to try - my $realFile = $ipprc->file_resolve($fpaObjects); - my $realFile1 = $ipprc->file_resolve($fpaObjects1); - my $realFile2 = $ipprc->file_resolve($fpaObjects2); - if (!defined($realFile1)) { - if (!defined($realFile2)) { - print "using $fpaObjects\n"; - - } else { - print "using $fpaObjects2\n"; - $fpaObjects = $fpaObjects2; - } - - } else { - print "using $fpaObjects1\n"; - $fpaObjects = $fpaObjects1; - } -} - - -# + +# XXX EAM 2024.02.04 : I am stripping out support for magic here: + +# the following stages generate files with extension 'cmf' not 'smf': +if ($stage eq 'skycal') { $fpaObjects =~ s/smf$/cmf/; } +if ($stage eq 'diff') { $fpaObjects =~ s/smf$/cmf/; } +if ($stage eq 'fullforce') { $fpaObjects =~ s/smf$/cmf/; } +if ($stage eq 'fullforce_summary') { $fpaObjects =~ s/smf$/cmf/; } + +# XXX EAM 2024.02.04 : I am stripping out support for staticsky output here: + # Step 3 Set up addtool/loadgalphot commands. -# - -my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + # convert supplied DVO database name to UNIX filename my $dvodbReal; -if (defined $dvodb) { - $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO - $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); -} - -my $dtime_addstar = 0; -if (defined $dvodbReal) { - if ($minidvodb) { - my $command = "$addtool -listminidvodbrun "; - $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; - $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; - $command .= " -state 'active' -limit 1"; - $command .= " -dbname $dbname" if defined $dbname; - print $command; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); - - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - $minidvodb_path = $comp->{minidvodb_path}; - $minidvodb_name = $comp->{minidvodb_name}; - $addrun_host = $comp->{addrun_host}; - - if (!defined($minidvodb_path)) { - &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); +$dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO +$dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); +unless (defined $dvodbReal) { &my_die("dvodbReal not defined", $add_id, $PS_EXIT_CONFIG_ERROR); } ## XXX probably redundant with previous line + +# the supplied dvodb entry is not the real target if a minidvodb is supplied +if ($minidvodb) { + my $command = "$addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + + my $comp = $$components[0]; + $minidvodb_path = $comp->{minidvodb_path}; + $minidvodb_name = $comp->{minidvodb_name}; + $addrun_host = $comp->{addrun_host}; + + if (!defined($minidvodb_path)) { &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); } + if (!defined($minidvodb_name)) { &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); } +} else { + $minidvodb_path = $dvodbReal; +} + +if ($no_op) { &update_database; exit 0; } + +print "target CATDIR: $dvodbReal\n"; +## addstar can either save the full set of detections, or just +## the image metadata, in the dvodb. this is set in the +## database table addRun + +# addstar requires the user to have a valid .ptolemyrc which +# in turn points at ippconfig/dvo.site + +# get the names for the camera and the real input file +my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + +my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); + +# most cameras use CHIP_HEADER. other exceptions: +# gpc1 @ camera : PHU_HEADER +# gpc1 @ staticsky : NOMINAL (is uncalibrated) + +# temporary hard-wired info +my $zeroPointOption = "CHIP_HEADER"; +my $zeroPointKeyword = "ZPT_OBS"; + +if (($camdir =~ /gpc1/) && ($stage =~ /cam/)) { $zeroPointOption = "PHU_HEADER"; } +if ($stage eq "diff") { $zeroPointKeyword = "FPA.ZP"; } +if ($stage eq "xcff") { $zeroPointKeyword = "FPA.ZP"; } +if ($stage eq "xcff_summary") { $zeroPointKeyword = "FPA.ZP"; } + +my $command; + +if (($stage ne 'fullforce_summary') && ($stage ne 'xcff_summary')) { + print "Using command ADDSTAR\n"; + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -D CAMERA $camdir"; + $command .= " -D CATDIR $minidvodb_path"; + $command .= " -D ZERO_POINT_OPTION $zeroPointOption"; + $command .= " -D ZERO_POINT_KEYWORD $zeroPointKeyword"; + $command .= " $realFile"; + $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference + $command .= " -image" if $image_only; + + # some stages require alternate options: + if ($stage eq "skycal") { $command .= " -quick-airmass"; } + if ($stage eq "diff") { $command .= " -accept-astrom"; } + if ($stage eq "fullforce") { $command .= " -accept-astrom -xrad"; } + if ($stage eq "xcff") { $command .= " -accept-astrom -xrad"; } + + # if ($stage =~ /staticsky/) { $command .= " -accept-astrom -quick-airmass"; } + +} else { # Full force summary case + print "Using command LOADGALPHOT\n"; + + $command = "$loadgalphot -v "; + $command .= " -D CAMERA $camdir "; + $command .= " -D CATDIR $minidvodb_path "; + $command .= " $realFile "; + $command .= " -image-id $stage_id "; +} + +### actually run the addstar (or loadgalphot) command here: +my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); +my $error_code = &parse_error_message ($success, $error_msg, $command); +unless ($success) { &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); } + +$dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + +&update_database; +exit 0; + +sub update_database { + # This needs to be updated when addtool is written. BROKEN + my $fpaCommand = "$addtool -addprocessedexp"; + $fpaCommand .= " -add_id $add_id"; + $fpaCommand .= " -dtime_addstar $dtime_addstar"; + + $fpaCommand .= " -path_base $outroot"; + $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; + $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + $fpaCommand .= " -addrun_host $addrun_host" if defined $addrun_host; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + print $fpaCommand; + + # Add the result into the database + unless ($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $fpaCommand, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + warn("Unable to add result to database: $error_code\n"); + exit($error_code); } - if (!defined($minidvodb_name)) { - &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); - } + } else { + print "skipping command: $fpaCommand\n"; } - else { - $minidvodb_path = $dvodbReal; - } - - unless ($no_op) { - print "$dvodbReal\n"; - ## addstar can either save the full set of detections, or just - ## the image metadata, in the dvodb. this is set in the - ## database table addRun - - # addstar requires the user to have a valid .ptolemyrc which - # in turn points at ippconfig/dvo.site - - # get the names for the camera and the real input file - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - - my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); - - # most cameras use CHIP_HEADER. other exceptions: - # gpc1 @ camera : PHU_HEADER - # gpc1 @ staticsky : NOMINAL (is uncalibrated) - - # temporary hard-wired info - my $zeroPointOption = "CHIP_HEADER"; - my $zeroPointKeyword = "ZPT_OBS"; - if (($camdir =~ /gpc1/) && ($stage =~ /staticsky/)) { - $zeroPointOption = "NOMINAL"; - } - if (($camdir =~ /gpc1/) && ($stage =~ /cam/)) { - $zeroPointOption = "PHU_HEADER"; - } - if ($stage eq "diff") { - $zeroPointKeyword = "FPA.ZP"; - } - - my $command; - - if ($stage ne 'fullforce_summary') { -# print "IN ADDSTAR VERSION\n"; - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - $command = "$addstar -update"; # XXX optionally set -update? - $command .= " -force-single-time"; # IPP-676 time difference for chips causes dvo mosaic issue - $command .= " -D CAMERA $camdir"; - $command .= " -D CATDIR $minidvodb_path"; - $command .= " -D ZERO_POINT_OPTION $zeroPointOption"; - $command .= " -D ZERO_POINT_KEYWORD $zeroPointKeyword"; - $command .= " $realFile"; - $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference - $command .= " -image" if $image_only; - if ($stage =~ /staticsky/) { - $command .= " -accept-astrom -quick-airmass"; - } #careful here - this matches staticsky and staticsky_multi - if ($stage =~ /skycal/) { - $command .= " -quick-airmass"; - } #careful here - this matches staticsky and staticsky_multi - if ($stage =~ /diff/) { - $command .= " -accept-astrom"; - } - if ($stage =~ /fullforce/) { - $command .= " -accept-astrom -xrad"; - } - } - else { # Full force summary case -# print "IN LOADGALPHOT VERSION\n"; - # We need to know the filter to set up the photcode, as the summaries do not include that in the header. - # We /could/ set up the addtool stuff to pass that in to the script when needed, but that would require adding - # a lot of extra hooks and option handlers. Therefore, call out to fftool, and get the filter that way, and - # construct the correct photcode. - - my $photcode = ''; - { - my $ff_command = "$fftool -dbname ${dbname} -summary -ff_id ${stage_id}"; - print "$ff_command\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $ff_command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to fetch filter via fftool: $error_code", $add_id, $error_code); - } - my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to determine ff summary information.", $add_id, $error_code); - my $metadata = parse_md_list($MDlist); - my $ffSummary = $metadata->[0]; - my $filter = $ffSummary->{filter}; - my $stack_id = $ffSummary->{stack_id}; - - $filter =~ s/\.00000//; - $photcode = "GPC1.${filter}.ForcedWarp"; - } - - $command = "$loadgalphot -v "; - $command .= " -D CAMERA $camdir "; - $command .= " -D CATDIR $minidvodb_path "; - $command .= " $realFile "; - $command .= " -p $photcode "; - $command .= " -image-id $stage_id "; - } - - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script - } -} - -# This needs to be updated when addtool is written. BROKEN -my $fpaCommand = "$addtool -addprocessedexp"; -$fpaCommand .= " -add_id $add_id"; -$fpaCommand .= " -dtime_addstar $dtime_addstar"; - -$fpaCommand .= " -path_base $outroot"; -$fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; -$fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; -$fpaCommand .= " -addrun_host $addrun_host" if defined $addrun_host; -$fpaCommand .= " -dbname $dbname" if defined $dbname; - -$fpaCommand .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; - -print $fpaCommand; - -# Add the result into the database -unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $fpaCommand, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to add result to database: $error_code\n"); - exit($error_code); - } -} else { - print "skipping command: $fpaCommand\n"; -} - +} sub my_die @@ -474,4 +304,31 @@ } +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + END { my $status = $?; Index: trunk/ippScripts/scripts/dist_bundle.pl =================================================================== --- trunk/ippScripts/scripts/dist_bundle.pl (revision 42730) +++ trunk/ippScripts/scripts/dist_bundle.pl (revision 42835) @@ -73,7 +73,10 @@ my %empty_cleaned = (); +# no longer needed: magic is no longer used +# my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); + # Look for programs we need my $missing_tools; -my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); +my $streaksrelease = '/bin/false'; my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); Index: trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- trunk/ippScripts/scripts/stack_skycell.pl (revision 42730) +++ trunk/ippScripts/scripts/stack_skycell.pl (revision 42835) @@ -185,6 +185,5 @@ my $recipe_psphot = $ipprc->reduction($reduction, 'STACK_PSPHOT'); # Recipe to use for psphot my $recipe_ppstats = 'STACKSTATS'; -if ($ -run_state eq 'update') { +if ($run_state eq 'update') { $recipe_ppstats = 'WARPSTATS'; } Index: trunk/ippScripts/scripts/staticsky.pl =================================================================== --- trunk/ippScripts/scripts/staticsky.pl (revision 42730) +++ trunk/ippScripts/scripts/staticsky.pl (revision 42835) @@ -162,4 +162,5 @@ print "input: $path_base\n"; my $stack_id = $file->{stack_id}; + print "stack_id: $stack_id\n"; my ($imageCnv, $maskCnv, $weightCnv, $expnumCnv); Index: trunk/ippScripts/scripts/xccal.pl =================================================================== --- trunk/ippScripts/scripts/xccal.pl (revision 42835) +++ trunk/ippScripts/scripts/xccal.pl (revision 42835) @@ -0,0 +1,322 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $xcskytool = can_run('xcskytool') or (warn "Can't find xcskytool" and $missing_tools = 1); +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $fhead = can_run('fhead') or (warn "Can't find fhead" and $missing_tools = 1); +my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +sub HelpMessage { + print "USAGE: xccal.pl --xccal_id (xccal_id) --outroot (root) --camera (camera) [options]\n"; + print " --xccal_id (xccal_id) : run identifier\n"; + print " --xcstack_id (xcstack_id) : run identifier\n"; + print " --camera (camera) : Camera name\n"; + print " --dbname (dbname) : Database name\n"; + print " --threads (N) : Number of threads to use\n"; + print " --outroot (root) : Output root name\n"; + print " --path_base (path_base) : path_base of input\n"; + print " --reduction (class) : Reduction class\n"; + print " --verbose : Be extra verbose\n"; + print " --no-update : Don't update the database?\n"; + print " --no-op : Don't do any operations?\n"; + print " --redirect-output : send output to the log file,\n"; + print " --save-temps : Save temporary files?\n"; + print " --help : show this message\n"; + exit 2; +} + +my ($xccal_id, $xcstack_id, $camera, $dbname, $threads, $outroot, $path_base, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +my $run_state = 'new'; +GetOptions( + 'xccal_id=s' => \$xccal_id, # xcsky run id + 'xcstack_id=s' => \$xcstack_id,# xcstack run id + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'run-state=s' => \$run_state, # 'new' or 'update' + 'path_base=s' => \$path_base, # path_base of input + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + 'help' => sub { HelpMessage() }, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --xccal_id --xcstack_id --outroot --camera --path_base", + -exitval => 3, + ) unless + defined $xccal_id and + defined $xcstack_id and + defined $path_base and + defined $outroot and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $xccal_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $xccal_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $stats = $ipprc->filename("PSASTRO.STATS", $outroot); + +my $outbase = basename($outroot); + +# Get list of input images to xcstack photometry +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +# generate the input +my $cmdflags = ""; + +unless ($run_state eq 'new' or $run_state eq 'update') { + &my_die("$run_state is not a valid value for run-state\n", $xccal_id, $PS_EXIT_CONFIG_ERROR); +} +my $updateMode = $run_state eq 'update'; + +my $recipe_psastro = $ipprc->reduction($reduction, 'STATICSKY_CALIBRATION'); # Recipe to use for psastro +unless ($recipe_psastro) { + &my_die("Couldn't find selected reduction for STATICSKY_CALIBRATION: $reduction\n", $xccal_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: We are making the bad assumption here that the staticksky run used the same reduction class +# as this xccalRun +my $recipe_psphot = $ipprc->reduction($reduction, "STACKPHOT_PSPHOT"); +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT_PSPHOT: $reduction\n", $xccal_id, $PS_EXIT_CONFIG_ERROR); +} + +print "reduction: $reduction\n"; +print "recipe_psastro: $recipe_psastro\n"; +print "recipe_psphot: $recipe_psphot\n"; + +my $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot); + +# find input file +# First check the expected file name where the xcstack_id is the FILE_ID + +my $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $xcstack_id); +if (! $ipprc->file_resolve($file)) { + print "\nfailed to resolve $file, checking for a single-filter stack\n"; + # input is from a single filter static sky run which used psphot instead of psphotStack + # The file rule is different for psphot + $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); + &my_die("Unable to find input for $xcstack_id", $xccal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($file); +} +if ($file) { + print "\nInput file for $xcstack_id: $file\n"; +} else { + &my_die("Unable to find input file for xcstack_id $xcstack_id.", $xccal_id, $PS_EXIT_SYS_ERROR); +} + +my $resolved = $ipprc->file_resolve($file); +if (!$resolved) { + &my_die("Unable to find input file: $file", $xccal_id, $PS_EXIT_SYS_ERROR); +} + +# Perform psastro analysis +my $output_sources_filerule = 'PSASTRO.OUTPUT.CMF'; +my $quality = 0; +{ + my $command = "$psastro $outroot"; + $command .= " -file $file"; + $command .= " -recipe PSASTRO $recipe_psastro"; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -F PSASTRO.OUTPUT $output_sources_filerule"; + $command .= " -Db PSASTRO.SAVE.CFF F" if $updateMode; + $command .= " -stats $stats"; + $command .= " -recipe PPSTATS SKYCALSTATS"; + $command .= " -dumpconfig $configuration" if $configuration; + $command .= " -tracedest $traceDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xccal_id, $error_code); + } + + &my_die("Couldn't find stats file: $stats", $xccal_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($stats); + + { + my $resolved = $ipprc->file_resolve($stats) + or &my_die("failed to resolve stats file: $stats", $xccal_id, $PS_EXIT_SYS_ERROR) ; + my $command = "$ppStatsFromMetadata $resolved - STATICSKY_CAL"; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xccal_id, $error_code); + } + $cmdflags = join "", @$stdout_buf; chomp $cmdflags; + } + + ($quality) = $cmdflags =~ /-quality (\d+)/; + if (!$quality) { + my $outputSources = $ipprc->filename($output_sources_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputSources", $xccal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($outputSources); + + &my_die("Couldn't find expected output file: $configuration", $xccal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($configuration); + } + } else { + print "Not executing: $command\n"; + } +} + +# Add the result to the database +if (!$updateMode) { + my $command = "$xcskytool -xccal_id $xccal_id"; + $command .= " -addxccalresult -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xccal_id, $error_code); + } + } else { + print "skipping $command\n"; + } +} else { + if ($quality) { + # First update the result + my $command = "$xcskytool -updatexccal -xccal_id $xccal_id -set_fault 0"; + $command .= " -set_quality $quality"; + $command .= " -dbname $dbname" if defined $dbname; + + unless($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xccal_id, $error_code); + } + } else { + print "skipping $command\n"; + } + } + my $command = "$xcskytool -xccal_id $xccal_id"; + $command .= ' -updatexccalrun -set_state full'; + $command .= " -dbname $dbname" if defined $dbname; + + unless($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xccal_id, $error_code); + } + } else { + print "skipping $command\n"; + } +} + +exit 0; + +sub my_die +{ + my $msg = shift; # Warning message on die + my $xccal_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $xccal_id and not $no_update) { + my $command = "$xcskytool -xccal_id $xccal_id"; + if ($updateMode) { + $command .= " -updatexccal -set_fault $exit_code"; + } else { + $command .= " -addxccalresult -fault $exit_code"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + } + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: trunk/ippScripts/scripts/xcff_summary.pl =================================================================== --- trunk/ippScripts/scripts/xcff_summary.pl (revision 42835) +++ trunk/ippScripts/scripts/xcff_summary.pl (revision 42835) @@ -0,0 +1,325 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); +use File::Temp qw( tempfile ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $psphotFullForceSummary = can_run('psphotFullForceSummary') or (warn "Can't find psphotFullForceSummary" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $xcfftool = can_run('xcfftool') or (warn "Can't find xcfftool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($xcff_id, $outroot, $reduction, $camera); +my ($dbname, $threads, $verbose, $save_temps, $no_update, $no_op, $redirect); + +GetOptions( + 'xcff_id=s' => \$xcff_id, + 'camera=s' => \$camera, # camera name of sources + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --xcff_id --outroot --camera", + -exitval => 3, + ) + unless defined $xcff_id, + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $xcff_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", + $xcff_id, $PS_EXIT_SYS_ERROR ) if $redirect; + + +my ($listFile, $listName) = tempfile("/tmp/fullforce.summary.list.XXXX", UNLINK => !$save_temps ); + +my $cff_file; + +{ + my $mdcParser = PS::IPP::Metadata::Config->new; + my $results; + { + my $command = "$xcfftool -result -xcff_id $xcff_id -quality 0"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $xcff_id, $PS_EXIT_PROG_ERROR); + $results = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $xcff_id, $PS_EXIT_PROG_ERROR); + } + + &my_die("result list is empty.", $xcff_id, $PS_EXIT_SYS_ERROR) + if scalar @$results == 0; + + print $listFile "SOURCES MULTI\n"; + foreach my $result (@$results) { + my $cmf = $ipprc->filename('PSPHOT.FULLFORCE.OUTPUT', $result->{path_base}); + &my_die("Couldn't find input cmf: $cmf", $xcff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cmf); + print $listFile "SOURCES STR $cmf\n"; + if (!$cff_file) { + $cff_file = $ipprc->filename('PSPHOT.OUTPUT.CFF', $result->{sources_path_base}); + &my_die("Couldn't find input cff: $cff_file", $xcff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($cff_file); + } + } + close $listFile; +} + +&my_die("No CFF found in results: $cff_file", $xcff_id, $PS_EXIT_PROG_ERROR) unless $cff_file; + + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", + $xcff_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: need to figure out whether this works or not +# We probably want to create a specific recipe that looks at the results +my $recipe_ppstats = 'WARPSTATS'; +my $doStats = 0; + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +my $dump_config = 1; + +# Get the output filenames +my $outputSources = prepare_output("PSPHOT.FULLFORCE.OUTPUT", $outroot, 1); +my $configuration = prepare_output("PSPHOT.SKY.CONFIG", $outroot, 1) if $dump_config; +my $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1) if $doStats; +my $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + +my $cmdflags = ""; + +# Perform psphotFullForceSummary +{ + my $command = "$psphotFullForceSummary $outroot"; + $command .= " -input $listName"; + $command .= " -cff $cff_file"; + $command .= " -threads $threads" if defined $threads; + if ($dump_config) { + $command .= " -dumpconfig $configuration"; + } + $command .= " -recipe PSPHOT $recipe_psphot"; + if ($doStats) { + $command .= " -stats $outputStats"; + $command .= " -recipe PPSTATS $recipe_ppstats"; + } +# $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $error_code); + } + + # Stats: TODO + if ($doStats) { + check_output($outputStats, 1); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + + # measure chip stats + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; + ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + } + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + check_output($outputSources, 1); + } + } else { + print "Not executing: $command\n"; + } +} + +# Add the result to the database +{ + my $command = "$xcfftool -xcff_id $xcff_id"; + $command .= " -addsummary -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $error_code); + } + } else { + print "Not executing $command\n"; + } +} + +exit 0; + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $xcff_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $xcff_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $xcff_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $check_command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $check_command); + unless ($success) { + &my_die("Unable to perform $check_command: $error_msg", $xcff_id, $error_code); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $xcff_id, $PS_EXIT_SYS_ERROR); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $xcff_id = shift; # full force run identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $xcff_id) { + my $command = "$xcfftool -xcff_id $xcff_id -fault $exit_code"; + $command .= " -addsummary"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + run(command => $command, verbose => $verbose); + } else { + print "not executing $command\n"; + } + } + exit $exit_code; +} + +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: trunk/ippScripts/scripts/xcff_warp.pl =================================================================== --- trunk/ippScripts/scripts/xcff_warp.pl (revision 42835) +++ trunk/ippScripts/scripts/xcff_warp.pl (revision 42835) @@ -0,0 +1,331 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $psphotFullForce = can_run('psphotFullForce') or (warn "Can't find psphotFullForce" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $xcfftool = can_run('xcfftool') or (warn "Can't find xcfftool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($xcff_id, $warp_id, $skycell_id, $path_base, $sourceroot, $camera); +my ($outroot, $reduction); +my ($dbname, $threads, $verbose, $no_update, $no_op, $redirect); + +sub HelpMessage { + print "USAGE: xcff_warp.pl --xcff_id (xccal_id) --outroot (root) --camera (camera) [options]\n"; + print " --xcff_id (xccal_id) : run identifier\n"; + print " --warp_id (warp_id) : warp identifier\n"; + print " --skycell_id (skycell_id) : skycell identifier\n"; + print " --warp_path_base (path_base) : path_base of input\n"; + print " --sources_path_base (path_base) : path_base of input\n"; + print " --camera (camera) : Camera name\n"; + print " --dbname (dbname) : Database name\n"; + print " --threads (N) : Number of threads to use\n"; + print " --outroot (root) : Output root name\n"; + print " --reduction (class) : Reduction class\n"; + print " --verbose : Be extra verbose\n"; + print " --no-update : Don't update the database?\n"; + print " --no-op : Don't do any operations?\n"; + print " --redirect-output : send output to the log file,\n"; + print " --help : show this message\n"; + exit 2; +} + +GetOptions( + 'xcff_id=s' => \$xcff_id, + 'warp_id=s' => \$warp_id, # warp identifier + 'skycell_id=s' => \$skycell_id,# Skycell identifier + 'warp_path_base=s' => \$path_base, # path_base of the warp skycell --- could get this from the db + 'sources_path_base=s' => \$sourceroot,# path_base of sources --- could get this from the db + 'camera=s' => \$camera, # camera name of sources + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'help' => sub { HelpMessage() }, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --xcff_id --warp_id --sources_path_base --skycell_id --warp_path_base --outroot --camera", -exitval => 3, + ) unless defined $xcff_id + and defined $warp_id + and defined $sourceroot + and defined $skycell_id + and defined $path_base + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { $neb = $ipprc->nebulous(); } + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id); + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_psphot = $ipprc->reduction($reduction, 'FULLFORCE_PSPHOT'); # Recipe to use for psphot +unless ($recipe_psphot) { &my_die("Couldn't find selected reduction for PSPHOT: $reduction\n", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); } + +my $doStats = 1; + +print "reduction: $reduction\n"; +print "recipe_psphot: $recipe_psphot\n"; + +# XXX: get this from recipe +# use psf measured on input warp (if false, PSF is generated by psphotFullForce?) +my $useWarpPSF = 0; + +my $input = $ipprc->filename('PSWARP.OUTPUT', $path_base); +my $inputMask = $ipprc->filename('PSWARP.OUTPUT.MASK', $path_base); +my $inputVariance = $ipprc->filename('PSWARP.OUTPUT.VARIANCE', $path_base); +my $inputPSF = $useWarpPSF ? $ipprc->filename('PSPHOT.PSF.SKY.SAVE', $path_base) : ""; +my $inputSources = $ipprc->filename('PSPHOT.OUTPUT.CFF', $sourceroot); + +if ($verbose) { + print "input: $input\n"; + print "inputMask: $inputMask\n"; + print "inputVariance: $inputVariance\n"; + print "inputPSF: $inputPSF\n" if $inputPSF; + print "inputSources: $inputSources\n"; +} + +# check that the inputs exist (and have non-zero size) +&my_die("Couldn't find input: $input", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input); +&my_die("Couldn't find input: $inputMask", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask); +&my_die("Couldn't find input: $inputVariance", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance); +&my_die("Couldn't find input: $inputPSF", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) if ($inputPSF && !$ipprc->file_exists($inputPSF)); +&my_die("Couldn't find input: $inputSources", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources); + +my $dump_config = 1; + +# Get the output filenames +my $outputSources = prepare_output("PSPHOT.OUT.CMF.MEF", $outroot, 1); +my $configuration = prepare_output("PSPHOT.SKY.CONFIG", $outroot, 1) if $dump_config; +my $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1) if $doStats; +my $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + +my $cmdflags = ""; +my $quality = 0; + +# Perform psphotFullForce +{ + my $command = "$psphotFullForce $outroot"; + $command .= " -force $inputSources"; + $command .= " -file $input"; + $command .= " -mask $inputMask"; + $command .= " -variance $inputVariance"; + $command .= " -psf $inputPSF" if $inputPSF; + $command .= " -threads $threads" if defined $threads; + if ($dump_config) { + $command .= " -dumpconfig $configuration"; + } + $command .= " -recipe PSPHOT $recipe_psphot"; + if ($doStats) { + $command .= " -stats $outputStats"; + } + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF"; + $command .= " -photcode-rule '{DETECTOR}.{FILTER.ID}.ForcedWarp'"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $warp_id, $skycell_id, $error_code); + } + + if ($doStats) { + check_output($outputStats, 1); + my $outputStatsReal = $ipprc->file_resolve($outputStats); + + # analyze the stats file + $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL"; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $warp_id, $skycell_id, $error_code); + } + foreach my $line (@$stdout_buf) { $cmdflags .= " $line"; } + chomp $cmdflags; + } + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + } else { + print "Not executing: $command\n"; + } +} + +if (!$quality) { + check_output($outputSources, 1); +} + +# Add the result to the database +{ + my $command = "$xcfftool -xcff_id $xcff_id -warp_id $warp_id"; + $command .= " -addresult -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcff_id, $warp_id, $skycell_id, $error_code); + } + } else { + print "Not executing $command\n"; + } +} + +exit 0; + + +# Prepare to write to an output file +# Lookup the filename in the rules. +# Make sure that if file exists and is a nebulous file that there is only one instance +# Deal with files that have been lost. +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $xcff_id, $warp_id, $skycell_id, $error); + return $output; +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $xcff_id, $warp_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $xcff_id = shift; # full force run identifier + my $warp_id = shift; # full force warp id + my $skycell_id = shift; # Skycell identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $xcff_id and defined $skycell_id) { + my $command = "$xcfftool -xcff_id $xcff_id -warp_id $warp_id -fault $exit_code"; + $command .= " -addresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + run(command => $command, verbose => $verbose); + } else { + print "not executing $command\n"; + } + } + exit $exit_code; +} + +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: trunk/ippScripts/scripts/xcsky.pl =================================================================== --- trunk/ippScripts/scripts/xcsky.pl (revision 42835) +++ trunk/ippScripts/scripts/xcsky.pl (revision 42835) @@ -0,0 +1,434 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $xcskytool = can_run('xcskytool') or (warn "Can't find xcskytool" and $missing_tools = 1); +my $psphotStack = can_run('psphotStack') or (warn "Can't find psphotStack" and $missing_tools = 1); +my $psphot = can_run('psphot') or (warn "Can't find psphot" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +sub HelpMessage { + print "USAGE: xcsky.pl --xcsky_id (xcsky_id) --outroot (root) --camera (camera) [options]\n"; + print " --xcsky_id (xcsky_id) : run identifier\n"; + print " --run-state (state) : run state (new or update)\n"; + print " --camera (camera) : Camera name\n"; + print " --dbname (dbname) : Database name\n"; + print " --threads (N) : Number of threads to use\n"; + print " --outroot (root) : Output root name\n"; + print " --reduction (class) : Reduction class\n"; + print " --require_sources : Require that the stack source files exist\n"; + print " --verbose : Be extra verbose\n"; + print " --no-update : Don't update the database?\n"; + print " --no-op : Don't do any operations?\n"; + print " --redirect-output : send output to the log file,\n"; + print " --save-temps : Save temporary files?\n"; + print " --help : show this message\n"; + exit 2; +} + +# XXX test: +print "run xcsky.pl @ARGV\n"; + +my ($xcsky_id, $camera, $dbname, $threads, $outroot, $reduction, $require_sources, $verbose, $no_update, $no_op, $redirect, $save_temps); +my $run_state ='new'; + +GetOptions( + 'xcsky_id=s' => \$xcsky_id, # Diff identifier + 'run-state=s' => \$run_state, + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'reduction=s' => \$reduction, # Reduction class + 'require_sources' => \$require_sources, # Require that the stack sources exist. + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + 'help' => sub { HelpMessage() }, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --xcsky_id --outroot --camera", + -exitval => 3, + ) unless + defined $xcsky_id and + defined $outroot and + defined $camera; + + +&my_die("unexpected run state: $run_state", $xcsky_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new') or ($run_state eq 'update'); +my $updatemode = $run_state eq 'update'; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $xcsky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $xcsky_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +# my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_XCSKY); + +my $outbase = basename($outroot); +my ($listFile, $listName) = tempfile("/tmp/$outbase.list.XXXX", UNLINK => !$save_temps ); + +# Get list of input images to stack photometry +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; +{ + my $command = "$xcskytool -inputs -xcsky_id $xcsky_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcsky_id, $error_code); + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $xcsky_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $xcsky_id, $PS_EXIT_PROG_ERROR); +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + + +# generate the input +print $listFile "INPUT MULTI\n"; +my $nInputs = @$files; + +{ + my $recipe_psphot = $ipprc->reduction($reduction, 'XCSTACK_PSPHOT'); # Recipe to use for psphot + # my $recipe_ppsub = $ipprc->reduction($reduction, 'XCSTACK_PPSUB'); # Recipe to use for ppsub + # my $recipe_ppstack = $ipprc->reduction($reduction, 'XCSTACK_PPSTACK'); # Recipe to use for ppstack + unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for XCSTACK_PSPHOT: $reduction\n", $xcsky_id, $PS_EXIT_CONFIG_ERROR); + } + + print "reduction: $reduction\n"; + print "recipe_psphot: $recipe_psphot\n"; + # print "recipe_ppsub: $recipe_ppsub\n"; + # print "recipe_ppstack: $recipe_ppstack\n"; + my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); + + # XXX this script and psphotStack use the recipe file inconsistently: + # XXX psphotStack respects the value of PSPHOT.STACK.USE.RAW from the config, + # XXX but this script ignores it: if PSPHOT.STACK.USE.RAW is FALSE (needConvolvedImages = 1) + # XXX then psphotStack will expect entries in the input mdc file (CNV:) that this script + # XXX is not supplying. + my $needConvolvedImages = 0; + + foreach my $file (@$files) { + print $listFile "INPUT METADATA\n"; + + # XXX if we take the input from 'warp', we will need to make different selections here + my $path_base = $file->{path_base}; + print "input: $path_base\n"; + my $xcstack_id = $file->{xcstack_id}; + + my ($imageCnv, $maskCnv, $weightCnv, $expnumCnv); + if ($needConvolvedImages) { + $imageCnv = $ipprc->filename("PPSTACK.OUTPUT", $path_base ); # Image name + $maskCnv = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base ); # Mask name + $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name + $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM", $path_base ); # Expnum name + } + + my $imageRaw = $ipprc->filename("PPSTACK.UNCONV", $path_base ); # Image name + my $maskRaw = $ipprc->filename("PPSTACK.UNCONV.MASK", $path_base ); # Mask name + my $weightRaw = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $path_base ); # Weight name + my $expnumRaw = $ipprc->filename("PPSTACK.UNCONV.EXPNUM", $path_base ); # Expnum name + + my $sources; + my ($psfRaw, $backmdlRaw); + if (!$updatemode) { + # sources from stack run. + # XXX: We don't need this file anymore. + $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name + } else { + # name of output sources from this run when in new state + $require_sources = 1 unless $no_op; + $sources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $xcstack_id); + # During update we rename the sources file to this + my $originalSources = "$sources.original"; + if (!$ipprc->file_exists($originalSources)) { + if ($ipprc->file_exists($sources)) { + unless ($no_op) { + if (!$ipprc->file_rename($sources, $originalSources)) { + &my_die("failed to rename $sources $originalSources", $xcsky_id, $PS_EXIT_PROG_ERROR); + } + } + } else { + # maybe this should be a fault 2 + &my_die("failed to to find either sources file to update as either $sources or $originalSources", $xcsky_id, $PS_EXIT_PROG_ERROR); + } + } + $sources = $originalSources; + $psfRaw = $ipprc->filename("PSPHOT.STACK.PSF.SAVE", $outroot, $xcstack_id); + $backmdlRaw = $ipprc->filename("PSPHOT.STACK.BACKMDL", $outroot, $xcstack_id); + } + + + # XXX is this the correct PSF file? + my $psfCnv = $ipprc->filename("PPSTACK.TARGET.PSF", $path_base ); # PSF name + + # XXX we could make some different choices if some inputs do not exist... + &my_die("Couldn't find input: $imageRaw", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageRaw"); + &my_die("Couldn't find input: $maskRaw", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskRaw"); + &my_die("Couldn't find input: $weightRaw", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw"); + &my_die("Couldn't find input: $expnumRaw", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumRaw"); + if ($needConvolvedImages) { + &my_die("Couldn't find input: $imageCnv", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv"); + &my_die("Couldn't find input: $maskCnv", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv"); + &my_die("Couldn't find input: $weightCnv", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv"); + &my_die("Couldn't find input: $expnumCnv", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv"); + &my_die("Couldn't find input: $psfCnv", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); + } + + my $have_sources = $ipprc->file_exists("$sources"); + if ($require_sources) { + &my_die("Couldn't find input: $sources", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $have_sources; + } + + print $listFile " STACK_ID S64 " . $xcstack_id . "\n"; + print $listFile " RAW:IMAGE STR " . $imageRaw . "\n"; + print $listFile " RAW:MASK STR " . $maskRaw . "\n"; + print $listFile " RAW:VARIANCE STR " . $weightRaw . "\n"; + print $listFile " RAW:EXPNUM STR " . $expnumRaw . "\n"; + if ($have_sources) { + print $listFile " SOURCES STR " . $sources . "\n"; + } + if ($updatemode) { + &my_die("Couldn't find input: $psfRaw", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfRaw"); + &my_die("Couldn't find input: $backmdlRaw", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$backmdlRaw"); + print $listFile " RAW:PSF STR " . $psfRaw . "\n"; + print $listFile " RAW:BACKMDL STR " . $backmdlRaw . "\n"; + } + if ($needConvolvedImages) { + print $listFile " CNV:IMAGE STR " . $imageCnv . "\n"; + print $listFile " CNV:MASK STR " . $maskCnv . "\n"; + print $listFile " CNV:VARIANCE STR " . $weightCnv . "\n"; + print $listFile " CNV:EXPNUM STR " . $expnumCnv . "\n"; + print $listFile " CNV:PSF STR " . $psfCnv . "\n"; + } + + print $listFile "END\n\n"; + } + + close $listFile; + + # Perform stack photometry analysis + { + my $command = "$psphotStack $outroot"; + $command .= " -updatemode" if $updatemode; + $command .= " -input $listName"; + $command .= " -threads $threads" if defined $threads; + $command .= " -recipe PSPHOT $recipe_psphot"; + # $command .= " -recipe PPSUB $recipe_ppsub"; + # $command .= " -recipe PPSTACK $recipe_ppstack"; + $command .= " -dumpconfig $configuration" if !$updatemode; + $command .= " -tracedest $traceDest -log $logDest"; + # $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_op) { + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform psphotStack: $error_msg", $xcsky_id, $error_code); + } + + # XXX need to enable the STATS output in psphotStack, then read the results here: + # my $outputStatsReal = $ipprc->file_resolve($outputStats); + # &my_die("Couldn't find expected output file: $outputStats", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + # measure chip stats + # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; + # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + # run(command => $command, verbose => $verbose); + # unless ($success) { + # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + # &my_die("Unable to perform ppStatsFromMetadata: $error_code", $xcsky_id, $error_code); + # } + # foreach my $line (@$stdout_buf) { + # $cmdflags .= " $line"; + # } + # chomp $cmdflags; + + # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + my $quality = 0; + if (!$quality) { + + # Get the output filenames + # we have one set of output files per input file set + for (my $i = 0; $i < @$files; $i++) { + my $xcstack_id = @$files[$i]->{xcstack_id}; + my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $xcstack_id); + my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $xcstack_id); + my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $xcstack_id); + my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $xcstack_id); + print "xcstack_id: $xcstack_id\n"; + print "outroot: $outroot\n"; + print "src: $outputSources\n"; + + # XXX these are optional and not generated by default + # &my_die("Couldn't find expected output file: $outputName", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); + # &my_die("Couldn't find expected output file: $outputMask", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); + # &my_die("Couldn't find expected output file: $outputVariance", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); + &my_die("Couldn't find expected output file: $outputSources", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + } + + #my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); + #my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); + #my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); + + my $chisqName = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot); + my $chisqMask = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot); + my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot); + + # XXX check the recipe -- should we expect these to exist? + # &my_die("Couldn't find expected output file: $chisqName", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName); + # &my_die("Couldn't find expected output file: $chisqMask", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask); + # &my_die("Couldn't find expected output file: $chisqVariance", $xcsky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance); + + if (!$updatemode) { + # XXX: should be checking for existence of psf and backmdl output files + } + } + } else { + print "Not executing: $command\n"; + } + } +} + +unless ($no_update) { + + # Add the xcsky result + { + my $command = "$xcskytool -xcsky_id $xcsky_id"; + $command .= " -dbname $dbname" if defined $dbname; + if ($updatemode) { + $command .= " -updaterun -set_state full"; + } else { + $command .= " -addresult -path_base $outroot"; + $command .= " -num_inputs $nInputs"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + } + + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + warn("Unable to perform: $command\n"); + exit $error_code; + } + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $xcsky_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $xcsky_id and not $no_update) { + + my $command = "$xcskytool -xcsky_id $xcsky_id -fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + if ($updatemode) { + $command .= " -updateresult"; + } else { + $command .= " -addresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + } + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ + +### + +# possible error messages from IPC::Cmd: +# "Failed to execute COMMAND : MESSAGE" +# "COMMAND died with signal NN, with coredump" +# "COMMAND died with signal NN, without coredump" +# "COMMAND exited with value NN" + +print "raw error_msg: $error_msg...\n"; +($error_code) = $error_msg =~ m/exited with value (\d+)/; Index: trunk/ippScripts/scripts/xcstack_skycell.pl =================================================================== --- trunk/ippScripts/scripts/xcstack_skycell.pl (revision 42835) +++ trunk/ippScripts/scripts/xcstack_skycell.pl (revision 42835) @@ -0,0 +1,532 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use Scalar::Util qw(looks_like_number); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $xcstacktool = can_run('xcstacktool') or (warn "Can't find xcstacktool" and $missing_tools = 1); +my $ppStack = can_run('ppStack') or (warn "Can't find ppStack" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($xcstack_id, $dbname, $camera, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'xcstack_id|d=s' => \$xcstack_id, # xcstack identifier + 'dbname|d=s' => \$dbname, # Database name + 'camera|c=s' => \$camera, # Camera name + 'outroot=s' => \$outroot, # Output root name + 'run-state=s' => \$run_state, # new or update? + 'debug' => \$debug, # Print to stdout + 'threads=s' => \$threads, # Number of threads to use for ppStack + 'reduction=s' => \$reduction, # Reduction class + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, # send output to the logfile + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --xcstack_id --outroot --run-state --dbname --camera", + -exitval => 3, + ) unless defined $xcstack_id and defined $outroot and defined $run_state and defined $dbname and defined $camera; + +# lists of file rules which we expect to produce output when convolving +my @outputList = qw( + PPSTACK.UNCONV + PPSTACK.UNCONV.MASK + PPSTACK.UNCONV.VARIANCE + PPSTACK.UNCONV.EXP + PPSTACK.UNCONV.EXPNUM + PPSTACK.UNCONV.EXPWT + PPSTACK.CONFIG +); + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $xcstack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$| = 1; +print "I've set up: $xcstack_id\n"; + +$ipprc->define_camera($camera); + +# XXX camera is not known here; cannot use filerules... +# my $logDest = $ipprc->filename("LOG.EXP", $outroot); +my $logDest = "$outroot.log"; + +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $xcstack_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $neb; +my $scheme = file_scheme($outroot); +if ($scheme and $scheme eq 'neb') { + $neb = $ipprc->nebulous(); +} + +my $temp_images_exist = 0; +my $image_id = $xcstack_id; +my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_XCSTACK); + +# temporary file to list the inputs (in metadata config format) +my ($listFile, $listName) = tempfile("/tmp/xcstk.$xcstack_id.list.XXXX", UNLINK => !$save_temps ); +my $num = 0; + +# Get list of cameras for this xcstack +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $cameras = &run_command_ipptool ("$xcstacktool -dbname $dbname -inputcameras -xcstack_id $xcstack_id"); +&my_die("no cameras listed for xcstack", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar @$cameras > 0; +print "I've loaded my cameras: $xcstack_id\n"; + +# Gather the list of input stacks from each of the cameras. There may be multiple stacks per camera +foreach my $cameraData (@$cameras) { + # Get list of components for xcstacking + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + + my $camera_name = $cameraData->{camera}; + + my $inputs = &run_command_ipptool ("$xcstacktool -dbname $dbname -inputskyfile -xcstack_id $xcstack_id -xcamera $camera_name"); + &my_die("no inputs for camera $camera_name", $xcstack_id, $PS_EXIT_DATA_ERROR) unless scalar @$inputs > 0; + + foreach my $input (@$inputs) { + my $stack_id = $input->{stack_id}; + + my $good_frac = $input->{good_frac}; + my $Nsources = $input->{sources}; + + print "camera: $camera_name, stack_id: $stack_id, good_frac: $good_frac, Nsources: $Nsources\n"; + if ($good_frac < 1e-4) { my_escape("low good_frac ($good_frac) for $camera_name, stack_id $stack_id", $xcstack_id, 6000); } + if ($Nsources < 1) { my_escape("no sources for $camera_name, stack_id $stack_id", $xcstack_id, 6001); } + + # check for the input files + my $image = $ipprc->filename("PPSTACK.UNCONV", $input->{path_base}); # Image name + my $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $input->{path_base}); # Mask name + my $weight = $ipprc->filename("PPSTACK.UNCONV.VARIANCE", $input->{path_base}); # Weight name + my $sources = $ipprc->filename("PPSTACK.OUTPUT.SOURCES", $input->{path_base}); # Sources name + + my $have_image = $ipprc->file_exists( $image ); + my $have_mask = $ipprc->file_exists( $mask ); + my $have_weight = $ipprc->file_exists( $weight ); + my $have_sources = $ipprc->file_exists( $sources ); + + &my_die("Image $image does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_image; + &my_die("Mask $mask does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_mask; + &my_die("Weight $weight does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_weight; + &my_die("Sources $sources does not exist for $camera_name, stack_id $stack_id", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $have_sources; + + print $listFile "INPUT$num\tMETADATA\n"; + + print $listFile " IMAGE STR " . $image . "\n"; + print $listFile " MASK STR " . $mask . "\n"; + print $listFile " VARIANCE STR " . $weight . "\n"; + print $listFile " SOURCES STR " . $sources . "\n"; + print $listFile "END\n\n"; + $num++; + } +} +close($listFile); + +if ($num < 2) { &my_escape("not enough input stacks to make an xcstack", $xcstack_id, 6002); } + +print "I've loaded my inputs: $xcstack_id\n"; + +print "I've configured everything: $xcstack_id\n"; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_ppStack = $ipprc->reduction($reduction, 'XCSTACK_PPSTACK'); # Recipe to use for ppStack +my $recipe_ppstats = 'STACKSTATS'; +&my_die("Couldn't find selected reduction for XCSTACK_PPSTACK: $reduction\n", $xcstack_id, $PS_EXIT_CONFIG_ERROR) unless ($recipe_ppStack); + +my $recipe = &run_command ("$ppConfigDump -camera $camera -recipe PPSTACK $recipe_ppStack -dump-recipe PPSTACK -"); + +my $output_nocomp = metadataLookupBool($recipe, 'OUTPUT.NOCOMP'); # change filerules to produced uncompressed output images +my $output_logflux = metadataLookupBool($recipe, 'OUTPUT.LOGFLUX'); # change filerules to produce logflux compressed output images. +my $output_deepexp = metadataLookupBool($recipe, 'OUTPUT.DEEPEXP'); # change filerules to produce exptime map with more depth +my $replicate_outputs = (defined($neb) and metadataLookupBool($recipe, 'OUTPUT.REPLICATE')); # replicate output images +if ($output_nocomp and $output_logflux) { + &my_die("Unable to not compress and logflux compress simultaneously. Check config.",$xcstack_id, $PS_EXIT_CONFIG_ERROR); +} +if ($output_nocomp and $output_deepexp) { + &my_die("Unable to not compress and use deepexp simultaneously. Check config.",$xcstack_id, $PS_EXIT_CONFIG_ERROR); +} +my $stack_type = metadataLookupStr($recipe, 'STACK.TYPE'); +&my_die("STACK.TYPE not found in recipe. Check config.",$xcstack_id, $PS_EXIT_CONFIG_ERROR) unless $stack_type; +# stack_type must be one of: DEEP_STACK, NIGHTLY_STACK, IQ_STACK. this does not affect the analysis, but is added to the header + +# Prepare the output files +my @outputFiles; +prepare_outputs(\@outputFiles, \@outputList, $outroot); # this list + +## use an explicit xcstack name for psphot output objects +my $do_stats = 1; +my $outputStats; +my $traceDest; +my $configuration = prepare_output("PPSTACK.CONFIG", $outroot, 1); +if ($run_state eq 'new') { + $outputStats = prepare_output("SKYCELL.STATS", $outroot, 1); + $traceDest = prepare_output("TRACE.EXP", $outroot, 1); + push @outputFiles, $configuration; + push @outputFiles, $outputStats; +} else { + # we need to do stats regardless because we need the quality flag + # $do_stats = 0; + $outputStats = prepare_output("SKYCELL.STATS.UPDATE", $outroot, 1) if $do_stats; + $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, 1); +} + +$temp_images_exist = 1; # failures after this point should attempt to delete the temp images + +my $cmdflags; + +# Perform stacking +unless ($no_op) { + my $command = "$ppStack -input $listName $outroot"; + $command .= " -stats $outputStats" if $do_stats; + $command .= " -recipe PPSTACK $recipe_ppStack"; + $command .= " -recipe PPSTATS $recipe_ppstats" if $do_stats;; + $command .= " -stack-type $stack_type"; + if ($output_nocomp) { + $command .= " -F PPSTACK.OUTPUT PPSTACK.OUTPUT.NOCOMP"; + $command .= " -F PPSTACK.OUTPUT.VARIANCE PPSTACK.OUTPUT.VARIANCE.NOCOMP"; + $command .= " -F PPSTACK.OUTPUT.EXPWT PPSTACK.OUTPUT.EXPWT.NOCOMP"; + $command .= " -F PPSTACK.OUTPUT.EXP PPSTACK.OUTPUT.EXP.NOCOMP"; + $command .= " -F PPSTACK.UNCONV PPSTACK.UNCONV.NOCOMP"; + $command .= " -F PPSTACK.UNCONV.VARIANCE PPSTACK.UNCONV.VARIANCE.NOCOMP"; + $command .= " -F PPSTACK.UNCONV.EXPWT PPSTACK.UNCONV.EXPWT.NOCOMP"; + $command .= " -F PPSTACK.UNCONV.EXP PPSTACK.UNCONV.EXP.NOCOMP"; + } + if ($output_logflux) { + $command .= " -R PPSTACK.OUTPUT FITS.TYPE STK_UNIONS_COMP"; + $command .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE STK_UNIONS_COMP"; + $command .= " -R PPSTACK.UNCONV FITS.TYPE STK_UNIONS_COMP"; + $command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE STK_UNIONS_COMP"; + } + # if the total exposure time is long (> 6500 sec), need to use + # a different compression options. + if ($output_deepexp) { + $command .= " -R PPSTACK.OUTPUT.EXP FITS.TYPE EXP_UNIONS "; + $command .= " -R PPSTACK.UNCONV.EXP FITS.TYPE EXP_UNIONS "; + } + $command .= " -threads $threads" if defined $threads; + $command .= " -debug-stack" if defined $debug; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -image_id $image_id" if defined $image_id; + $command .= " -source_id $source_id" if defined $source_id; + + if ($run_state eq 'new') { + $command .= " -dumpconfig $configuration"; + } else { + $command .= " -ipprc $configuration"; + } + + + # cannot use run_command here (no metadata output to parse) + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); + } + + # extract the stats flags, and capture the -quality flag + my $quality; + if ($do_stats) { + my $outputStatsReal = $ipprc->file_resolve($outputStats); + &my_die("Couldn't find expected output file: $outputStats", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); + + # measure stats + $command = "$ppStatsFromMetadata $outputStatsReal - XCSTACK_SKYCELL"; + ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); + } + foreach my $line (@$stdout_buf) { $cmdflags .= " $line"; } + chomp $cmdflags; + + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + } + + if (!$quality) { + check_outputs(\@outputFiles, $replicate_outputs); + } + + print "I've stacked $listName: $xcstack_id\n"; +} + +delete_temps() unless ($save_temps); + +unless ($no_update) { + + # Add the stack result + { + my $command = "$xcstacktool"; + my $mode; + if ($run_state eq 'new') { + $mode = "-addsumskyfile"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $mode = "-updaterun -set_state full"; + } + $command .= " $mode -xcstack_id $xcstack_id"; + $command .= " $cmdflags" if $do_stats; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); + } + } +} + +print "I've updated the database: $xcstack_id\n"; +print "I've reached the end of processing with a code of $?: $xcstack_id\n"; + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $xcstack_id = shift; # Xcstack identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + if (defined $xcstack_id and not $no_update) { + my $command = "$xcstacktool -xcstack_id $xcstack_id -fault $exit_code"; + if ($run_state eq 'new') { + $command .= " -addsumskyfile"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -hostname $host" if defined $host; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $command .= " -updatesumskyfile"; + } + $command .= " -dbname $dbname" if defined $dbname; + print "running in my_die: $command\n"; + system ($command); + } + exit $exit_code; +} + +sub my_escape +{ + my $msg = shift; # Warning message on exit + my $xcstack_id = shift; # Xcstack identifier + my $quality = shift; # Exit code to add + + carp($msg); + + if (defined $xcstack_id and not $no_update) { + my $command = "$xcstacktool -xcstack_id $xcstack_id"; + if ($run_state eq 'new') { + $command .= " -addsumskyfile"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -hostname $host" if defined $host; + $command .= " -quality $quality" if defined $quality; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $command .= " -updatesumskyfile"; + $command .= " -set_quality $quality" if defined $quality; + } + $command .= " -dbname $dbname" if defined $dbname; + print "running in my_die: $command\n"; + system ($command); + } + exit 0; +} + + +# Ensure temporary images are deleted +sub delete_temps +{ + unless ($temp_images_exist) { + print "No temporary images yet generated\n"; + return 1; + } + + print "Ensuring temporary images are deleted.\n"; + + my $temp_delete = defined($recipe) and metadataLookupBool($recipe, 'TEMP.DELETE'); # Delete temp files? + if ($temp_delete) { + my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, 'TEMP.DIR') or &my_die("Unable to find temporary directory in site configuration", $xcstack_id, $PS_EXIT_CONFIG_ERROR); # Directory with temporary files + my $temp_image = metadataLookupStr($recipe, 'TEMP.IMAGE'); # Suffix for image + my $temp_mask = metadataLookupStr($recipe, 'TEMP.MASK'); # Suffix for mask + my $temp_weight = metadataLookupStr($recipe, 'TEMP.VARIANCE'); # Suffix for weight + my $temp_root = basename($outroot); # Root name for temporary files + + if (not defined $temp_dir or not defined $temp_image or not defined $temp_mask or + not defined $temp_weight) { + &my_die("Unable to find details for temporary images.", $xcstack_id, $PS_EXIT_CONFIG_ERROR); + } + + for (my $i = 0; $i < $num; $i++) { + foreach my $ext ( $temp_image, $temp_mask, $temp_weight ) { + my $file = $temp_dir . '/' . $temp_root . '.' . $i . '.' . $ext; + unlink $file if -e $file; + } + } + } +} + +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, undef, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $xcstack_id, $error); + return $output; +} + +sub prepare_outputs +{ + my $fileList = shift; + my $ruleList = shift; + my $outroot = shift; + + foreach my $rule (@$ruleList) { + push @$fileList, prepare_output($rule, $outroot, 1); + } +} + +sub check_output +{ + my $file = shift; + my $replicate = shift; + + if (!defined $file) { + return; + } + + &my_die("Couldn't find expected output file: $file", $xcstack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $xcstack_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $xcstack_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { + $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $xcstack_id, $PS_EXIT_SYS_ERROR); + } +} + +sub check_outputs +{ + my $fileList = shift; + my $replicate = shift; + foreach my $file (@$fileList) { + check_output($file, $replicate); + } +} + +sub run_command { + + my $command = shift; + my $result; + + # Get list of cameras for this xcstack + my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + + my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my $error_code = &parse_error_message ($success, $error_msg, $command); + unless ($success) { + &my_die("Unable to perform $command: $error_msg", $xcstack_id, $error_code); + } + + # is the result allowed to be empty? + if (@$stdout_buf == 0) { + &my_die("No output from command $command", $xcstack_id, $PS_EXIT_PROG_ERROR); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $xcstack_id, $PS_EXIT_PROG_ERROR); + + return $metadata; +} + +sub run_command_ipptool { + + my $command = shift; + + my $metadata = &run_command ($command); + my $result = parse_md_list($metadata) or &my_die("Unable to parse metadata list", $xcstack_id, $PS_EXIT_PROG_ERROR); + return $result; +} + +# if the program exited normally, the exit value is returned +# if the program exited with a signal (abort, crash, etc), PS_EXIT_PROG_ERROR (4) is returned +# if the program failed to run, -PS_EXIT_PROG_ERROR (-4) is returned +sub parse_error_message { + my $success = shift; + my $error_msg = shift; + my $command = shift; + + if ($success) { return 0; } + + print "raw error_msg: $error_msg\n"; + print "full command: $command\n"; + + if (&looks_like_number($error_msg)) { return (($error_msg >> 8) or ($PS_EXIT_PROG_ERROR)); } + + # which of these match? + my ($error_code) = $error_msg =~ m/exited with value (\d+)/; + if (defined $error_code) { return $error_code; } + + ($error_code) = $error_msg =~ m/died with signal (\d+)/; +# if (defined $error_code) { return (128 + $error_code); } + if (defined $error_code) { return ($PS_EXIT_PROG_ERROR); } + + # probably failed to execute: + return (-1*$PS_EXIT_PROG_ERROR); +} + +__END__ +