Index: branches/czw_branch/20100519/ippScripts/scripts/addstar_run.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/addstar_run.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/addstar_run.pl (revision 28304) @@ -37,5 +37,5 @@ } -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update, +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -48,4 +48,7 @@ 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database + 'minidvodb_name|w=s'=> \$minidvodb_name, # miniDVO database name + 'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group + 'minidvodb' => \$minidvodb, # use minidvodb? 'image-only' => \$image_only, # Print to stdout 'verbose' => \$verbose, # Print to stdout @@ -58,5 +61,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --exp_tag --add_id --camera --outroot", + -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot", -exitval => 3, ) unless @@ -65,6 +68,9 @@ defined $outroot and defined $camroot and + defined $dvodb and defined $camera; - +if ($minidvodb && !defined($minidvodb_group)) { + my_die( "missing minidvodb_group", $add_id, 3 ); + } my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -98,11 +104,45 @@ if (defined $dvodb) { $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO - $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); -} + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); +} + + + my $dtime_addstar = 0; +if (defined $dvodbReal) { + if ($minidvodb) { + + if (!defined($minidvodb_name)) { + #take the active one, if it's not defined on the command line + #reverts would have this already set, for example. + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + + } + # tack on the minidvodb part to the db. + $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; + + } unless ($no_op) { - if (defined $dvodbReal) { + print $dvodbReal; ## addstar can either save the full set of detections, or just @@ -142,6 +182,8 @@ $fpaCommand .= " -path_base $outroot"; $fpaCommand .= " -dtime_addstar $dtime_addstar"; +$fpaCommand .= " -dvodb_path $dvodbReal" if defined $dvodbReal; +$fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; $fpaCommand .= " -dbname $dbname" if defined $dbname; - +print $fpaCommand; # Add the result into the database unless ($no_update) { @@ -171,6 +213,8 @@ $command .= " -addprocessedexp"; $command .= " -fault $exit_code"; + $command .= " -dvodb_path $dvodbReal" if defined $dvodbReal; $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; $command .= " -dbname $dbname" if defined $dbname; system ($command); Index: branches/czw_branch/20100519/ippScripts/scripts/automate_stacks.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/automate_stacks.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/automate_stacks.pl (revision 28304) @@ -20,4 +20,5 @@ my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); @@ -59,4 +60,5 @@ my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); +my ( $check_detrends, $queue_detrends); GetOptions( @@ -81,4 +83,6 @@ 'check_stacks' => \$check_stacks, 'queue_stacks' => \$queue_stacks, + 'check_detrends' => \$check_detrends, + 'queue_detrends' => \$queue_detrends, 'check_diffs' => \$check_diffs, 'queue_diffs' => \$queue_diffs, @@ -105,4 +109,6 @@ --check_stacks Confirm that stacks can be built. --queue_stacks Issue stacktool commands to queue stacks. + --check_detrends Confirm that detrend verify runs can be built. + --queue_detrends Issue dettool commands to queue detrend verify runs. --check_diffs Confirm that diffs can be done. --queue_diffs Issue difftool commands to queue diffs. @@ -115,6 +121,6 @@ ) unless defined $check_registration or defined $define_burntool or defined $queue_burntool or - defined $queue_chips or defined $queue_stacks or - defined $check_chips or defined $check_stacks or + defined $queue_chips or defined $queue_stacks or $queue_detrends or + defined $check_chips or defined $check_stacks or $check_detrends or defined $test_mode or defined $clean_old or defined $check_mode; @@ -128,4 +134,11 @@ my %stackable_list = (); my %reduction_class = (); +my @detrend_list = (); +my %dettype_list = (); +my %exptype_list = (); +my %refID_list = (); +my %refIter_list = (); +my %detfilter_list = (); +my %detmax_list = (); my %clean_commands = (); my %clean_retention = (); @@ -158,7 +171,4 @@ } } -# if (${ $entry }{name} eq 'RETENTION_TIME') { -# $retention_time = ${ $entry }{value}; -# } elsif (${ $entry }{name} eq 'FILTERS') { push @filter_list, ${ $entry }{value}; @@ -195,8 +205,33 @@ } } -} - - - + elsif (${ $entry }{name} eq 'DETRENDS') { + my @detrend_data = @{ ${ $entry }{value} }; + my $this_detrend = ''; + foreach my $dentry (@detrend_data) { + if (${ $dentry }{name} eq 'NAME') { + $this_detrend = ${ $dentry }{value}; + push @detrend_list, $this_detrend; + } + elsif (${ $dentry }{name} eq 'DETTYPE') { + $dettype_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'EXPTYPE') { + $exptype_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'REF_ID') { + $refID_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'REF_ITER') { + $refIter_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'FILTER') { + $detfilter_list{$this_detrend} = ${ $dentry }{value}; + } + elsif (${ $dentry }{name} eq 'MAX') { + $detmax_list{$this_detrend} = ${ $dentry }{value}; + } + } + } +} unless(defined($date)) { @@ -256,4 +291,16 @@ return_metadata($date); unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($check_detrends) || defined($test_mode)) { + $metadata_out{nsState} = 'CHECKDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + unless (defined($test_mode) || defined($check_mode)) { exit(0); } +} +if (defined($queue_detrends)) { + $metadata_out{nsState} = 'QUEUEDETRENDS'; + &execute_detrends($date,"pretend"); + return_metadata($date); + exit(0); } if (defined($define_burntool) || defined($test_mode)) { @@ -498,5 +545,5 @@ my $cmd = "$chiptool"; - $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp '; + $cmd .= " -simple -dbname $dbname -definebyquery -set_end_stage warp "; $cmd .= " -set_label $label "; $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; @@ -566,4 +613,6 @@ } + + sub chip_queue { my $date = shift; @@ -612,4 +661,101 @@ # +# Detrend verification +################################################################################ + +sub construct_dettool_cmd { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; + $date =~ s/-//g; + + my $cmd = "$dettool"; + $cmd .= " -pretend "; + $cmd .= " -simple -dbname $dbname -definebyquery -det_type $det_type "; + $cmd .= " -mode verify -ref_det_id $ref_det_id -ref_iter $ref_iter "; + $cmd .= " $select "; + $cmd .= " -inst $camera "; + $cmd .= " -select_exp_type $exp_type "; + $cmd .= " -select_filter $filter " if defined($filter); + $cmd .= " -workdir $workdir "; + $cmd .= " -label $label "; + if ($maxN > 0) { + $cmd .= " -random_subset -random_limit $maxN "; + } + if ($debug == 1) { + $cmd .= ' -pretend '; + } + print STDERR "$cmd\n"; + return($cmd); +} + +sub verify_uniqueness_detverify { + my $date = shift; + my $target = shift; + + my ($label,$workdir,$filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN) = get_dettool_parameters($date,$target); + + my $db = init_gpc_db(); + my $sth = "SELECT * FROM detRun WHERE workdir = '$workdir' AND ref_det_id = $ref_det_id AND ref_iter = $ref_iter"; + my $data_ref = $db->selectall_arrayref( $sth ); + + return($#{ $data_ref } + 1); +} + +sub pre_detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target) . ' -pretend '; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); + } + + my @input_exposures = split /\n/, (join '', @$stdout_buf); + return($#input_exposures + 1); +} + +sub detrend_queue { + my $date = shift; + my $target = shift; + + my $command = construct_dettool_cmd($date,$target); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + $metadata_out{nsState} = 'DETREND_QUEUED'; + return(0); +} + +sub execute_detrends { + my $date = shift; + my $pretend = shift; + my $exposures = 0; + foreach my $target (@detrend_list) { + my ($Nexposures) = pre_detrend_queue($date,$target); + if ($Nexposures == 0) { + print STDERR "execute_detrends: Target $target on $date had no exposures.\n"; + next; + } + $exposures++; + unless(defined($pretend)) { + detrend_queue($date,$target); + } + } + if ($exposures == 0) { + $metadata_out{nsState} = 'DETREND_DROP'; + } +} + +# # Stacking ################################################################################ @@ -627,5 +773,5 @@ my $cmd = "$stacktool"; # $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite - $cmd .= ' -simple -dbname gpc1 -definebyquery '; + $cmd .= " -simple -dbname $dbname -definebyquery "; $cmd .= " -set_label $label -select_label $label "; $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; @@ -779,5 +925,5 @@ my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target); my $args = $command; - $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; + $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label -data_group $data_group "; if ($debug == 1) { $args .= ' -pretend '; @@ -826,5 +972,5 @@ my $label = "${target}.nightlyscience"; - my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}"; + my $workdir = "neb://\@HOST\@.0/${dbname}/${target}.nt/${workdir_date}"; my $obs_mode = $obsmode_list{$target}; my $object = $object_list{$target}; @@ -837,4 +983,30 @@ } +sub get_dettool_parameters { + my $date = shift; + my $target = shift; + my $workdir_date = $date; $workdir_date =~ s%-%/%g; + my $trunc_date = $date; $trunc_date =~ s/-//g; + + my $exp_type = $exptype_list{$target}; + my $det_type = $dettype_list{$target}; + my $ref_det_id = $refID_list{$target}; + my $ref_iter = $refIter_list{$target}; + my $det_filter = $detfilter_list{$target}; + my $internal_filter; + if (defined($det_filter)) { + $internal_filter = $det_filter; $internal_filter =~ s/\..*//; + $internal_filter = '.' . $internal_filter; + } + else { + $internal_filter = ''; + } + my $maxN = $detmax_list{$target}; + + my $lc_type = lc($exp_type); + my $label = "${lc_type}${internal_filter}.$trunc_date"; + my $workdir = 'neb://@HOST@.0/' . $dbname . "/detverify.nt/${workdir_date}/${lc_type}${internal_filter}"; + return($label,$workdir,$det_filter,$exp_type,$det_type,$ref_det_id,$ref_iter,$maxN); +} sub init_gpc_db { Index: branches/czw_branch/20100519/ippScripts/scripts/diff_skycell.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/diff_skycell.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/diff_skycell.pl (revision 28304) @@ -197,4 +197,5 @@ my $recipe_ppSub = $ipprc->reduction($reduction, 'DIFF_PPSUB'); # Recipe to use for ppSub my $recipe_psphot = $ipprc->reduction($reduction, 'DIFF_PSPHOT'); # Recipe to use for psphot +my $recipe_ppstats = 'DIFFSTATS'; unless ($recipe_ppSub and $recipe_psphot) { &my_die("Couldn't find selected reduction for DIFF_PPSUB and DIFF_PSPHOT: $reduction\n", $diff_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); @@ -257,4 +258,6 @@ $traceDest .= '.update'; $outputStats .= '.update'; + + $recipe_ppstats = 'WARPSTATS'; } @@ -292,5 +295,5 @@ $command .= " -recipe PPSUB $recipe_ppSub"; $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSTATS DIFFSTATS"; + $command .= " -recipe PPSTATS $recipe_ppstats"; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; Index: branches/czw_branch/20100519/ippScripts/scripts/lossy_compress_imfile.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/lossy_compress_imfile.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/lossy_compress_imfile.pl (revision 28304) @@ -61,7 +61,7 @@ defined $state; -my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $exp_name, $class_id, $uri, $PS_EXIT_CONFIG_ERROR ); +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $exp_name, $class_id, $uri, $state, $PS_EXIT_CONFIG_ERROR ); if (defined($logfile)) { - $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $exp_name, $class_id, $uri, $PS_EXIT_SYS_ERROR ); + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $exp_name, $class_id, $uri, $state, $PS_EXIT_SYS_ERROR ); } @@ -87,5 +87,5 @@ if ($state eq 'goto_compressed') { # Find the actual filename for this run: - &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR) unless ($ipprc->file_exists($uri)); my $uriReal = $ipprc->file_resolve( $uri ); @@ -93,5 +93,5 @@ # Create a compressed version: my $compUri = $uri . ".fz"; - &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR) if ($ipprc->file_exists($compUri)); my $compReal= $ipprc->file_resolve( $compUri, 'create'); @@ -109,5 +109,5 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to uncompress file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri, $state, $PS_EXIT_SYS_ERROR); } my $compress_command = "$fpack -h -s 8 -S $tempReal > $compReal"; @@ -118,10 +118,10 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to compress file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri, $state, $PS_EXIT_SYS_ERROR); } my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state compressed"; $database_command .= " -dbname $dbname" if defined $dbname; - &my_die("Expected output compressed file not found: $compUri\n", $exp_id,$exp_name,$class_id, $uri,$PS_EXIT_SYS_ERROR) + &my_die("Expected output compressed file not found: $compUri\n", $exp_id,$exp_name,$class_id, $uri,$state, $PS_EXIT_SYS_ERROR) unless ($ipprc->file_exists($compUri)); @@ -135,5 +135,5 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to update database file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri, $state, $PS_EXIT_SYS_ERROR); } } @@ -149,5 +149,5 @@ if ($state eq 'goto_lossy') { # Confirm we have both files - &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR) unless($ipprc->file_exists($uri)); my $uriReal = $ipprc->file_resolve( $uri ); @@ -156,5 +156,5 @@ unless($ipprc->file_exists($compUri)) { &my_die("Couldn't find compressed version of the file: $compUri\n", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR); # If that die is removed, this will compress things as well. @@ -185,5 +185,5 @@ $neb->swap($uri,$compUri) or &my_die("Nebulous swap failed between $uri and $compUri", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR); } # Update database @@ -204,7 +204,7 @@ $neb->swap($uri,$compUri) or &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri", - $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri,$state,$PS_EXIT_SYS_ERROR); &my_die("Unable to update database file: $uri -> $compUri: $error_code", - $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + $exp_id,$exp_name,$class_id,$uri, $state,$PS_EXIT_SYS_ERROR); } # remove original version @@ -224,6 +224,16 @@ my $class_id = shift; # Class identifier my $uri = shift; # uri for the file + my $state = shift; my $exit_code = shift; # Exit code # outputImage and path_base are globals + my $fail_state = $state; + $fail_state =~ s/goto_/error_/; + my $error_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state $fail_state "; + $error_command .= " -dbname $dbname" if defined $dbname; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $error_command, verbose => $verbose); + unless($success) { + warn("Failed at setting the error state too."); + } $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; Index: branches/czw_branch/20100519/ippScripts/scripts/magic_process.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/magic_process.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/magic_process.pl (revision 28304) @@ -151,5 +151,5 @@ my $kernel = $ipprc->filename("PPSUB.OUTPUT.KERNELS", $diff_base); # Name of kernel file - &my_die("Unable to find kernel file", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel); + &my_die("Unable to find kernel file $kernel", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel); my ($image, $mask); # Image and mask @@ -191,5 +191,5 @@ { - &my_die("Unable to find image and mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask); + &my_die("Unable to find image and mask: $image $mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask); my $command = "$ppSubConvolve $tempPath -image $image -mask $mask -kernel $kernel"; @@ -204,5 +204,5 @@ $template = $ipprc->filename("PPSUB.INPUT.CONV", $tempPath) or &my_die("Unable to determine filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR); $template = $ipprc->file_resolve($template) or &my_die("Unable to resolve filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR); - &my_die("Unable to find created template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template); + &my_die("Unable to find created template: $template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template); } } Index: branches/czw_branch/20100519/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/minidvodb_createdb.pl (revision 28304) +++ branches/czw_branch/20100519/ippScripts/scripts/minidvodb_createdb.pl (revision 28304) @@ -0,0 +1,255 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $dvodb,$minidvodb_interval, $minidvodb_group, $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'dvodb|w=s' => \$dvodb, # output DVO database + 'interval|w=s' => \$minidvodb_interval, #interval between creation of minidvodbs (default = 1day) + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --camera --dvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + defined $camera and + defined $outroot and + defined $dvodb; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_group , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +# XXX This is now not used: do we still need it? + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} else { + warn("dvodb undefined:\n"); + exit(4); +} + +if (!defined $minidvodb_interval) { + $minidvodb_interval = 1; +} + + +unless ($no_op) { + + + +#see if there is already one in new state + my $fpaCommand1 = "$addtool -listminidvodbrun"; + $fpaCommand1 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand1 .= " -state 'new'"; + $fpaCommand1 .= " -dbname $dbname" if defined $dbname; + + +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand1, verbose => $verbose); + + + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to list minidvodb database: $error_code\n"); + exit($error_code); + } + + if (scalar(@{$stdout_buf})) { + $error_code = 3; + warn("Unwilling to create minidvodb, one already exists in new state: $error_code\n"); + exit($error_code); + } + + my $creation_date; + my $addRun_count; + my $minidvodb_name; +#find the active one's date, and find out if it has more than 1 addRun in it + + my $fpaCommand2 = "$addtool -listminidvodbrun"; + $fpaCommand2 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand2 .= " -state 'active'"; + $fpaCommand2 .= " -limit 1"; + $fpaCommand2 .= " -dbname $dbname" if defined $dbname; + +#print $fpaCommand2; + + + my ( $success2, $error_code2, $full_buf2, $stdout_buf2, $stderr_buf2 ) = + run(command => $fpaCommand2, verbose => $verbose); + &my_die( "Unable to get listminidvodbrun",$minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success2; + # if it didn't list something in active state (what?) then we definitely need to create a new one + if (defined(@$stdout_buf2)) { + my $metadata2 = $mdcParser->parse(join "", @$stdout_buf2) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components2 = parse_md_list($metadata2) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp2 = $$components2[0]; + $minidvodb_name = $comp2->{minidvodb_name}; + $creation_date = $comp2->{creation_date}; + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + if (!defined($creation_date)) { + &my_die("Unable to parse creation_date", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + } else { + $create_new = 1; #this is to force it to make a new one + } + + #find the number of add_ids that have been proccessed + my $fpaCommand3 = "$addtool -checkminidvodbrunaddrun"; + $fpaCommand3 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand3 .= " -state 'active'"; + $fpaCommand3 .= " -minidvodb_name '$minidvodb_name'" if defined $minidvodb_name; + $fpaCommand3 .= " -limit 1"; + $fpaCommand3 .= " -dbname $dbname" if defined $dbname; + + +my ( $success3, $error_code3, $full_buf3, $stdout_buf3, $stderr_buf3 ) = + run(command => $fpaCommand3, verbose => $verbose); + &my_die( "Unable to get checkminidvodbunaddrun", $minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success3; + + if (defined(@$stdout_buf3)) { #checkminidvodb returns nothing IF there have been no addruns added to the db yet + my $metadata3 = $mdcParser->parse(join "", @$stdout_buf3) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components3 = parse_md_list($metadata3) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp = $$components3[0]; + $addRun_count = $comp->{addRun_count}; + } + if (!defined($addRun_count)) { + ## there's nothing to parse if there's nothing + $addRun_count = 0; + } + + + if ($addRun_count > 30000) { + #it's too big, create_new + $create_new = 1; + } + if ($create_new == 0) { + my $parser = DateTime::Format::Strptime->new( pattern => '%Y-%m-%dT%H:%M:%S', time_zone => "HST" ); + my $creation_dt = $parser->parse_datetime( $creation_date )->mjd; + if ($mjd_start- $creation_dt > $minidvodb_interval && $addRun_count > 0 ) { + #db is old and has stuff in it, want to create_new + $create_new = 1; + } + } + + +} +#create the minidvodb entry (well, the command for it) + my $fpaCommand = "$addtool -addminidvodbrun"; + $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + unless ($no_update or !$create_new) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $minidvodb_group = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: branches/czw_branch/20100519/ippScripts/scripts/minidvodb_merge.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/minidvodb_merge.pl (revision 28304) +++ branches/czw_branch/20100519/ippScripts/scripts/minidvodb_merge.pl (revision 28304) @@ -0,0 +1,254 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_merge; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb, $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb|w=s' => \$mergedvodb, # output DVO database + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $mergedvodb and + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodb, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} + +my $dtime_addstar = 0; +#my $dtime_relphot = 0; + +unless ($no_op) { + if (defined $mergedvodbReal) { + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $mergedvodb, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + $dtime_resort = $dtime_addstar; + print "addstar -resort time $dtime_addstar\n"; + } + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $minidvodb"; + print "$command\n"; + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $mergedvodb, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $command = "$addtool -listminidvodbrun -state merged -minidvodb_group " . $minidvodb_group; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $minidvodb_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $minidvodb_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 minidvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb/Image.dat\n"; + + + if (-e "$mergedvodb/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $minidvodb_id, 4); + } + $this_is_the_first =0; + } + + print "$this_is_the_first $mergedvodb/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + + $merge_command = "cp -rp $minidvodb/* $mergedvodb"; + } else { + $merge_command = "$dvomerge $minidvodb into $mergedvodb"; + } + + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $mergedvodb, $error_code); + } + + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; +# } + } + + { + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code); + } + } + + } else { + &my_die("dvodb: $mergedvodb not found", $mergedvodb, $PS_EXIT_UNKNOWN_ERROR); + } +} else { + print "skipping processing for $mergedvodbReal\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + + + + #print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: branches/czw_branch/20100519/ippScripts/scripts/skycell_jpeg.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/skycell_jpeg.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/skycell_jpeg.pl (revision 28304) @@ -38,5 +38,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV; pod2usage( - -msg => "Required options: --stage --stage_id --outroot\nHelpful options: --camera --dbname", + -msg => "Required options: --stage --stage_id --path_base\nHelpful options: --camera --dbname", -exitval => 3, ) unless @@ -125,5 +125,5 @@ } # Update database: - $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -outroot $outroot"; + $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot"; if (defined($dbname)) { $command .= " -dbname $dbname"; @@ -136,4 +136,17 @@ } } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$warptool -addsummary -warp_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } } #end warp stage if ($stage eq 'diff') { @@ -165,5 +178,5 @@ my $state = $imfile->{data_state}; my $projection_cell = $skycell_id; - if ($quality == 8007 or $state ne 'full') { + if ($quality == 8007 or $quality = 1010 or $state ne 'full') { next; } @@ -209,5 +222,5 @@ } # Update database: - $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -outroot $outroot"; + $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot"; if (defined($dbname)) { $command .= " -dbname $dbname"; @@ -220,8 +233,21 @@ } } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$difftool -addsummary -diff_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } } #end diff stage if ($stage eq 'stack') { my $imfiles; - my $command = "$stacktool -stackskyfile -stack_id $stage_id"; + my $command = "$stacktool -sassskyfile -sass_id $stage_id"; if (defined($dbname)) { $command .= " -dbname $dbname"; @@ -248,6 +274,7 @@ my $quality = $imfile->{quality}; my $state = $imfile->{data_state}; + my $fault = $imfile->{fault}; my $projection_cell = $skycell_id; - if ($quality == 8007 or $state ne 'full') { + if ($quality == 8007 or $state ne 'full' or $fault != 0) { next; } @@ -293,5 +320,5 @@ } # Update database: - $command = "$stacktool -addsummary -stack_id $stage_id -projection_cell $projection_cell -outroot $outroot"; + $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot"; if (defined($dbname)) { $command .= " -dbname $dbname"; @@ -304,4 +331,18 @@ } } + if (scalar (keys %tangents) == 0) { + my $projection_cell = 'NULL'; + $command = "$stacktool -addsummary -stack_id $stage_id -projection_cell $projection_cell -path_base $outroot"; + if (defined($dbname)) { + $command .= " -dbname $dbname"; + } + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to update diff summary: $error_code", $stage_id, $error_code); + } + } + } #end stack stage Index: branches/czw_branch/20100519/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/stack_skycell.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/stack_skycell.pl (revision 28304) @@ -149,4 +149,8 @@ my $recipe_ppSub = $ipprc->reduction($reduction, 'STACK_PPSUB'); # Recipe to use for ppSub my $recipe_psphot = $ipprc->reduction($reduction, 'STACK_PSPHOT'); # Recipe to use for psphot +my $recipe_ppstats = 'STACKSTATS'; +if ($run_state eq 'update') { + $recipe_ppstats = 'WARPSTATS'; +} unless ($recipe_ppStack and $recipe_ppSub and $recipe_psphot) { &my_die("Couldn't find selected reduction for STACK_PPSTACK, STACK_PPSUB and STACK_PSPHOT: $reduction\n", $stack_id, $PS_EXIT_CONFIG_ERROR); @@ -232,5 +236,5 @@ $command .= " -recipe PPSUB $recipe_ppSub"; $command .= " -recipe PSPHOT $recipe_psphot"; - $command .= " -recipe PPSTATS STACKSTATS" if $do_stats;; + $command .= " -recipe PPSTATS $recipe_ppstats" if $do_stats;; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; Index: branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl =================================================================== --- branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl (revision 28245) +++ branches/czw_branch/20100519/ippScripts/scripts/staticsky.pl (revision 28304) @@ -38,5 +38,9 @@ } -my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $inverse, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps); +# XXX test: +print "run staticsky.pl @ARGV\n"; +# exit 0; + +my ($sky_id, $camera, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( 'sky_id=s' => \$sky_id, # Diff identifier @@ -44,5 +48,4 @@ 'dbname|d=s' => \$dbname, # Database name 'threads=s' => \$threads, # Number of threads to use - 'run-state=s' => \$run_state, # state for run: 'new' or 'update' 'outroot=s' => \$outroot, # Output root name 'reduction=s' => \$reduction, # Reduction class @@ -56,5 +59,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --sky_id --outroot --camera (--run_state)", + -msg => "Required options: --sky_id --outroot --camera", -exitval => 3, ) unless @@ -62,10 +65,8 @@ defined $outroot and defined $camera; -# and defined $run_state; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $sky_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration my $logDest = $ipprc->filename("LOG.EXP", $outroot); -# $logDest .= ".update" if $run_state eq "update"; $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $sky_id, $PS_EXIT_SYS_ERROR ) if $redirect; @@ -85,5 +86,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool -inputskyfile: $error_code", $sky_id, $error_code); + &my_die("Unable to perform staticskytool -inputs: $error_code", $sky_id, $error_code); } @@ -152,7 +153,7 @@ print "recipe_psphot: $recipe_psphot\n"; -my $cmdflags; - -# Perform subtraction +# my $cmdflags; + +# Perform stack photometry analysis { my $command = "$psphotStack $outroot"; @@ -167,10 +168,9 @@ unless ($no_op) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $sky_id, $error_code); - } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code); + } # my $outputStatsReal = $ipprc->file_resolve($outputStats); @@ -190,6 +190,7 @@ # chomp $cmdflags; - my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag - + # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + my $quality = 0; if (!$quality) { @@ -197,8 +198,8 @@ # we have one set of output files per input file set for (my $i = 0; $i < @$files; $i++) { - my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot); - my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot); - my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot); - my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot); + my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i); + my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i); + my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i); + my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i); &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); @@ -211,9 +212,4 @@ #my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); #my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); - - # if ($run_state eq 'update') { - # $traceDest .= '.update'; - # $outputStats .= '.update'; - # } my $chisqName = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot); @@ -235,13 +231,9 @@ { my $command = "$staticskytool -sky_id $sky_id"; - # if ($run_state eq 'new') { $command .= " -addresult -path_base $outroot"; $command .= " -num_inputs $nInputs"; - $command .= " $cmdflags"; + # $command .= " $cmdflags"; $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); $command .= " -hostname $host" if defined $host; - # } else { - # $command .= " -tofullskyfile"; - # } $command .= " -dbname $dbname" if defined $dbname; @@ -250,8 +242,7 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - my $err_message = $run_state eq "update" ? - "Unable to perform difftool -adddiffskyfile" : - "Unable to perform difftool -tofullskyfile"; - &my_die("$err_message: $error_code", $sky_id, $error_code); + my $err_message = "Unable to perform staticskytool -addresult"; + warn($err_message); + exit $error_code; } } @@ -270,13 +261,9 @@ if (defined $sky_id and not $no_update) { my $command = "$staticskytool -sky_id $sky_id -fault $exit_code"; - if ($run_state eq 'new') { - $command .= " -adddiffskyfile"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -hostname $host" if defined $host; - $command .= " -path_base $outroot" if defined $outroot; - $command .= " -dbname $dbname" if defined $dbname; - } else { - $command .= " -updatediffskyfile"; - } + $command .= " -addresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; run(command => $command, verbose => $verbose); }