Index: branches/czw_branch/20120906/Nebulous-Server/bin/neb-host =================================================================== --- branches/czw_branch/20120906/Nebulous-Server/bin/neb-host (revision 34785) +++ branches/czw_branch/20120906/Nebulous-Server/bin/neb-host (revision 34786) @@ -141,6 +141,6 @@ $dbh->commit; -my $format = "%-15s %-15s %-7s %-8s %-9s %-5s %s\n"; -my @columns = qw(host name mounted allocate available xattr note); +my $format = "%-15s %-15s %-7s %-8s %-9s %-5s %15s %s\n"; +my @columns = qw(host name mounted allocate available xattr last_modified note); printf($format, @columns); Index: branches/czw_branch/20120906/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- branches/czw_branch/20120906/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 34785) +++ branches/czw_branch/20120906/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 34786) @@ -389,4 +389,5 @@ mountedvol.vol_id IS NOT NULL as mounted, v.cab_id, + v.last_modified, v.note FROM volume AS v @@ -694,4 +695,5 @@ cab_id INT, note VARCHAR(255), + last_modified TIMESTAMP NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, # SC: Added by Haydn with ALTER TABLE(?) on 2012-12-03 PRIMARY KEY(vol_id), UNIQUE KEY(name), Index: branches/czw_branch/20120906/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/czw_branch/20120906/ippScripts/scripts/chip_imfile.pl (revision 34785) +++ branches/czw_branch/20120906/ippScripts/scripts/chip_imfile.pl (revision 34786) @@ -150,4 +150,5 @@ $outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); my $do_binned_images = 1; + if ($run_state eq 'new') { # prepare the files that are only created for a new run @@ -586,5 +587,7 @@ # check whether psphot outputs should be regenerated. -# Whether we need to or not is a somewhat complicated question. +# Whether we need to or not was a somewhat complicated question. Since we clean cmfs now we only rerun +# photometry if the psf file has gone missing since it is needed to run warps + sub rerun_photometry { @@ -594,4 +597,6 @@ or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + my $update_sources_if_gone = 0; # set this to regenerate sources if gone. + # We no longer do this as of 2012-12 my $make_sources = 0; my $sources_available = 0; @@ -599,15 +604,17 @@ $sources_available = 1; } else { - carp "WARNING: photometry sources file $outputSources is not available"; - my $gone; - if (storage_object_exists($outputSources, \$gone)) { - # check whether the file is permanantely or temporarily gone - if ($gone) { - carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + if ($update_sources_if_gone) { + carp "WARNING: photometry sources file $outputSources is not available"; + my $gone; + if (storage_object_exists($outputSources, \$gone)) { + # check whether the file is permanantely or temporarily gone + if ($gone) { + carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + $make_sources = 1; + } + } else { + # storage object must have been deleted $make_sources = 1; } - } else { - # storage object must have been deleted - $make_sources = 1; } } @@ -616,5 +623,7 @@ my $psf_available = 0; my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE", $outroot, $class_id) - or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", + $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + if ($ipprc->file_exists($outputPsf)) { $psf_available = 1; @@ -638,10 +647,12 @@ } - # if either of the files are gone rerun photometry unless the other file is temporarily not available - - if (!$sources_available && !$make_sources) { - # destreak will die if the sources is not available - &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } + if ($update_sources_if_gone) { + if (!$sources_available && !$make_sources) { + # destreak will die if the sources is not available + # but magic is dead.... + &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + if (!$psf_available && !$make_psf) { # warp updates need the psf file Index: branches/czw_branch/20120906/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/czw_branch/20120906/ippScripts/scripts/ipp_cleanup.pl (revision 34785) +++ branches/czw_branch/20120906/ippScripts/scripts/ipp_cleanup.pl (revision 34786) @@ -18,5 +18,12 @@ # Parse the command-line arguments -my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile); +my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all); +my $very_verbose = 0; + +# this gets set to 1 the first time we set the corresponding destreak run to be cleaned +#my $ds_done = 0; +# magic is dead +my $ds_done = 1; + GetOptions('stage=s' => \$stage, # which analysis stage to clean? 'camera|i=s' => \$camera, # user-supplied camera name @@ -24,4 +31,5 @@ 'mode|m=s' => \$mode, # cleanup mode (clean / purge) 'path_base=s' => \$path_base, # basename for files + 'check-all' => \$check_all, # if set clean all chips regardless of data_state 'dbname|d=s' => \$dbname, # Database name 'verbose' => \$verbose, # Print to stdout @@ -56,4 +64,7 @@ $ipprc->redirect_output($logfile) or &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; + + +my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n'; # set this to 1 to enable checking for files on dead nodes @@ -91,4 +102,5 @@ # this stage uses 'chiptool' my $chiptool = can_run('chiptool') or die "Can't find chiptool"; + my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects"; # Get list of component imfiles @@ -96,6 +108,9 @@ my $imfiles; # Array of component files my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run + $command .= ' -all' if ($check_all); $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $very_verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -114,5 +129,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code); } exit 0; @@ -123,28 +138,104 @@ &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + my $numchips = scalar @$imfiles; + print "Found $numchips to clean\n"; + + my $clean_sources = 0; + if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) { + # go and find the smf file(s) for the associated camRun and check the status of the file + # if a good one is found we have the sources for this chipRun and thus can clean the cmfs + my $command = "$chiptool -listrun -chip_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code); + } + my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + my $good_smf = 0; + foreach my $entry (@$entries) { + my $camRun_state = $entry->{camRun_state}; + next if $camRun_state ne 'full'; + my $cam_id = $entry->{cam_id}; + if (!$cam_id) { + carp('no cam_id for listrun entry'); + next; + } + my $cam_path_base = $entry->{cam_path_base}; + if ( !defined $cam_path_base ) { + carp("no path_base for $cam_id\n"); + next; + } + + # XXX: This assumes that the filerules are filerules-split + my $smf = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); + if (!$ipprc->file_exists($smf)) { + carp("smf for $cam_path_base not found"); + next; + } + # we run the program censorObjects in the check mode + # If this program succeeds the smf is a valid fits file and each of the + # extensions was succesfully read. + # XXX: create a new program outside of magic that performs this check + + my $command = "$censorObjects -checkinputonly -file $smf"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $very_verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR "censorObjects failed:\n"; + print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf; + print STDERR "\nSTDERR:\n" . join "", @$stderr_buf; + &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code); + } + $good_smf++; + } + if ($good_smf) { + # we have a good one so we can clean the sources + $clean_sources = 1; + print "Found $good_smf good smf files will clean sources\n"; + } else { + print "Unable to find good smf file will NOT clean sources\n"; + } + } + # loop over all of the imfiles, determine the path_base and class_id for each + my $num_errors = 0; foreach my $imfile (@$imfiles) { my $class_id = $imfile->{class_id}; my $path_base = $imfile->{path_base}; + my $data_state = $imfile->{data_state}; my $status = 1; $status = 0 unless defined $path_base and $path_base ne "NULL"; - my $poor_quality = $imfile->{quality} > 0; + my $quality = $imfile->{quality}; + my $good_quality = ($quality == 0); + + print "Starting cleanup for $class_id\n"; # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent # goto_scrubbed now requires the config file to not exist. - if ($status and !$poor_quality) { + if ($status and $good_quality) { if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); unless ($ipprc->file_exists($config_file)) { + my $fault = $imfile->{fault}; + if (file_gone($config_file)) { - print STDERR "forcing cleanup for chipRun $stage_id $class_id " + # config file was lost. Clean up. If the chip is ever updated a new config + # file will be created + print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality" . " because config file ($config_file) is gone\n"; + } elsif ($fault == 0 and $quality == 0) { + print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality" + . " because config file ($config_file) is missing\n"; + $status = 0; } else { - print STDERR "skipping cleanup for chipRun $stage_id $class_id " - . " because config file ($config_file) is missing\n"; - $status = 0; + # config file is missing but this is a bad chip anyways so clean it + print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality" + . " even though config file ($config_file) is missing\n"; } } @@ -154,5 +245,5 @@ if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + print STDERR "skipping scrubbed for chip $stage_id $class_id " . " because config file ($config_file) is present\n"; $status = 0; @@ -165,21 +256,30 @@ my @files = (); - # delete the temporary image datafiles -# addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); -# addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); -# addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0); + if ($clean_sources) { + addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + } if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode + if (!$clean_sources) { + # these weren't added above but we do want to clean it + addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + } + + # background model is needed to build stack background images so we do not remove it + # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id); + + addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean? + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); } @@ -188,29 +288,42 @@ $status = &delete_files (\@files); } + bzip2_file("LOG.IMFILE", $path_base, $class_id); + bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id); if ($status) { + my $update_chip = 1; my $command = "$chiptool -chip_id $stage_id -class_id $class_id"; if ($mode eq "goto_purged") { $command .= " -topurgedimfile"; + if ($data_state eq 'purged') { + $update_chip = 0; + } } elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; + if ($data_state eq 'cleaned') { + $update_chip = 0; + } } elsif ($mode eq "goto_scrubbed") { $command .= " -toscrubbedimfile"; - } - - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); - } - - set_destreak_goto_cleaned(); - + if ($data_state eq 'scrubbed') { + $update_chip = 0; + } + } + + if ($update_chip) { + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + set_destreak_goto_cleaned(); + } } else { + $num_errors++; # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) @@ -218,6 +331,4 @@ $command .= " -dbname $dbname" if defined $dbname; -if (0) { - # XXX Don't set components to error cleaned anymore my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -226,5 +337,4 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } -} # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. @@ -241,4 +351,7 @@ } } + print "Cleanup completed for chip_id $stage_id."; + print " num_errors: $num_errors" if $num_errors; + print "\n"; exit 0; } @@ -350,7 +463,8 @@ my $skyfiles; # Array of component files my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run + $command .= ' -all' if $check_all; $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => $very_verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -378,8 +492,13 @@ &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); + my $numskycells = scalar @$skyfiles; + print "Found $numskycells to clean\n"; + my @files = (); + my $num_errors = 0; foreach my $skyfile (@$skyfiles) { my $path_base = $skyfile->{path_base}; my $skycell_id = $skyfile->{skycell_id}; + my $data_state = $skyfile->{data_state}; my $status = 1; @@ -391,11 +510,17 @@ unless ($ipprc->file_exists($config_file)) { + my $fault = $skyfile->{fault}; + my $quality = $skyfile->{quality}; if (file_gone($config_file)) { - print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" . - " because config file is gone\n"; + print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is gone\n"; + } elsif ($fault == 0 and $quality == 0) { + print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is missing\n"; + $status = 0; } else { - print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . - " because config file is missing\n"; - $status = 0; + # config file is missing but this is a bad warp anyways so clean it + print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality" + . " even though config file ($config_file) is missing\n"; } } @@ -413,5 +538,6 @@ if ($status) { - if ($skyfile->{quality} != 8007) { + # XXX: what is special about quality == 8007? + if ($skyfile->{quality} != 8007 || $check_all) { my @files = (); @@ -420,5 +546,7 @@ addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1); addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1); -# addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + # these are rebuilt during update so we can delete them here + addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id ); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode @@ -426,30 +554,58 @@ addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); - # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); - - # XXX: do we want to delete these? + addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + + # XXX: do we want to delete these? trace file is empty # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); - # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); - } - # actual command to delete the files + } + # actual command to delete the files $status = &delete_files (\@files); } } + bzip2_file("LOG.EXP", $path_base, $skycell_id); + bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id); if ($status) { + my $update_skyfile = 1; my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id"; if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; + if ($data_state eq 'purged') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; + if ($data_state eq 'cleaned') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_scrubbed") { $command .= " -toscrubbedskyfile"; - } - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + if ($data_state eq 'scrubbed') { + $update_skyfile = 0; + } + } + $command .= " -dbname $dbname" if defined $dbname; + + if ($update_skyfile) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + + set_destreak_goto_cleaned(); + } + + } else { + $num_errors++; + my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -457,11 +613,9 @@ } - set_destreak_goto_cleaned(); - - } else { - my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { @@ -469,8 +623,9 @@ &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); } - - # exit $PS_EXIT_UNKNOWN_ERROR; - } - } + } + } + print "Cleanup completed for warp_id $stage_id."; + print " num_errors: $num_errors" if $num_errors; + print "\n"; exit 0; } @@ -603,5 +758,4 @@ &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); } -# exit $PS_EXIT_UNKNOWN_ERROR; } } @@ -619,6 +773,7 @@ my $skyfiles; # Array reference of component files my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run + $command .= ' -all' if $check_all; $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -646,8 +801,10 @@ &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); + my $num_errors = 0; my @files = (); foreach my $skyfile (@{ $skyfiles }) { my $path_base = $skyfile->{path_base}; my $skycell_id = $skyfile->{skycell_id}; + my $data_state = $skyfile->{data_state}; my $status = 1; @@ -661,4 +818,5 @@ my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + if (0) { unless ($ipprc->file_exists($config_file)) { if (file_gone($config_file)) { @@ -672,4 +830,21 @@ } } + unless ($ipprc->file_exists($config_file)) { + my $fault = $skyfile->{fault}; + my $quality = $skyfile->{quality}; + if (file_gone($config_file)) { + print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is gone\n"; + } elsif ($fault == 0 and $quality == 0) { + print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is missing\n"; + $status = 0; + } else { + # config file is missing but this is a bad diff anyways so clean it + print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality" + . " even though config file ($config_file) is missing\n"; + } + } + } elsif ($mode eq "goto_scrubbed") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); @@ -712,21 +887,49 @@ } -# print STDERR "MY FILES: @files\n"; $status = &delete_files(\@files); } -# print STDERR "MY STATUS: $status\n"; + + bzip2_file("LOG.EXP", $path_base, $skycell_id); + bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id); + if ($status) { my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + my $update_skyfile = 1; if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; + if ($data_state eq 'purged') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; + if ($data_state eq 'cleaned') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_scrubbed") { $command .= " -toscrubbedskyfile"; - } - + if ($data_state eq 'scrubbed') { + $update_skyfile = 0; + } + } + + $command .= " -dbname $dbname" if defined $dbname; + + if ($update_skyfile) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + } + set_destreak_goto_cleaned(); + } + + + } else { + $num_errors++; + my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; @@ -738,12 +941,9 @@ } - set_destreak_goto_cleaned(); - - } else { - my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; - - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; + + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { @@ -751,7 +951,9 @@ &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); } -# exit $PS_EXIT_UNKNOWN_ERROR; - } - } + } + } + print "Cleanup completed for diff_id $stage_id."; + print " num_errors: $num_errors" if $num_errors; + print "\n"; exit 0; } @@ -1917,5 +2119,5 @@ foreach my $file (@$files) { - print STDERR "unlinking $stage $stage_id $file\n"; + print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose; my $error_code = $ipprc->kill_file($file); @@ -2008,6 +2210,4 @@ } -# this gets set to 1 the first time we set the corresponding destreak run to be cleaned -my $ds_done = 0; sub set_destreak_goto_cleaned { @@ -2027,4 +2227,37 @@ } +sub bzip2_file { + my $filerule = shift; + my $path_base = shift; + my $component = shift; + + my $filename = $ipprc->filename($filerule, $path_base, $component); + if (!$ipprc->file_exists($filename)) { + return 1; + } + if (my $resolved = $ipprc->file_resolve($filename)) { + my $bzip2_filename = $filename . '.bz2'; + if ($ipprc->file_exists($bzip2_filename)) { + $ipprc->kill_file($bzip2_filename); + } + my $bzip2_file = $ipprc->file_create($bzip2_filename); + my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file; + + my $command = "$bzip2 < $resolved > $bzip2_file"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $very_verbose); + if ($success) { + # success delete the original file + my $error_code = $ipprc->kill_file($filename); + } else { + # if bzip2 failed. Carry on but don't delete the existing file + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR "Failed to bzip2 $filename: $error_code\n"; + return 0; + } + } + return 1; +} + # XXX we currently do not set the error state in the db on my_die sub my_die Index: branches/czw_branch/20120906/ippTasks/science.cleanup.pro =================================================================== --- branches/czw_branch/20120906/ippTasks/science.cleanup.pro (revision 34785) +++ branches/czw_branch/20120906/ippTasks/science.cleanup.pro (revision 34786) @@ -13,4 +13,17 @@ check.globals + +# if set tell cleanup script to check for files from all components regardless of the data_state +$CHECK_ALL_COMPONENTS = 0 +macro set.check.all.components + $CHECK_ALL_COMPONENTS = 1 +end +macro clear.check.all.components + $CHECK_ALL_COMPONENTS = 0 +end +macro get.check.all.components + echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS +end + ## chip.cleanup.pro : -*- sh -*- @@ -130,14 +143,22 @@ book getword chipPendingCleanup $pageName state -var CLEANUP_MODE book getword chipPendingCleanup $pageName chip_id -var CHIP_ID + book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG + book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE book getword chipPendingCleanup $pageName dbname -var DBNAME - # specify choice of local or remote host based on camera and chip (class_id) + # specify choice of local or remote host based on camera and chip (in this case FPA) set.host.for.camera $CAMERA FPA + + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID stdout $LOGDIR/chip.cleanup.log stderr $LOGDIR/chip.cleanup.log - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + if ($CHECK_ALL_COMPONENTS) + $run = $run --check-all + end add_standard_args run @@ -282,4 +303,6 @@ book getword camPendingCleanup $pageName state -var CLEANUP_MODE book getword camPendingCleanup $pageName cam_id -var CAM_ID + book getword camPendingCleanup $pageName exp_tag -var EXP_TAG + book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE book getword camPendingCleanup $pageName dbname -var DBNAME @@ -287,9 +310,14 @@ set.host.for.camera $CAMERA FPA + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + ## generate LOGFILE specific to this exposure (& cam_id) + sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + stdout $LOGDIR/camera.cleanup.log stderr $LOGDIR/camera.cleanup.log - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE add_standard_args run @@ -591,14 +619,21 @@ book getword warpPendingCleanup $pageName state -var CLEANUP_MODE book getword warpPendingCleanup $pageName warp_id -var WARP_ID + book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG + book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE book getword warpPendingCleanup $pageName dbname -var DBNAME # specify choice of local or remote host based on camera and warp (class_id) set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID stdout $LOGDIR/warp.cleanup.log stderr $LOGDIR/warp.cleanup.log - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + if ($CHECK_ALL_COMPONENTS) + $run = $run --check-all + end add_standard_args run @@ -748,14 +783,21 @@ book getword diffCleanup $pageName state -var CLEANUP_MODE book getword diffCleanup $pageName diff_id -var DIFF_ID + book getword diffCleanup $pageName tess_id -var TESS_ID + book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE book getword diffCleanup $pageName dbname -var DBNAME # specify choice of local or remote host based on camera and diff (class_id) set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR + sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID + stdout $LOGDIR/diff.cleanup.log stderr $LOGDIR/diff.cleanup.log - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE + $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE + if ($CHECK_ALL_COMPONENTS) + $run = $run --check-all + end add_standard_args run @@ -899,4 +941,5 @@ book getword stackCleanup $pageName state -var CLEANUP_MODE book getword stackCleanup $pageName stack_id -var STACK_ID + book getword stackCleanup $pageName path_base -var PATH_BASE book getword stackCleanup $pageName dbname -var DBNAME @@ -907,6 +950,6 @@ stderr $LOGDIR/stack.cleanup.log - # XXX is everything listed here needed? - $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE + sprintf LOGFILE "%s.cleanup.log" PATH_BASE + $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE add_standard_args run Index: branches/czw_branch/20120906/ippTasks/skycal.pro =================================================================== --- branches/czw_branch/20120906/ippTasks/skycal.pro (revision 34785) +++ branches/czw_branch/20120906/ippTasks/skycal.pro (revision 34786) @@ -8,4 +8,20 @@ # test for required global variables check.globals + +# skcal.pro should have a more restricted polling limit (to avoid stress with getstar) +if ($?POLL_LIMIT_SKYCAL == 0) set POLL_LIMIT_SKYCAL = 25 + +macro set.skycal.poll + if ($0 != 2) + echo "USAGE:set.skycal.poll (value)" + break + end + + $POLL_LIMIT_SKYCAL = $1 +end + +macro get.skycal.poll + echo "skycal poll limit: $POLL_LIMIT_SKYCAL" +end ### Initialise the books containing the tasks to do @@ -79,5 +95,5 @@ # save the DB name for the exit tasks option $DB:$skycal_DB - $run = $run -dbname $DB:$skycal_DB -limit 40 + $run = $run -dbname $DB:$skycal_DB $skycal_DB ++ if ($skycal_DB >= $DB:n) set skycal_DB = 0 @@ -85,4 +101,5 @@ add_poll_args run add_poll_labels run + $run = $run -limit $POLL_LIMIT_SKYCAL command $run end @@ -197,9 +214,9 @@ host local - periods -poll 60.0 - periods -exec 1800.0 + periods -poll 10.0 + periods -exec 1200.0 periods -timeout 120.0 npending 1 - active false + active true stdout NULL Index: branches/czw_branch/20120906/ippTasks/staticsky.pro =================================================================== --- branches/czw_branch/20120906/ippTasks/staticsky.pro (revision 34785) +++ branches/czw_branch/20120906/ippTasks/staticsky.pro (revision 34786) @@ -70,6 +70,6 @@ periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 + periods -exec 30 + periods -timeout 60 npending 1 Index: branches/czw_branch/20120906/ippTasks/summit.copy.pro =================================================================== --- branches/czw_branch/20120906/ippTasks/summit.copy.pro (revision 34785) +++ branches/czw_branch/20120906/ippTasks/summit.copy.pro (revision 34786) @@ -223,5 +223,5 @@ periods -exec 10 periods -poll 1 - periods -timeout 20 + periods -timeout 60 npending 1 Index: branches/czw_branch/20120906/ippToPsps/jython/objectbatch.py =================================================================== --- branches/czw_branch/20120906/ippToPsps/jython/objectbatch.py (revision 34785) +++ branches/czw_branch/20120906/ippToPsps/jython/objectbatch.py (revision 34786) @@ -134,5 +134,5 @@ ," + filter[1] + "80pct = MAG_80/1000 \ ," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \ - ," + filter[1] + "StackDetectID = STACK_DETECT_ID + " + str(filterID) + ," + filter[1] + "StackDetectID = STACK_DETECT_ID" #self.logger.info(sql) Index: branches/czw_branch/20120906/ippTools/share/Makefile.am =================================================================== --- branches/czw_branch/20120906/ippTools/share/Makefile.am (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/Makefile.am (revision 34786) @@ -88,4 +88,5 @@ chiptool_listrun.sql \ chiptool_pendingcleanupimfile.sql \ + chiptool_pendingcleanupimfile_all.sql \ chiptool_pendingcleanuprun.sql \ chiptool_pendingimfile.sql \ @@ -172,4 +173,5 @@ difftool_pendingcleanuprun.sql \ difftool_pendingcleanupskyfile.sql \ + difftool_pendingcleanupskyfile_all.sql \ difftool_revertdiffskyfile_delete.sql \ difftool_revertdiffskyfile_updated.sql \ @@ -424,4 +426,5 @@ warptool_pendingcleanuprun.sql \ warptool_pendingcleanupskyfile.sql \ + warptool_pendingcleanupskyfile_all.sql \ warptool_revertoverlap.sql \ warptool_revertwarped_delete.sql \ Index: branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql (revision 34786) @@ -2,4 +2,6 @@ camRun.cam_id, rawExp.camera, + rawExp.exp_tag, + camRun.workdir, camRun.state FROM camRun Index: branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanupimfile_all.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanupimfile_all.sql (revision 34786) +++ branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanupimfile_all.sql (revision 34786) @@ -0,0 +1,16 @@ +-- list of chipProcessedImfiles for runs in a goto_cleaned like state without regard to +-- data_state or quality. Used for confirming that a run is really in the cleaned state +SELECT + chipProcessedImfile.*, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.reduction, + chipRun.expgroup, + chipRun.dvodb, + chipRun.tess_id, + chipRun.end_stage +FROM chipRun +JOIN chipProcessedImfile + USING(chip_id) +WHERE ( chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged' ) Index: branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql (revision 34786) @@ -1,6 +1,8 @@ -SELECT +SELECT DISTINCT chipRun.chip_id, rawExp.camera, - chipRun.state + rawExp.exp_tag, + chipRun.state, + chipRun.workdir FROM chipRun JOIN rawExp Index: branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql (revision 34786) @@ -6,4 +6,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun @@ -35,4 +37,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun @@ -64,4 +68,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun @@ -93,4 +99,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun Index: branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanupskyfile_all.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanupskyfile_all.sql (revision 34786) +++ branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanupskyfile_all.sql (revision 34786) @@ -0,0 +1,10 @@ +SELECT + diffSkyfile.*, + diffRun.state, + diffRun.workdir, + diffRun.dvodb, + diffRun.tess_id +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +WHERE ( diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') Index: branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql (revision 34786) @@ -4,4 +4,6 @@ SET pstampDependent.fault = 0 -- fault count hook %s -WHERE pstampDependent.state = 'new' +WHERE pstampRequest.state ='run' + AND pstampJob.state ='run' + AND pstampDependent.state = 'new' AND (pstampDependent.fault > 0) Index: branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql (revision 34786) @@ -8,6 +8,8 @@ stackRun.stack_id, rawExp.camera, - stackRun.state + stackRun.state, + stackSumSkyfile.path_base FROM stackRun +JOIN stackSumSkyfile using(stack_id) JOIN stackInputSkyfile USING(stack_id) Index: branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql (revision 34786) @@ -2,4 +2,6 @@ sky_id, stack_id, + skycell_id, + filter, staticskyRun.workdir, staticskyRun.label, @@ -11,4 +13,5 @@ JOIN stackRun USING(stack_id) JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) LEFT JOIN skycalRun USING(sky_id, stack_id) WHERE staticskyRun.state = 'full' Index: branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql (revision 34786) @@ -1,5 +1,7 @@ SELECT warpRun.warp_id, + warpRun.workdir, rawExp.camera, + rawExp.exp_tag, warpRun.state FROM warpRun Index: branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanupskyfile_all.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanupskyfile_all.sql (revision 34786) +++ branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanupskyfile_all.sql (revision 34786) @@ -0,0 +1,11 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.dvodb, + warpRun.end_stage +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_scrubbed' OR warpRun.state = 'goto_purged') Index: branches/czw_branch/20120906/ippTools/src/chiptool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/chiptool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/chiptool.c (revision 34786) @@ -1311,4 +1311,5 @@ PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -1319,5 +1320,6 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = pxDataGet("chiptool_pendingcleanupimfile.sql"); + char *sql_file = all ? "chiptool_pendingcleanupimfile_all.sql" : "chiptool_pendingcleanupimfile.sql"; + psString query = pxDataGet(sql_file); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); Index: branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c (revision 34786) @@ -296,4 +296,5 @@ psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-all", 0, "list all components regardless of data_state", false); psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/czw_branch/20120906/ippTools/src/difftool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/difftool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/difftool.c (revision 34786) @@ -2561,4 +2561,5 @@ PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -2570,5 +2571,6 @@ pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); - psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); + char * sql_file = all ? "difftool_pendingcleanupskyfile_all.sql": "difftool_pendingcleanupskyfile.sql" ; + psString query = pxDataGet(sql_file); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -2740,17 +2742,35 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - - if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!state) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } psFree (where); - return false; - } - psFree (where); + } else { + if (strcmp(state,"error_cleaned") == 0) { + change_skyfile_data_state(config,"error_cleaned", "goto_cleaned"); + } + else if (strcmp(state, "error_scrubbed") == 0) { + change_skyfile_data_state(config,"error_scrubbed", "goto_scrubbed"); + } + else if (strcmp(state, "error_purged") == 0) { + change_skyfile_data_state(config,"error_purged", "goto_purged"); + } + else { + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); + } + } return true; Index: branches/czw_branch/20120906/ippTools/src/difftoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/difftoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/difftoolConfig.c (revision 34786) @@ -395,4 +395,5 @@ psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all", 0, "list all skycells regardless of data_state", false); psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -411,4 +412,5 @@ psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -updatediffskyfile psMetadata *updatediffskyfileArgs = psMetadataAlloc(); @@ -416,5 +418,6 @@ psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); - psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set data_state state", NULL); // -tocleanedskyfile Index: branches/czw_branch/20120906/ippTools/src/staticskytool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/staticskytool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/staticskytool.c (revision 34786) @@ -132,14 +132,4 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); -#ifdef notdef - PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min", "skycell.glat", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max", "skycell.glat", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min", "skycell.radeg/15", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max", "skycell.radeg/15", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min", "skycell.radeg", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max", "skycell.radeg", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min", "skycell.decdeg", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max", "skycell.decdeg", "<="); -#endif if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -686,5 +676,5 @@ psStringAppend(&query, "\nGROUP BY sky_id"); if (num_filters) { - psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters); + psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters); } @@ -1049,4 +1039,8 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, whereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -1128,4 +1122,5 @@ psString sky_label = psMetadataLookupStr(&status, row, "label"); psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString sky_dist_group = psMetadataLookupStr(&status, row, "dist_group"); // create a staticskyRun @@ -1138,5 +1133,5 @@ label ? label : sky_label, data_group ? data_group : sky_data_group, - dist_group, + dist_group ? dist_group : sky_dist_group, reduction, registered, Index: branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c (revision 34786) @@ -126,5 +126,5 @@ psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); - psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0); pxskycellAddArguments(resultArgs); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -166,4 +166,5 @@ psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + pxskycellAddArguments(defineskycalrunArgs); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -183,4 +184,5 @@ psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + pxskycellAddArguments(updateskycalrunArgs); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); Index: branches/czw_branch/20120906/ippTools/src/warptool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/warptool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/warptool.c (revision 34786) @@ -1838,4 +1838,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); psMetadata *where = psMetadataAlloc(); @@ -1843,5 +1844,6 @@ pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); + char * sql_file = all ? "warptool_pendingcleanupskyfile_all.sql" : "warptool_pendingcleanupskyfile.sql"; + psString query = pxDataGet(sql_file); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); Index: branches/czw_branch/20120906/ippTools/src/warptoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/warptoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/warptoolConfig.c (revision 34786) @@ -364,4 +364,5 @@ psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID (required)", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all", 0, "list all skycells regardless of data_state", false); psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/czw_branch/20120906/ippconfig/gpc1/psphot.config =================================================================== --- branches/czw_branch/20120906/ippconfig/gpc1/psphot.config (revision 34785) +++ branches/czw_branch/20120906/ippconfig/gpc1/psphot.config (revision 34786) @@ -49,4 +49,14 @@ DIFF_STATS BOOL TRUE # Calculate diff-specific stats OUTPUT.FORMAT STR PS1_DV2 # Format of output file + + PSPHOT.EXT.NSIGMA.LIMIT F32 0.0 + PSPHOT.EXT.DIFF.ALTERNATE BOOL TRUE + PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.25 +END + +SSTF_DIFF METADATA + PSPHOT.EXT.NSIGMA.LIMIT F32 0.0 + PSPHOT.EXT.DIFF.ALTERNATE BOOL TRUE + PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.25 END Index: branches/czw_branch/20120906/ippconfig/recipes/psphot.config =================================================================== --- branches/czw_branch/20120906/ippconfig/recipes/psphot.config (revision 34785) +++ branches/czw_branch/20120906/ippconfig/recipes/psphot.config (revision 34786) @@ -345,4 +345,6 @@ PSPHOT.EXT.NSIGMA.LIMIT F32 3.0 # sources with extNsigma greater that this get tagged as likely extended sources PSPHOT.EXT.NSIGMA.MOMENTS F32 3.0 # sources with extNsigma greater that this get tagged as likely extended sources +PSPHOT.EXT.DIFF.ALTERNATE BOOL FALSE # use alternate extended source calculation; designed for diff trails +PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.5 # threshold for alternate extended source calculations PSPHOT.CR.GROW S32 1 # Number of pixels to grow CR mask PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights Index: branches/czw_branch/20120906/magic/censorObjects/src/censorLoop.c =================================================================== --- branches/czw_branch/20120906/magic/censorObjects/src/censorLoop.c (revision 34785) +++ branches/czw_branch/20120906/magic/censorObjects/src/censorLoop.c (revision 34786) @@ -29,6 +29,4 @@ - - // select the current recipe psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, CENSOR_RECIPE); @@ -58,4 +56,6 @@ // check on the mask file at this level if possible badMaskFile = censorBadFileByView(badMaskFile, inMask, config, view); + + bool checkInputOnly = psMetadataLookupBool(NULL, config->arguments, "CHECK.INPUT.ONLY"); // files associated with the science image @@ -116,5 +116,7 @@ if (!readout->data_exists) { continue; } - if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE; + if (!checkInputOnly) { + if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE; + } if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE; Index: branches/czw_branch/20120906/ppTranslate/src/ppMopsRead.c =================================================================== --- branches/czw_branch/20120906/ppTranslate/src/ppMopsRead.c (revision 34785) +++ branches/czw_branch/20120906/ppTranslate/src/ppMopsRead.c (revision 34786) @@ -28,5 +28,5 @@ for (int i = 0; i < num; i++) { const char *name = inNames->data[i]; - printf("%s\n", name); + printf("Input filename:\n%s\n", name); psFits *fits = psFitsOpen(name, "r"); // FITS file @@ -84,4 +84,5 @@ } ppMopsDetections *det = ppMopsDetectionsAlloc(size); + det->platescale = NAN; detections->data[i] = det; det->component = psStringNCopy(name, strrchr(name, '.') - name); // Strip off extension @@ -280,4 +281,5 @@ if (isfinite(scale)) { det->platescale = scale; + //printf("platescale = %g\n", scale); } numGood++; @@ -306,4 +308,5 @@ } + printf("Detection platescale = %g\n", det->platescale); psTrace("ppMops.read", 2, "Read %ld good rows from %s\n", numGood, (const char*)name); } Index: branches/czw_branch/20120906/ppTranslate/src/ppMopsWrite.c =================================================================== --- branches/czw_branch/20120906/ppTranslate/src/ppMopsWrite.c (revision 34785) +++ branches/czw_branch/20120906/ppTranslate/src/ppMopsWrite.c (revision 34786) @@ -51,4 +51,5 @@ float seeing = 0.; int totalGood = 0; + int platescale_has_been_set = 0; for (int d = 0; d < detections->n; d++) { if ( (detections->data[d] != NULL) && (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) ) { @@ -56,9 +57,19 @@ totalGood += 1; if (isfinite(((ppMopsDetections*) detections->data[d])->platescale)) { + if (platescale_has_been_set == 0) { det->platescale = ((ppMopsDetections*) detections->data[d])->platescale; - } - } - } - printf("plate-scale = [%g]\n", det->platescale); + platescale_has_been_set = 1; + } else { + if (det->platescale != ((ppMopsDetections*) detections->data[d])->platescale) { + printf("Different values for platescale: %g - %g in detection %d\n", + det->platescale, + ((ppMopsDetections*) detections->data[d])->platescale, + d); + } + } + } + } + } + //printf("plate-scale = [%g]\n", det->platescale); seeing /= (float) totalGood; if (det != NULL) { Index: branches/czw_branch/20120906/psphot/src/psphotSourceSize.c =================================================================== --- branches/czw_branch/20120906/psphot/src/psphotSourceSize.c (revision 34785) +++ branches/czw_branch/20120906/psphot/src/psphotSourceSize.c (revision 34786) @@ -11,4 +11,6 @@ float nSigmaMoments; float nSigmaCR; + bool altDiffExt; + float altDiffExtThresh; float soft; int grow; @@ -120,4 +122,11 @@ assert (status); + // Optional extended source measurement algorithm to improve diff image trails + options.altDiffExt = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE"); + assert (status); + // Threshold for this alternate method + options.altDiffExtThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE.THRESH"); + assert (status); + // location of a single test source options.xtest = psMetadataLookupS32 (&status, recipe, "PSPHOT.CRMASK.XTEST"); @@ -601,5 +610,24 @@ continue; } - + // Alternate extended source limit calculation + if (options->altDiffExt) { + // ratio of major to minor axes + // MRV = Major / Minor + // MRV = (0.5 * (Mxx + Myy) + 0.5 * sqrt( (Mxx + Myy)^2 + 4 Mxy^2)) / + // (0.5 * (Mxx + Myy) - 0.5 * sqrt( (Mxx + Myy)^2 + 4 Mxy^2)) + // MRV = (2 * 0.5 * (Mxx + Myy) - Minor) / Minor + float momentRatioVeres = (Mxx + Myy - Mminor) / Mminor; + bool isAltEXT = (momentRatioVeres > options->altDiffExtThresh); + if (isAltEXT) { + psTrace("psphotSourceClassRegion.EXT",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g ALTEXT\t%g %g\n", + source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid, + momentRatioVeres,options->altDiffExtThresh); + source->mode |= PM_SOURCE_MODE_EXT_LIMIT; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Next ++; + continue; + } + } + // Everything else should just be treated as a PSF psTrace("psphotSourceClassRegion.PSF",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g PSF\n",