IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 10, 2012, 4:35:46 PM (14 years ago)
Author:
watersc1
Message:

Merge from trunk

Location:
branches/czw_branch/20120906
Files:
41 edited
3 copied

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20120906

  • branches/czw_branch/20120906/Nebulous-Server/bin/neb-host

    r32919 r34786  
    141141$dbh->commit;
    142142
    143 my $format  = "%-15s %-15s %-7s %-8s %-9s %-5s %s\n";
    144 my @columns = qw(host name mounted allocate available xattr note);
     143my $format  = "%-15s %-15s %-7s %-8s %-9s %-5s %15s %s\n";
     144my @columns = qw(host name mounted allocate available xattr last_modified note);
    145145printf($format, @columns);
    146146

  • branches/czw_branch/20120906/Nebulous-Server/lib/Nebulous/Server/SQL.pm

    r33826 r34786  
    389389            mountedvol.vol_id IS NOT NULL as mounted,
    390390            v.cab_id,
     391            v.last_modified,
    391392            v.note
    392393        FROM volume AS v
    … …  
    694695    cab_id INT,
    695696    note VARCHAR(255),
     697    last_modified TIMESTAMP NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, # SC: Added by Haydn with ALTER TABLE(?) on 2012-12-03
    696698    PRIMARY KEY(vol_id),
    697699    UNIQUE KEY(name),

  • branches/czw_branch/20120906/Ohana/src/libohana/src

  • branches/czw_branch/20120906/Ohana/src/opihi

  • branches/czw_branch/20120906/Ohana/src/opihi/cmd.astro

  • branches/czw_branch/20120906/Ohana/src/opihi/cmd.data

  • branches/czw_branch/20120906/ippScripts/scripts/chip_imfile.pl

    r33052 r34786  
    150150$outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
    151151my $do_binned_images = 1;
     152
    152153if ($run_state eq 'new') {
    153154    # prepare the files that are only created for a new run
    … …  
    586587
    587588# check whether psphot outputs should be regenerated.
    588 # Whether we need to or not is a somewhat complicated question.
     589# Whether we need to or not was a somewhat complicated question. Since we clean cmfs now we only rerun
     590# photometry if the psf file has gone missing since it is needed to run warps
     591
    589592sub rerun_photometry
    590593{
    … …  
    594597                    or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    595598
     599    my $update_sources_if_gone = 0; # set this to regenerate sources if gone.
     600                                    # We no longer do this as of 2012-12
    596601    my $make_sources = 0;
    597602    my $sources_available = 0;
    … …  
    599604        $sources_available = 1;
    600605    } else {
    601         carp "WARNING: photometry sources file $outputSources is not available";
    602         my $gone;
    603         if (storage_object_exists($outputSources, \$gone)) {
    604             # check whether the file is permanantely or temporarily gone
    605             if ($gone) {
    606                 carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     606        if ($update_sources_if_gone) {
     607            carp "WARNING: photometry sources file $outputSources is not available";
     608            my $gone;
     609            if (storage_object_exists($outputSources, \$gone)) {
     610                # check whether the file is permanantely or temporarily gone
     611                if ($gone) {
     612                    carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     613                    $make_sources = 1;
     614                }
     615            } else {
     616                # storage object must have been deleted
    607617                $make_sources = 1;
    608618            }
    609         } else {
    610             # storage object must have been deleted
    611             $make_sources = 1;
    612619        }
    613620    }
    … …  
    616623    my $psf_available = 0;
    617624    my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
    618                     or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     625                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE",
     626                                $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     627
    619628    if ($ipprc->file_exists($outputPsf)) {
    620629        $psf_available = 1;
    … …  
    638647    }
    639648
    640     # if either of the files are gone rerun photometry unless the other file is temporarily not available
    641 
    642     if (!$sources_available && !$make_sources) {
    643         # destreak will die if the sources is not available
    644         &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
    645     }
     649    if ($update_sources_if_gone) {
     650        if (!$sources_available && !$make_sources) {
     651            # destreak will die if the sources is not available
     652            # but magic is dead....
     653            &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     654        }
     655    }
     656
    646657    if (!$psf_available && !$make_psf) {
    647658        # warp updates need the psf file

  • branches/czw_branch/20120906/ippScripts/scripts/ipp_apply_burntool_single.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20120906/ippScripts/scripts/ipp_cleanup.pl

    r33664 r34786  
    1818
    1919# Parse the command-line arguments
    20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile);
     20my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all);
     21my $very_verbose = 0;
     22
     23# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
     24#my $ds_done = 0;
     25# magic is dead
     26my $ds_done = 1;
     27
    2128GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
    2229           'camera|i=s'     => \$camera,    # user-supplied camera name
    … …  
    2431           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
    2532           'path_base=s'    => \$path_base, # basename for files
     33           'check-all'      => \$check_all, # if set clean all chips regardless of data_state
    2634           'dbname|d=s'     => \$dbname,    # Database name
    2735           'verbose'        => \$verbose,   # Print to stdout
    … …  
    5664$ipprc->redirect_output($logfile) or
    5765        &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
     66
     67
     68my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n';
    5869
    5970# set this to 1 to enable checking for files on dead nodes
    … …  
    91102    # this stage uses 'chiptool'
    92103    my $chiptool = can_run('chiptool') or die "Can't find chiptool";
     104    my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects";
    93105
    94106    # Get list of component imfiles
    … …  
    96108    my $imfiles;                      # Array of component files
    97109    my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
     110    $command .= ' -all' if ($check_all);
    98111    $command .= " -dbname $dbname" if defined $dbname;
    99     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     112
     113    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
     114        = run(command => $command, verbose => $very_verbose);
    100115    unless ($success) {
    101116        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    114129        unless ($success) {
    115130            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    116             &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     131            &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code);
    117132        }
    118133        exit 0;
    … …  
    123138        &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
    124139
     140    my $numchips = scalar @$imfiles;
     141    print "Found $numchips to clean\n";
     142
     143    my $clean_sources = 0;
     144    if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) {
     145        # go and find the smf file(s) for the associated camRun and check the status of the file
     146        # if a good one is found we have the sources for this chipRun and thus can clean the cmfs
     147        my $command = "$chiptool -listrun -chip_id $stage_id";
     148        $command .= " -dbname $dbname" if defined $dbname;
     149        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
     150        unless ($success) {
     151            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     152            &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code);
     153        }
     154        my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
     155            &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
     156        my $good_smf = 0;
     157        foreach my $entry (@$entries) {
     158            my $camRun_state = $entry->{camRun_state};
     159            next if $camRun_state  ne 'full';
     160            my $cam_id = $entry->{cam_id};
     161            if (!$cam_id) {
     162                carp('no cam_id for listrun entry');
     163                next;
     164            }
     165            my $cam_path_base = $entry->{cam_path_base};
     166            if ( !defined $cam_path_base ) {
     167                carp("no path_base for $cam_id\n");
     168                next;
     169            }
     170
     171            # XXX: This assumes that the filerules are filerules-split
     172            my $smf =  $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
     173            if (!$ipprc->file_exists($smf)) {
     174                carp("smf for $cam_path_base not found");
     175                next;
     176            }
     177            # we run the program censorObjects in the check mode
     178            # If this program succeeds the smf is a valid fits file and each of the
     179            # extensions was succesfully read.
     180            # XXX: create a new program outside of magic that performs this check
     181
     182            my $command = "$censorObjects -checkinputonly -file $smf";
     183            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     184                                        run(command => $command, verbose => $very_verbose);
     185            unless ($success) {
     186                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     187                print STDERR "censorObjects failed:\n";
     188                print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf;
     189                print STDERR "\nSTDERR:\n" . join "", @$stderr_buf;
     190                &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
     191            }
     192            $good_smf++;
     193        }
     194        if ($good_smf) {
     195            # we have a good one so we can clean the sources
     196            $clean_sources = 1;
     197            print "Found $good_smf good smf files will clean sources\n";
     198        } else {
     199            print "Unable to find good smf file will NOT clean sources\n";
     200        }
     201    }
     202
    125203    # loop over all of the imfiles, determine the path_base and class_id for each
     204    my $num_errors = 0;
    126205    foreach my $imfile (@$imfiles) {
    127206        my $class_id = $imfile->{class_id};
    128207        my $path_base = $imfile->{path_base};
     208        my $data_state = $imfile->{data_state};
    129209        my $status = 1;
    130210        $status = 0 unless defined $path_base and $path_base ne "NULL";
    131211
    132         my $poor_quality = $imfile->{quality} > 0;
     212        my $quality = $imfile->{quality};
     213        my $good_quality = ($quality == 0);
     214
     215        print "Starting cleanup for $class_id\n";
    133216
    134217        # don't clean up unless the data needed to update is available
    135218        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    136219        # goto_scrubbed now requires the config file to not exist.
    137         if ($status and !$poor_quality) {
     220        if ($status and $good_quality) {
    138221            if ($mode eq "goto_cleaned") {
    139222                my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    140223
    141224                unless ($ipprc->file_exists($config_file)) {
     225                    my $fault = $imfile->{fault};
     226
    142227                    if (file_gone($config_file)) {
    143                         print STDERR "forcing cleanup for chipRun $stage_id $class_id "
     228                        # config file was lost. Clean up. If the chip is ever updated a new config
     229                        # file will be created
     230                        print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality"
    144231                            . " because config file ($config_file) is gone\n";
     232                    } elsif ($fault == 0 and $quality == 0) {
     233                            print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
     234                                . " because config file ($config_file) is missing\n";
     235                            $status = 0;
    145236                    } else {
    146                         print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    147                             . " because config file ($config_file) is missing\n";
    148                         $status = 0;
     237                            # config file is missing but this is a bad chip anyways so clean it
     238                            print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
     239                                . " even though config file ($config_file) is missing\n";
    149240                    }
    150241                }
    … …  
    154245
    155246                if ($ipprc->file_exists($config_file)) {
    156                     print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
     247                    print STDERR "skipping scrubbed for chip $stage_id $class_id "
    157248                        . " because config file ($config_file) is present\n";
    158249                    $status = 0;
    … …  
    165256            my @files = ();
    166257
    167             # delete the temporary image datafiles
    168 #            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    169 #            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
    170 #            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
    171258            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
    172259            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
    173260            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
    174261            addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0);
     262            if ($clean_sources) {
     263                addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
     264                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     265            }
    175266            if ($mode eq "goto_purged") {
    176267                # additional files to remove for 'purge' mode
     268                if (!$clean_sources) {
     269                    # these weren't added above but we do want to clean it
     270                    addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
     271                    addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     272                }
     273               
     274                # background model is needed to build stack background images so we do not remove it
     275                # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id);
     276
     277                addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id);
    177278                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
    178279                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
    179                 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);          # clean?
    180                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);          # clean?
     280                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    181281                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
    182282                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
    183                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);         #clean?
     283                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    184284                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
    185285            }
    … …  
    188288            $status = &delete_files (\@files);
    189289        }
     290        bzip2_file("LOG.IMFILE", $path_base, $class_id);
     291        bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id);
    190292
    191293        if ($status)  {
     294            my $update_chip = 1;
    192295            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
    193296            if ($mode eq "goto_purged") {
    194297                $command .= " -topurgedimfile";
     298                if ($data_state eq 'purged') {
     299                    $update_chip = 0;
     300                }
    195301            }
    196302            elsif ($mode eq "goto_cleaned") {
    197303                $command .= " -tocleanedimfile";
     304                if ($data_state eq 'cleaned') {
     305                    $update_chip = 0;
     306                }
    198307            }
    199308            elsif ($mode eq "goto_scrubbed") {
    200309                $command .= " -toscrubbedimfile";
    201             }
    202 
    203             $command .= " -dbname $dbname" if defined $dbname;
    204 
    205             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    206                     run(command => $command, verbose => $verbose);
    207             unless ($success) {
    208                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    209                 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    210             }
    211 
    212             set_destreak_goto_cleaned();
    213 
     310                if ($data_state eq 'scrubbed') {
     311                    $update_chip = 0;
     312                }
     313            }
     314
     315            if ($update_chip) {
     316                $command .= " -dbname $dbname" if defined $dbname;
     317
     318                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     319                        run(command => $command, verbose => $verbose);
     320                unless ($success) {
     321                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     322                    &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     323                }
     324                set_destreak_goto_cleaned();
     325            }
    214326        } else {
     327            $num_errors++;
    215328
    216329            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    … …  
    218331            $command .= " -dbname $dbname" if defined $dbname;
    219332
    220 if (0) {
    221         # XXX Don't set components to error cleaned anymore
    222333            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    223334                    run(command => $command, verbose => $verbose);
    … …  
    226337                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    227338            }
    228 }
    229339
    230340            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
    … …  
    241351        }
    242352    }
     353    print "Cleanup completed for chip_id $stage_id.";
     354    print " num_errors: $num_errors" if $num_errors;
     355    print "\n";
    243356    exit 0;
    244357}
    … …  
    350463    my $skyfiles;                      # Array of component files
    351464    my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
     465    $command .= ' -all' if $check_all;
    352466    $command .= " -dbname $dbname" if defined $dbname;
    353467    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    354             run(command => $command, verbose => $verbose);
     468            run(command => $command, verbose => $very_verbose);
    355469    unless ($success) {
    356470        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    378492        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
    379493
     494    my $numskycells = scalar @$skyfiles;
     495    print "Found $numskycells to clean\n";
     496
    380497    my @files = ();
     498    my $num_errors = 0;
    381499    foreach my $skyfile (@$skyfiles) {
    382500        my $path_base = $skyfile->{path_base};
    383501        my $skycell_id = $skyfile->{skycell_id};
     502        my $data_state = $skyfile->{data_state};
    384503
    385504        my $status = 1;
    … …  
    391510
    392511                unless ($ipprc->file_exists($config_file)) {
     512                    my $fault = $skyfile->{fault};
     513                    my $quality = $skyfile->{quality};
    393514                    if (file_gone($config_file)) {
    394                         print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
    395                             " because config file is gone\n";
     515                        print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality"
     516                            . " because config file ($config_file) is gone\n";
     517                    } elsif ($fault == 0 and $quality == 0) {
     518                            print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
     519                                . " because config file ($config_file) is missing\n";
     520                            $status = 0;
    396521                    } else {
    397                         print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
    398                             " because config file is missing\n";
    399                         $status = 0;
     522                            # config file is missing but this is a bad warp anyways so clean it
     523                            print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
     524                                . " even though config file ($config_file) is missing\n";
    400525                    }
    401526                }
    … …  
    413538
    414539        if ($status) {
    415             if ($skyfile->{quality} != 8007) {
     540            # XXX: what is special about quality == 8007?
     541            if ($skyfile->{quality} != 8007 || $check_all) {
    416542                my @files = ();
    417543
    … …  
    420546                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
    421547                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
    422 #            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     548                # these are rebuilt during update so we can delete them here
     549                addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     550                addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id );
    423551                if ($mode eq "goto_purged") {
    424552                    # additional files to remove for 'purge' mode
    … …  
    426554                    addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
    427555                    addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
    428                     # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
    429 
    430                     # XXX: do we want to delete these?
     556                    addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id );
     557                    addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
     558
     559                    # XXX: do we want to delete these? trace file is empty
    431560                    # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
    432                     # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
    433                 }
    434             # actual command to delete the files
     561                }
     562                # actual command to delete the files
    435563                $status = &delete_files (\@files);
    436564            }
    437565        }
     566        bzip2_file("LOG.EXP", $path_base, $skycell_id);
     567        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
    438568
    439569        if ($status)  {
     570            my $update_skyfile = 1;
    440571            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
    441572            if ($mode eq "goto_purged") {
    442573                $command .= " -topurgedskyfile";
     574                if ($data_state eq 'purged') {
     575                    $update_skyfile = 0;
     576                }
    443577            }
    444578            elsif ($mode eq "goto_cleaned") {
    445579                $command .= " -tocleanedskyfile";
     580                if ($data_state eq 'cleaned') {
     581                    $update_skyfile = 0;
     582                }
    446583            }
    447584            elsif ($mode eq "goto_scrubbed") {
    448585                $command .= " -toscrubbedskyfile";
    449             }
    450             $command .= " -dbname $dbname" if defined $dbname;
    451 
    452             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    453                     run(command => $command, verbose => $verbose);
     586                if ($data_state eq 'scrubbed') {
     587                    $update_skyfile = 0;
     588                }
     589            }
     590            $command .= " -dbname $dbname" if defined $dbname;
     591
     592            if ($update_skyfile) {
     593                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     594                        run(command => $command, verbose => $verbose);
     595                unless ($success) {
     596                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     597                    &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     598                }
     599
     600                set_destreak_goto_cleaned();
     601            }
     602
     603         } else {
     604            $num_errors++;
     605            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
     606            $command .= " -dbname $dbname" if defined $dbname;
     607
     608            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     609                run(command => $command, verbose => $verbose);
    454610            unless ($success) {
    455611                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    457613            }
    458614
    459             set_destreak_goto_cleaned();
    460 
    461          } else {
    462             my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    463             $command .= " -dbname $dbname" if defined $dbname;
    464 
    465             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     615            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
     616            $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state";
     617            $command .= " -dbname $dbname" if defined $dbname;
     618
     619            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    466620                run(command => $command, verbose => $verbose);
    467621            unless ($success) {
    … …  
    469623                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    470624            }
    471 
    472             #            exit $PS_EXIT_UNKNOWN_ERROR;
    473         }
    474     }
     625        }
     626    }
     627    print "Cleanup completed for warp_id $stage_id.";
     628    print " num_errors: $num_errors" if $num_errors;
     629    print "\n";
    475630    exit 0;
    476631}
    … …  
    603758                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    604759            }
    605 #           exit $PS_EXIT_UNKNOWN_ERROR;
    606760        }
    607761    }
    … …  
    619773    my $skyfiles;                  # Array reference of component files
    620774    my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
     775    $command .= ' -all' if $check_all;
    621776    $command .= " -dbname $dbname" if defined $dbname;
    622     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     777    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
    623778    unless ($success) {
    624779        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    646801        &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
    647802
     803    my $num_errors = 0;
    648804    my @files = ();
    649805    foreach my $skyfile (@{ $skyfiles }) {
    650806        my $path_base = $skyfile->{path_base};
    651807        my $skycell_id = $skyfile->{skycell_id};
     808        my $data_state = $skyfile->{data_state};
    652809
    653810        my $status = 1;
    … …  
    661818                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    662819
     820            if (0) {
    663821                unless ($ipprc->file_exists($config_file)) {
    664822                    if (file_gone($config_file)) {
    … …  
    672830                }
    673831            }
     832                unless ($ipprc->file_exists($config_file)) {
     833                    my $fault = $skyfile->{fault};
     834                    my $quality = $skyfile->{quality};
     835                    if (file_gone($config_file)) {
     836                        print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality"
     837                            . " because config file ($config_file) is gone\n";
     838                    } elsif ($fault == 0 and $quality == 0) {
     839                            print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
     840                                . " because config file ($config_file) is missing\n";
     841                            $status = 0;
     842                    } else {
     843                            # config file is missing but this is a bad diff anyways so clean it
     844                            print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
     845                                . " even though config file ($config_file) is missing\n";
     846                    }
     847                }
     848            }
    674849            elsif ($mode eq "goto_scrubbed") {
    675850                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    … …  
    712887
    713888            }
    714 #           print STDERR "MY FILES: @files\n";
    715889            $status = &delete_files(\@files);
    716890        }
    717 #       print STDERR "MY STATUS: $status\n";
     891
     892        bzip2_file("LOG.EXP", $path_base, $skycell_id);
     893        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
     894
    718895        if ($status) {
    719896            my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
     897            my $update_skyfile = 1;
    720898
    721899            if ($mode eq "goto_purged") {
    722900                $command .= " -topurgedskyfile";
     901                if ($data_state eq 'purged') {
     902                    $update_skyfile = 0;
     903                }
    723904            }
    724905            elsif ($mode eq "goto_cleaned") {
    725906                $command .= " -tocleanedskyfile";
     907                if ($data_state eq 'cleaned') {
     908                    $update_skyfile = 0;
     909                }
    726910            }
    727911            elsif ($mode eq "goto_scrubbed") {
    728912                $command .= " -toscrubbedskyfile";
    729             }
    730 
     913                if ($data_state eq 'scrubbed') {
     914                    $update_skyfile = 0;
     915                }
     916            }
     917
     918            $command .= " -dbname $dbname" if defined $dbname;
     919
     920            if ($update_skyfile) {
     921                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     922                    run(command => $command, verbose => $verbose);
     923                unless ($success) {
     924                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     925                    &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     926                }
     927                set_destreak_goto_cleaned();
     928            }
     929
     930
     931        } else {
     932            $num_errors++;
     933            my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    731934            $command .= " -dbname $dbname" if defined $dbname;
    732935
    … …  
    738941            }
    739942
    740             set_destreak_goto_cleaned();
    741 
    742         } else {
    743             my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
    744 
    745             $command .= " -dbname $dbname" if defined $dbname;
    746 
    747             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     943            $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
     944
     945            $command .= " -dbname $dbname" if defined $dbname;
     946
     947            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    748948                run(command => $command, verbose => $verbose);
    749949            unless ($success) {
    … …  
    751951                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    752952            }
    753 #           exit $PS_EXIT_UNKNOWN_ERROR;
    754         }
    755     }
     953        }
     954    }
     955    print "Cleanup completed for diff_id $stage_id.";
     956    print " num_errors: $num_errors" if $num_errors;
     957    print "\n";
    756958    exit 0;
    757959}
    … …  
    19172119
    19182120    foreach my $file (@$files) {
    1919         print STDERR "unlinking $stage $stage_id $file\n";
     2121        print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose;
    19202122
    19212123        my $error_code = $ipprc->kill_file($file);
    … …  
    20082210}
    20092211
    2010 # this gets set to 1 the first time we set the corresponding destreak run to be cleaned
    2011 my $ds_done = 0;
    20122212sub set_destreak_goto_cleaned {
    20132213
    … …  
    20272227}
    20282228
     2229sub bzip2_file {
     2230    my $filerule = shift;
     2231    my $path_base = shift;
     2232    my $component = shift;
     2233
     2234    my $filename = $ipprc->filename($filerule, $path_base, $component);
     2235    if (!$ipprc->file_exists($filename)) {
     2236        return 1;
     2237    }
     2238    if (my $resolved = $ipprc->file_resolve($filename)) {
     2239        my $bzip2_filename = $filename . '.bz2';
     2240        if ($ipprc->file_exists($bzip2_filename)) {
     2241            $ipprc->kill_file($bzip2_filename);
     2242        }
     2243        my $bzip2_file = $ipprc->file_create($bzip2_filename);
     2244        my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
     2245
     2246        my $command = "$bzip2 < $resolved > $bzip2_file";
     2247        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     2248            run(command => $command, verbose => $very_verbose);
     2249        if ($success) {
     2250            # success delete the original file
     2251            my $error_code = $ipprc->kill_file($filename);
     2252        } else {
     2253            # if bzip2 failed. Carry on but don't delete the existing file
     2254            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     2255            print STDERR "Failed to bzip2 $filename: $error_code\n";
     2256            return 0;
     2257        }
     2258    }
     2259    return 1;
     2260}
     2261
    20292262# XXX we currently do not set the error state in the db on my_die
    20302263sub my_die

  • branches/czw_branch/20120906/ippTasks/science.cleanup.pro

    r34203 r34786  
    1313
    1414check.globals
     15
     16# if set tell cleanup script to check for files from all components regardless of the data_state
     17$CHECK_ALL_COMPONENTS = 0
     18macro set.check.all.components
     19    $CHECK_ALL_COMPONENTS = 1
     20end
     21macro clear.check.all.components
     22    $CHECK_ALL_COMPONENTS = 0
     23end
     24macro get.check.all.components
     25    echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS
     26end
     27
    1528
    1629## chip.cleanup.pro : -*- sh -*-
    … …  
    130143    book getword chipPendingCleanup $pageName state -var CLEANUP_MODE
    131144    book getword chipPendingCleanup $pageName chip_id -var CHIP_ID
     145    book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG
     146    book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
    132147    book getword chipPendingCleanup $pageName dbname -var DBNAME
    133148
    134     # specify choice of local or remote host based on camera and chip (class_id)
     149    # specify choice of local or remote host based on camera and chip (in this case FPA)
    135150    set.host.for.camera $CAMERA FPA
     151
     152    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     153
     154    sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID
    136155
    137156    stdout $LOGDIR/chip.cleanup.log
    138157    stderr $LOGDIR/chip.cleanup.log
    139158
    140     # XXX is everything listed here needed?
    141     $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE
     159    $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     160    if ($CHECK_ALL_COMPONENTS)
     161        $run = $run --check-all
     162    end
    142163    add_standard_args run
    143164
    … …  
    282303    book getword camPendingCleanup $pageName state  -var CLEANUP_MODE
    283304    book getword camPendingCleanup $pageName cam_id -var CAM_ID
     305    book getword camPendingCleanup $pageName exp_tag -var EXP_TAG
     306    book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
    284307    book getword camPendingCleanup $pageName dbname -var DBNAME
    285308
    … …  
    287310    set.host.for.camera $CAMERA FPA
    288311
     312    # set the WORKDIR variable
     313    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     314
     315    ## generate LOGFILE specific to this exposure (& cam_id)
     316    sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
     317
    289318    stdout $LOGDIR/camera.cleanup.log
    290319    stderr $LOGDIR/camera.cleanup.log
    291320
    292     # XXX is everything listed here needed?
    293     $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE
     321    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
    294322    add_standard_args run
    295323
    … …  
    591619    book getword warpPendingCleanup $pageName state -var CLEANUP_MODE
    592620    book getword warpPendingCleanup $pageName warp_id -var WARP_ID
     621    book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG
     622    book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
    593623    book getword warpPendingCleanup $pageName dbname -var DBNAME
    594624
    595625    # specify choice of local or remote host based on camera and warp (class_id)
    596626    set.host.for.camera $CAMERA FPA
     627    set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     628
     629    sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID
    597630
    598631    stdout $LOGDIR/warp.cleanup.log
    599632    stderr $LOGDIR/warp.cleanup.log
    600633
    601     # XXX is everything listed here needed?
    602     $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE
     634    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     635    if ($CHECK_ALL_COMPONENTS)
     636        $run = $run --check-all
     637    end
    603638    add_standard_args run
    604639
    … …  
    748783    book getword diffCleanup $pageName state -var CLEANUP_MODE
    749784    book getword diffCleanup $pageName diff_id -var DIFF_ID
     785    book getword diffCleanup $pageName tess_id -var TESS_ID
     786    book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE
    750787    book getword diffCleanup $pageName dbname -var DBNAME
    751788
    752789    # specify choice of local or remote host based on camera and diff (class_id)
    753790    set.host.for.camera $CAMERA FPA
     791    set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     792    sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID
     793
    754794
    755795    stdout $LOGDIR/diff.cleanup.log
    756796    stderr $LOGDIR/diff.cleanup.log
    757797
    758     # XXX is everything listed here needed?
    759     $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE
     798    $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     799    if ($CHECK_ALL_COMPONENTS)
     800        $run = $run --check-all
     801    end
    760802    add_standard_args run
    761803
    … …  
    899941    book getword stackCleanup $pageName state -var CLEANUP_MODE
    900942    book getword stackCleanup $pageName stack_id -var STACK_ID
     943    book getword stackCleanup $pageName path_base -var PATH_BASE
    901944    book getword stackCleanup $pageName dbname -var DBNAME
    902945
    … …  
    907950    stderr $LOGDIR/stack.cleanup.log
    908951
    909     # XXX is everything listed here needed?
    910     $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE
     952    sprintf LOGFILE "%s.cleanup.log" PATH_BASE
     953    $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
    911954    add_standard_args run
    912955

  • branches/czw_branch/20120906/ippTasks/skycal.pro

    r32983 r34786  
    88# test for required global variables
    99check.globals
     10
     11# skcal.pro should have a more restricted polling limit (to avoid stress with getstar)
     12if ($?POLL_LIMIT_SKYCAL == 0) set POLL_LIMIT_SKYCAL = 25
     13
     14macro set.skycal.poll
     15  if ($0 != 2)
     16    echo "USAGE:set.skycal.poll (value)"
     17    break
     18  end
     19 
     20  $POLL_LIMIT_SKYCAL = $1
     21end
     22
     23macro get.skycal.poll
     24  echo "skycal poll limit: $POLL_LIMIT_SKYCAL"
     25end
    1026
    1127### Initialise the books containing the tasks to do
    … …  
    7995      # save the DB name for the exit tasks
    8096      option $DB:$skycal_DB
    81       $run = $run -dbname $DB:$skycal_DB -limit 40
     97      $run = $run -dbname $DB:$skycal_DB
    8298      $skycal_DB ++
    8399      if ($skycal_DB >= $DB:n) set skycal_DB = 0
    … …  
    85101    add_poll_args run
    86102    add_poll_labels run
     103    $run = $run -limit $POLL_LIMIT_SKYCAL
    87104    command $run
    88105  end
    … …  
    197214  host         local
    198215
    199   periods      -poll 60.0
    200   periods      -exec 1800.0
     216  periods      -poll 10.0
     217  periods      -exec 1200.0
    201218  periods      -timeout 120.0
    202219  npending     1
    203   active false
     220  active true
    204221 
    205222  stdout NULL

  • branches/czw_branch/20120906/ippTasks/staticsky.pro

    r33957 r34786  
    7070
    7171  periods      -poll $LOADPOLL
    72   periods      -exec $LOADEXEC
    73   periods      -timeout 30
     72  periods      -exec 30
     73  periods      -timeout 60
    7474  npending     1
    7575

  • branches/czw_branch/20120906/ippTasks/summit.copy.pro

    r33168 r34786  
    223223    periods      -exec     10
    224224    periods      -poll     1
    225     periods      -timeout  20
     225    periods      -timeout  60
    226226    npending     1
    227227

  • branches/czw_branch/20120906/ippToPsps/jython/objectbatch.py

    r34772 r34786  
    134134                   ," + filter[1] + "80pct = MAG_80/1000 \
    135135                   ," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \
    136                    ," + filter[1] + "StackDetectID = STACK_DETECT_ID + " + str(filterID)
     136                   ," + filter[1] + "StackDetectID = STACK_DETECT_ID"
    137137
    138138            #self.logger.info(sql)

  • branches/czw_branch/20120906/ippTools/share/Makefile.am

    r34772 r34786  
    8888        chiptool_listrun.sql \
    8989        chiptool_pendingcleanupimfile.sql \
     90        chiptool_pendingcleanupimfile_all.sql \
    9091        chiptool_pendingcleanuprun.sql \
    9192        chiptool_pendingimfile.sql \
    … …  
    172173        difftool_pendingcleanuprun.sql \
    173174        difftool_pendingcleanupskyfile.sql \
     175        difftool_pendingcleanupskyfile_all.sql \
    174176        difftool_revertdiffskyfile_delete.sql \
    175177        difftool_revertdiffskyfile_updated.sql \
    … …  
    424426        warptool_pendingcleanuprun.sql \
    425427        warptool_pendingcleanupskyfile.sql \
     428        warptool_pendingcleanupskyfile_all.sql \
    426429        warptool_revertoverlap.sql \
    427430        warptool_revertwarped_delete.sql \

  • branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql

    r25324 r34786  
    22    camRun.cam_id,
    33    rawExp.camera,
     4    rawExp.exp_tag,
     5    camRun.workdir,
    46    camRun.state
    57FROM camRun

  • branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql

    r22751 r34786  
    1 SELECT
     1SELECT DISTINCT
    22    chipRun.chip_id,
    33    rawExp.camera,
    4     chipRun.state
     4    rawExp.exp_tag,
     5    chipRun.state,
     6    chipRun.workdir
    57FROM chipRun
    68JOIN rawExp

  • branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql

    r32218 r34786  
    66            rawExp.camera,
    77            diffRun.state,
     8            diffRun.workdir,
     9            diffRun.tess_id,
    810            diffRun.label
    911        FROM diffRun
    … …  
    3537            rawExp.camera,
    3638            diffRun.state,
     39            diffRun.workdir,
     40            diffRun.tess_id,
    3741            diffRun.label
    3842        FROM diffRun
    … …  
    6468            rawExp.camera,
    6569            diffRun.state,
     70            diffRun.workdir,
     71            diffRun.tess_id,
    6672            diffRun.label
    6773        FROM diffRun
    … …  
    9399            rawExp.camera,
    94100            diffRun.state,
     101            diffRun.workdir,
     102            diffRun.tess_id,
    95103            diffRun.label
    96104        FROM diffRun

  • branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql

    r29286 r34786  
    44SET pstampDependent.fault = 0
    55-- fault count hook %s
    6 WHERE pstampDependent.state = 'new'
     6WHERE pstampRequest.state ='run'
     7    AND pstampJob.state ='run'
     8    AND pstampDependent.state = 'new'
    79    AND (pstampDependent.fault > 0)

  • branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql

    r24186 r34786  
    88    stackRun.stack_id,
    99    rawExp.camera,
    10     stackRun.state
     10    stackRun.state,
     11    stackSumSkyfile.path_base
    1112FROM stackRun
     13JOIN stackSumSkyfile using(stack_id)
    1214JOIN stackInputSkyfile
    1315    USING(stack_id)

  • branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql

    r32960 r34786  
    22    sky_id,
    33    stack_id,
     4    skycell_id,
     5    filter,
    46    staticskyRun.workdir,
    57    staticskyRun.label,
    … …  
    1113    JOIN stackRun USING(stack_id)
    1214    JOIN stackSumSkyfile USING(stack_id)
     15    JOIN skycell USING(tess_id, skycell_id)
    1316    LEFT JOIN skycalRun USING(sky_id, stack_id)
    1417WHERE staticskyRun.state = 'full'

  • branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql

    r23418 r34786  
    11SELECT
    22    warpRun.warp_id,
     3    warpRun.workdir,
    34    rawExp.camera,
     5    rawExp.exp_tag,
    46    warpRun.state
    57FROM warpRun

  • branches/czw_branch/20120906/ippTools/src/chiptool.c

    r34772 r34786  
    13111311    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false);
    13121312    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     1313    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    13131314    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    13141315
    … …  
    13191320    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    13201321
    1321     psString query = pxDataGet("chiptool_pendingcleanupimfile.sql");
     1322    char *sql_file = all ? "chiptool_pendingcleanupimfile_all.sql" : "chiptool_pendingcleanupimfile.sql";
     1323    psString query = pxDataGet(sql_file);
    13221324    if (!query) {
    13231325        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");

  • branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c

    r34772 r34786  
    296296    psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0,          "search by chip ID", 0);
    297297    psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id",                 0,            "search by exp_id", NULL);
     298    psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-all",  0,            "list all components regardless of data_state", false);
    298299    psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    299300    psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);

  • branches/czw_branch/20120906/ippTools/src/difftool.c

    r34772 r34786  
    25612561
    25622562    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false);
     2563    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    25632564    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    25642565    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    … …  
    25702571    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
    25712572
    2572     psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
     2573    char * sql_file  = all ? "difftool_pendingcleanupskyfile_all.sql": "difftool_pendingcleanupskyfile.sql" ;
     2574    psString query = pxDataGet(sql_file);
    25732575    if (!query) {
    25742576        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    … …  
    27402742    PS_ASSERT_PTR_NON_NULL(config, false);
    27412743
    2742     PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
    2743     PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
    2744 
    2745     psMetadata *where = psMetadataAlloc();
    2746     PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");
    2747     PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id",   "==");
    2748 
    2749     if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) {
    2750         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
     2744    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
     2745
     2746    if (!state) {
     2747        psMetadata *where = psMetadataAlloc();
     2748        PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");
     2749        PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id",   "==");
     2750
     2751        PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
     2752        PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
     2753
     2754        if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) {
     2755            psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
     2756            psFree (where);
     2757            return false;
     2758        }
    27512759        psFree (where);
    2752         return false;
    2753     }
    2754     psFree (where);
     2760    } else {
     2761      if (strcmp(state,"error_cleaned") == 0) {
     2762        change_skyfile_data_state(config,"error_cleaned", "goto_cleaned");
     2763      }
     2764      else if (strcmp(state, "error_scrubbed") == 0) {
     2765        change_skyfile_data_state(config,"error_scrubbed", "goto_scrubbed");
     2766      }
     2767      else if (strcmp(state, "error_purged") == 0) {
     2768        change_skyfile_data_state(config,"error_purged", "goto_purged");
     2769      }
     2770      else {
     2771        psError(PS_ERR_UNKNOWN, false, "unhandled state given");
     2772        return(false);
     2773      }
     2774    }
    27552775
    27562776    return true;

  • branches/czw_branch/20120906/ippTools/src/difftoolConfig.c

    r34772 r34786  
    395395    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
    396396    psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,          "search by diff ID", 0);
     397    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all",  0,            "list all skycells regardless of data_state", false);
    397398    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    398399    psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    … …  
    411412    psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
    412413    psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     414
    413415    // -updatediffskyfile
    414416    psMetadata *updatediffskyfileArgs = psMetadataAlloc();
    … …  
    416418    psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,   "search by skycell ID", NULL);
    417419    psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0,        "set fault code (required)", 0);
    418     psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0,      "set quality", 0);
     420    psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0,  "set quality", 0);
     421    psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-set_state", 0,    "set data_state state", NULL);
    419422
    420423    // -tocleanedskyfile

  • branches/czw_branch/20120906/ippTools/src/staticskytool.c

    r34772 r34786  
    132132    pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "stackRun.data_group",       "LIKE");
    133133    pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
    134 #ifdef notdef
    135     PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min",      "skycell.glat",              ">=");
    136     PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max",      "skycell.glat",              "<=");
    137     PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min",   "skycell.radeg/15",          ">=");
    138     PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max",   "skycell.radeg/15",          "<=");
    139     PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min",     "skycell.radeg",             ">=");
    140     PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max",     "skycell.radeg",             "<=");
    141     PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min",    "skycell.decdeg",             ">=");
    142     PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max",    "skycell.decdeg",             "<=");
    143 #endif
    144134    if (!pxskycellAddWhere(config, whereMD)) {
    145135        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
    … …  
    686676    psStringAppend(&query, "\nGROUP BY sky_id");
    687677    if (num_filters) {
    688         psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters);
     678        psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters);
    689679    }
    690680       
    … …  
    10491039    pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "staticskyRun.data_group",       "LIKE");
    10501040    pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
     1041    if (!pxskycellAddWhere(config, whereMD)) {
     1042        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
     1043        return false;
     1044    }
    10511045
    10521046    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
    … …  
    11281122        psString sky_label =  psMetadataLookupStr(&status, row, "label");
    11291123        psString sky_data_group =  psMetadataLookupStr(&status, row, "data_group");
     1124        psString sky_dist_group =  psMetadataLookupStr(&status, row, "dist_group");
    11301125
    11311126        // create a staticskyRun
    … …  
    11381133                                label ? label : sky_label,
    11391134                                data_group ? data_group : sky_data_group,
    1140                                 dist_group,
     1135                                dist_group ? dist_group : sky_dist_group,
    11411136                                reduction,
    11421137                                registered,

  • branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c

    r34772 r34786  
    126126    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
    127127    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
    128     psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0);
     128    psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0);
    129129    pxskycellAddArguments(resultArgs);
    130130    psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
    … …  
    166166    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
    167167    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
     168    pxskycellAddArguments(defineskycalrunArgs);
    168169    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
    169170    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
    … …  
    183184    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
    184185    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
     186    pxskycellAddArguments(updateskycalrunArgs);
    185187    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL);
    186188    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);

  • branches/czw_branch/20120906/ippTools/src/warptool.c

    r34772 r34786  
    18381838    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    18391839    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1840    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    18401841
    18411842    psMetadata *where = psMetadataAlloc();
    … …  
    18431844    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
    18441845
    1845     psString query = pxDataGet("warptool_pendingcleanupskyfile.sql");
     1846    char * sql_file = all ? "warptool_pendingcleanupskyfile_all.sql" : "warptool_pendingcleanupskyfile.sql";
     1847    psString query = pxDataGet(sql_file);
    18461848    if (!query) {
    18471849        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");

  • branches/czw_branch/20120906/ippTools/src/warptoolConfig.c

    r34772 r34786  
    364364    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
    365365    psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,          "search by warp ID (required)", 0);
     366    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all",  0,            "list all skycells regardless of data_state", false);
    366367    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    367368    psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);

  • branches/czw_branch/20120906/ippconfig/gpc1/psphot.config

    r34307 r34786  
    4949  DIFF_STATS                        BOOL TRUE          # Calculate diff-specific stats
    5050  OUTPUT.FORMAT                     STR  PS1_DV2       # Format of output file
     51
     52  PSPHOT.EXT.NSIGMA.LIMIT           F32  0.0
     53  PSPHOT.EXT.DIFF.ALTERNATE         BOOL TRUE
     54  PSPHOT.EXT.DIFF.ALTERNATE.THRESH  F32  1.25
     55END
     56
     57SSTF_DIFF METADATA
     58  PSPHOT.EXT.NSIGMA.LIMIT           F32  0.0
     59  PSPHOT.EXT.DIFF.ALTERNATE         BOOL TRUE
     60  PSPHOT.EXT.DIFF.ALTERNATE.THRESH  F32  1.25
    5161END
    5262

  • branches/czw_branch/20120906/ippconfig/recipes/ppSub.config

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20120906/ippconfig/recipes/psphot.config

    r34772 r34786  
    345345PSPHOT.EXT.NSIGMA.LIMIT             F32   3.0  # sources with extNsigma greater that this get tagged as likely extended sources
    346346PSPHOT.EXT.NSIGMA.MOMENTS           F32   3.0  # sources with extNsigma greater that this get tagged as likely extended sources
     347PSPHOT.EXT.DIFF.ALTERNATE           BOOL  FALSE           # use alternate extended source calculation; designed for diff trails
     348PSPHOT.EXT.DIFF.ALTERNATE.THRESH    F32   1.5             # threshold for alternate extended source calculations
    347349PSPHOT.CR.GROW                      S32   1               # Number of pixels to grow CR mask
    348350PSPHOT.CR.NSIGMA.SOFTEN             F32   0.0025          # Softening parameter for weights

  • branches/czw_branch/20120906/ippconfig/recipes/reductionClasses.mdc

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/czw_branch/20120906/magic/censorObjects/src/censorLoop.c

    r33626 r34786  
    2929
    3030
    31 
    32 
    3331    // select the current recipe
    3432    psMetadata *recipe  = psMetadataLookupPtr (NULL, config->recipes, CENSOR_RECIPE);
    … …  
    5856    // check on the mask file at this level if possible
    5957    badMaskFile = censorBadFileByView(badMaskFile, inMask, config, view);
     58
     59    bool checkInputOnly = psMetadataLookupBool(NULL, config->arguments, "CHECK.INPUT.ONLY");
    6060
    6161    // files associated with the science image
    … …  
    116116                if (!readout->data_exists) { continue; }
    117117
    118                 if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE;
     118                if (!checkInputOnly) {
     119                    if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE;
     120                }
    119121
    120122                if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;

  • branches/czw_branch/20120906/ppTranslate/src/ppMopsRead.c

    r34772 r34786  
    2828  for (int i = 0; i < num; i++) {
    2929    const char *name = inNames->data[i];
    30     printf("%s\n", name);
     30    printf("Input filename:\n%s\n", name);
    3131
    3232    psFits *fits = psFitsOpen(name,  "r"); // FITS file
    … …  
    8484    }
    8585    ppMopsDetections *det = ppMopsDetectionsAlloc(size);
     86    det->platescale = NAN;
    8687    detections->data[i] = det;
    8788    det->component = psStringNCopy(name, strrchr(name, '.') - name); // Strip off extension
    … …  
    280281      if (isfinite(scale)) {
    281282          det->platescale = scale;
     283          //printf("platescale = %g\n", scale);
    282284      }
    283285      numGood++;
    … …  
    306308    }
    307309
     310    printf("Detection platescale = %g\n", det->platescale);
    308311    psTrace("ppMops.read", 2, "Read %ld good rows from %s\n", numGood, (const char*)name);
    309312  }

  • branches/czw_branch/20120906/ppTranslate/src/ppMopsWrite.c

    r34772 r34786  
    5151  float seeing = 0.;
    5252  int totalGood = 0;
     53  int platescale_has_been_set = 0;
    5354  for (int d = 0; d < detections->n; d++) {
    5455    if ( (detections->data[d] != NULL) && (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) ) {
    … …  
    5657      totalGood += 1;
    5758      if (isfinite(((ppMopsDetections*) detections->data[d])->platescale)) {
     59          if (platescale_has_been_set == 0) {
    5860          det->platescale = ((ppMopsDetections*) detections->data[d])->platescale;
    59       }
    60     }
    61   }
    62   printf("plate-scale = [%g]\n", det->platescale);
     61              platescale_has_been_set = 1;
     62          } else {
     63              if (det->platescale != ((ppMopsDetections*) detections->data[d])->platescale) {
     64                  printf("Different values for platescale: %g - %g in detection %d\n",
     65                         det->platescale,
     66                         ((ppMopsDetections*) detections->data[d])->platescale,
     67                         d);
     68      }
     69    }
     70  }
     71    }
     72  }
     73  //printf("plate-scale = [%g]\n", det->platescale);
    6374  seeing /= (float) totalGood;
    6475  if (det != NULL) {

  • branches/czw_branch/20120906/psphot/src

  • branches/czw_branch/20120906/psphot/src/psphotSourceSize.c

    r34772 r34786  
    1111    float nSigmaMoments;
    1212    float nSigmaCR;
     13    bool altDiffExt;
     14    float altDiffExtThresh;
    1315    float soft;
    1416    int grow;
    … …  
    120122    assert (status);
    121123
     124    // Optional extended source measurement algorithm to improve diff image trails
     125    options.altDiffExt = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE");
     126    assert (status);
     127    // Threshold for this alternate method
     128    options.altDiffExtThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE.THRESH");
     129    assert (status);
     130   
    122131    // location of a single test source
    123132    options.xtest = psMetadataLookupS32 (&status, recipe, "PSPHOT.CRMASK.XTEST");
    … …  
    601610            continue;
    602611        }
    603 
     612        // Alternate extended source limit calculation
     613        if (options->altDiffExt) {
     614          // ratio of major to minor axes
     615          // MRV = Major / Minor
     616          // MRV = (0.5 * (Mxx + Myy) + 0.5 * sqrt( (Mxx + Myy)^2 + 4 Mxy^2)) /
     617          //       (0.5 * (Mxx + Myy) - 0.5 * sqrt( (Mxx + Myy)^2 + 4 Mxy^2))
     618          // MRV = (2 * 0.5 * (Mxx + Myy) - Minor) / Minor
     619          float momentRatioVeres = (Mxx + Myy - Mminor) / Mminor;
     620          bool  isAltEXT = (momentRatioVeres > options->altDiffExtThresh);
     621          if (isAltEXT) {
     622            psTrace("psphotSourceClassRegion.EXT",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g ALTEXT\t%g %g\n",
     623                    source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid,
     624                    momentRatioVeres,options->altDiffExtThresh);
     625            source->mode |= PM_SOURCE_MODE_EXT_LIMIT;
     626            source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED;
     627            Next ++;
     628            continue;
     629          }
     630        }
     631       
    604632        // Everything else should just be treated as a PSF
    605633        psTrace("psphotSourceClassRegion.PSF",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g PSF\n",

  • branches/czw_branch/20120906/psphot/src/psphotStackImageLoop.c

    • Property svn:mergeinfo changed (with no actual effect on merging)
Note: See TracChangeset for help on using the changeset viewer.