IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 18, 2012, 5:57:19 AM (14 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20121130
Files:
58 edited
6 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-host

    r32919 r34839  
    141141$dbh->commit;
    142142
    143 my $format  = "%-15s %-15s %-7s %-8s %-9s %-5s %s\n";
    144 my @columns = qw(host name mounted allocate available xattr note);
     143my $format  = "%-15s %-15s %-7s %-8s %-9s %-5s %15s %s\n";
     144my @columns = qw(host name mounted allocate available xattr last_modified note);
    145145printf($format, @columns);
    146146

  • branches/eam_branches/ipp-20121130/Nebulous-Server/bin/neb-voladm

    r32615 r34839  
    6868        if (defined $allocate and $allocate !~ m/^[01]$/)
    6969        or (defined $available and $available !~ m/^[01]$/)
    70         or (defined $xattr and $xattr !~ m/^[012]$/);
     70        or (defined $xattr and $xattr !~ m/^[0123]$/);
    7171}
    7272

  • branches/eam_branches/ipp-20121130/Nebulous-Server/lib/Nebulous/Server/SQL.pm

    r33826 r34839  
    389389            mountedvol.vol_id IS NOT NULL as mounted,
    390390            v.cab_id,
     391            v.last_modified,
    391392            v.note
    392393        FROM volume AS v
    … …  
    694695    cab_id INT,
    695696    note VARCHAR(255),
     697    last_modified TIMESTAMP NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, # SC: Added by Haydn with ALTER TABLE(?) on 2012-12-03
    696698    PRIMARY KEY(vol_id),
    697699    UNIQUE KEY(name),

  • branches/eam_branches/ipp-20121130/ippScripts/Build.PL

    r33627 r34839  
    128128        scripts/listvideocells.pl
    129129        scripts/skycalibration.pl
     130        scripts/regenerate_background.pl
    130131    )],
    131132    dist_abstract => 'Scripts for running the Pan-STARRS IPP',

  • branches/eam_branches/ipp-20121130/ippScripts/MANIFEST

    r31435 r34839  
    4646scripts/skycell_jpeg.pl
    4747scripts/lap_science.pl
     48scripts/regenerate_background.pl
    4849t/00_distribution.t

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl

    r33052 r34839  
    150150$outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
    151151my $do_binned_images = 1;
     152
    152153if ($run_state eq 'new') {
    153154    # prepare the files that are only created for a new run
    … …  
    586587
    587588# check whether psphot outputs should be regenerated.
    588 # Whether we need to or not is a somewhat complicated question.
     589# Whether we need to or not was a somewhat complicated question. Since we clean cmfs now we only rerun
     590# photometry if the psf file has gone missing since it is needed to run warps
     591
    589592sub rerun_photometry
    590593{
    … …  
    594597                    or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    595598
     599    my $update_sources_if_gone = 0; # set this to regenerate sources if gone.
     600                                    # We no longer do this as of 2012-12
    596601    my $make_sources = 0;
    597602    my $sources_available = 0;
    … …  
    599604        $sources_available = 1;
    600605    } else {
    601         carp "WARNING: photometry sources file $outputSources is not available";
    602         my $gone;
    603         if (storage_object_exists($outputSources, \$gone)) {
    604             # check whether the file is permanantely or temporarily gone
    605             if ($gone) {
    606                 carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     606        if ($update_sources_if_gone) {
     607            carp "WARNING: photometry sources file $outputSources is not available";
     608            my $gone;
     609            if (storage_object_exists($outputSources, \$gone)) {
     610                # check whether the file is permanantely or temporarily gone
     611                if ($gone) {
     612                    carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     613                    $make_sources = 1;
     614                }
     615            } else {
     616                # storage object must have been deleted
    607617                $make_sources = 1;
    608618            }
    609         } else {
    610             # storage object must have been deleted
    611             $make_sources = 1;
    612619        }
    613620    }
    … …  
    616623    my $psf_available = 0;
    617624    my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
    618                     or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     625                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE",
     626                                $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     627
    619628    if ($ipprc->file_exists($outputPsf)) {
    620629        $psf_available = 1;
    … …  
    638647    }
    639648
    640     # if either of the files are gone rerun photometry unless the other file is temporarily not available
    641 
    642     if (!$sources_available && !$make_sources) {
    643         # destreak will die if the sources is not available
    644         &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
    645     }
     649    if ($update_sources_if_gone) {
     650        if (!$sources_available && !$make_sources) {
     651            # destreak will die if the sources is not available
     652            # but magic is dead....
     653            &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     654        }
     655    }
     656
    646657    if (!$psf_available && !$make_psf) {
    647658        # warp updates need the psf file

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/destreak_restore_camera.pl

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl

    r33666 r34839  
    2727my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
    2828my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
     29my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
     30
    2931if ($missing_tools) {
    3032    warn("Can't find required tools.");
    … …  
    124126# Run ppImage
    125127unless ($no_op) {
     128    my $repair_cmd = "$nebrepair $input_uri";
     129    my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
     130    unless ($repair_success) {
     131        &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id $PS_EXIT_SYS_ERROR);
     132    }
     133
     134
    126135    my $command = "$ppImage -file $input_uri $outroot";
    127136    $command .= " -recipe PPIMAGE $ppimage_recipe";

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_apply_burntool_single.pl

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl

    r33664 r34839  
    1818
    1919# Parse the command-line arguments
    20 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile);
     20my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all);
     21my $very_verbose = 0;
     22
     23# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
     24#my $ds_done = 0;
     25# magic is dead
     26my $ds_done = 1;
     27
    2128GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
    2229           'camera|i=s'     => \$camera,    # user-supplied camera name
    … …  
    2431           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
    2532           'path_base=s'    => \$path_base, # basename for files
     33           'check-all'      => \$check_all, # if set clean all chips regardless of data_state
    2634           'dbname|d=s'     => \$dbname,    # Database name
    2735           'verbose'        => \$verbose,   # Print to stdout
    … …  
    5664$ipprc->redirect_output($logfile) or
    5765        &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
     66
     67
     68my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n';
    5869
    5970# set this to 1 to enable checking for files on dead nodes
    … …  
    91102    # this stage uses 'chiptool'
    92103    my $chiptool = can_run('chiptool') or die "Can't find chiptool";
     104    my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects";
    93105
    94106    # Get list of component imfiles
    … …  
    96108    my $imfiles;                      # Array of component files
    97109    my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
     110    $command .= ' -all' if ($check_all);
    98111    $command .= " -dbname $dbname" if defined $dbname;
    99     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     112
     113    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
     114        = run(command => $command, verbose => $very_verbose);
    100115    unless ($success) {
    101116        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    114129        unless ($success) {
    115130            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    116             &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     131            &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code);
    117132        }
    118133        exit 0;
    … …  
    123138        &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
    124139
     140    my $numchips = scalar @$imfiles;
     141    print "Found $numchips to clean\n";
     142
     143    my $clean_sources = 0;
     144    if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) {
     145        # go and find the smf file(s) for the associated camRun and check the status of the file
     146        # if a good one is found we have the sources for this chipRun and thus can clean the cmfs
     147        my $command = "$chiptool -listrun -chip_id $stage_id";
     148        $command .= " -dbname $dbname" if defined $dbname;
     149        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
     150        unless ($success) {
     151            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     152            &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code);
     153        }
     154        my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
     155            &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
     156        my $good_smf = 0;
     157        foreach my $entry (@$entries) {
     158            my $camRun_state = $entry->{camRun_state};
     159            next if $camRun_state  ne 'full';
     160            my $cam_id = $entry->{cam_id};
     161            if (!$cam_id) {
     162                carp('no cam_id for listrun entry');
     163                next;
     164            }
     165            my $cam_path_base = $entry->{cam_path_base};
     166            if ( !defined $cam_path_base ) {
     167                carp("no path_base for $cam_id\n");
     168                next;
     169            }
     170
     171            # XXX: This assumes that the filerules are filerules-split
     172            my $smf =  $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
     173            if (!$ipprc->file_exists($smf)) {
     174                carp("smf for $cam_path_base not found");
     175                next;
     176            }
     177            # we run the program censorObjects in the check mode
     178            # If this program succeeds the smf is a valid fits file and each of the
     179            # extensions was succesfully read.
     180            # XXX: create a new program outside of magic that performs this check
     181
     182            my $command = "$censorObjects -checkinputonly -file $smf";
     183            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     184                                        run(command => $command, verbose => $very_verbose);
     185            unless ($success) {
     186                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     187                print STDERR "censorObjects failed:\n";
     188                print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf;
     189                print STDERR "\nSTDERR:\n" . join "", @$stderr_buf;
     190                &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
     191            }
     192            $good_smf++;
     193        }
     194        if ($good_smf) {
     195            # we have a good one so we can clean the sources
     196            $clean_sources = 1;
     197            print "Found $good_smf good smf files will clean sources\n";
     198        } else {
     199            print "Unable to find good smf file will NOT clean sources\n";
     200        }
     201    }
     202
    125203    # loop over all of the imfiles, determine the path_base and class_id for each
     204    my $num_errors = 0;
    126205    foreach my $imfile (@$imfiles) {
    127206        my $class_id = $imfile->{class_id};
    128207        my $path_base = $imfile->{path_base};
     208        my $data_state = $imfile->{data_state};
    129209        my $status = 1;
    130210        $status = 0 unless defined $path_base and $path_base ne "NULL";
    131211
    132         my $poor_quality = $imfile->{quality} > 0;
     212        my $quality = $imfile->{quality};
     213        my $good_quality = ($quality == 0);
     214
     215        print "Starting cleanup for $class_id\n";
    133216
    134217        # don't clean up unless the data needed to update is available
    135218        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
    136219        # goto_scrubbed now requires the config file to not exist.
    137         if ($status and !$poor_quality) {
     220        if ($status and $good_quality) {
    138221            if ($mode eq "goto_cleaned") {
    139222                my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
    140223
    141224                unless ($ipprc->file_exists($config_file)) {
     225                    my $fault = $imfile->{fault};
     226
    142227                    if (file_gone($config_file)) {
    143                         print STDERR "forcing cleanup for chipRun $stage_id $class_id "
     228                        # config file was lost. Clean up. If the chip is ever updated a new config
     229                        # file will be created
     230                        print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality"
    144231                            . " because config file ($config_file) is gone\n";
     232                    } elsif ($fault == 0 and $quality == 0) {
     233                            print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
     234                                . " because config file ($config_file) is missing\n";
     235                            $status = 0;
    145236                    } else {
    146                         print STDERR "skipping cleanup for chipRun $stage_id $class_id "
    147                             . " because config file ($config_file) is missing\n";
    148                         $status = 0;
     237                            # config file is missing but this is a bad chip anyways so clean it
     238                            print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
     239                                . " even though config file ($config_file) is missing\n";
    149240                    }
    150241                }
    … …  
    154245
    155246                if ($ipprc->file_exists($config_file)) {
    156                     print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
     247                    print STDERR "skipping scrubbed for chip $stage_id $class_id "
    157248                        . " because config file ($config_file) is present\n";
    158249                    $status = 0;
    … …  
    165256            my @files = ();
    166257
    167             # delete the temporary image datafiles
    168 #            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
    169 #            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
    170 #            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
    171258            addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
    172259            addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
    173260            addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
    174261            addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0);
     262            if ($clean_sources) {
     263                addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
     264                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     265            }
    175266            if ($mode eq "goto_purged") {
    176267                # additional files to remove for 'purge' mode
     268                if (!$clean_sources) {
     269                    # these weren't added above but we do want to clean it
     270                    addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
     271                    addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
     272                }
     273               
     274                # background model is needed to build stack background images so we do not remove it
     275                # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id);
     276
     277                addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id);
    177278                addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
    178279                addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
    179                 addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);          # clean?
    180                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);          # clean?
     280                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
    181281                addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
    182282                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
    183                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);         #clean?
     283                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
    184284                addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
    185285            }
    … …  
    188288            $status = &delete_files (\@files);
    189289        }
     290        bzip2_file("LOG.IMFILE", $path_base, $class_id);
     291        bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id);
    190292
    191293        if ($status)  {
     294            my $update_chip = 1;
    192295            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
    193296            if ($mode eq "goto_purged") {
    194297                $command .= " -topurgedimfile";
     298                if ($data_state eq 'purged') {
     299                    $update_chip = 0;
     300                }
    195301            }
    196302            elsif ($mode eq "goto_cleaned") {
    197303                $command .= " -tocleanedimfile";
     304                if ($data_state eq 'cleaned') {
     305                    $update_chip = 0;
     306                }
    198307            }
    199308            elsif ($mode eq "goto_scrubbed") {
    200309                $command .= " -toscrubbedimfile";
    201             }
    202 
    203             $command .= " -dbname $dbname" if defined $dbname;
    204 
    205             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    206                     run(command => $command, verbose => $verbose);
    207             unless ($success) {
    208                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    209                 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    210             }
    211 
    212             set_destreak_goto_cleaned();
    213 
     310                if ($data_state eq 'scrubbed') {
     311                    $update_chip = 0;
     312                }
     313            }
     314
     315            if ($update_chip) {
     316                $command .= " -dbname $dbname" if defined $dbname;
     317
     318                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     319                        run(command => $command, verbose => $verbose);
     320                unless ($success) {
     321                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     322                    &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
     323                }
     324                set_destreak_goto_cleaned();
     325            }
    214326        } else {
     327            $num_errors++;
    215328
    216329            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
    … …  
    218331            $command .= " -dbname $dbname" if defined $dbname;
    219332
    220 if (0) {
    221         # XXX Don't set components to error cleaned anymore
    222333            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    223334                    run(command => $command, verbose => $verbose);
    … …  
    226337                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
    227338            }
    228 }
    229339
    230340            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
    … …  
    241351        }
    242352    }
     353    print "Cleanup completed for chip_id $stage_id.";
     354    print " num_errors: $num_errors" if $num_errors;
     355    print "\n";
    243356    exit 0;
    244357}
    … …  
    350463    my $skyfiles;                      # Array of component files
    351464    my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
     465    $command .= ' -all' if $check_all;
    352466    $command .= " -dbname $dbname" if defined $dbname;
    353467    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    354             run(command => $command, verbose => $verbose);
     468            run(command => $command, verbose => $very_verbose);
    355469    unless ($success) {
    356470        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    378492        &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
    379493
     494    my $numskycells = scalar @$skyfiles;
     495    print "Found $numskycells to clean\n";
     496
    380497    my @files = ();
     498    my $num_errors = 0;
    381499    foreach my $skyfile (@$skyfiles) {
    382500        my $path_base = $skyfile->{path_base};
    383501        my $skycell_id = $skyfile->{skycell_id};
     502        my $data_state = $skyfile->{data_state};
    384503
    385504        my $status = 1;
    … …  
    391510
    392511                unless ($ipprc->file_exists($config_file)) {
     512                    my $fault = $skyfile->{fault};
     513                    my $quality = $skyfile->{quality};
    393514                    if (file_gone($config_file)) {
    394                         print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
    395                             " because config file is gone\n";
     515                        print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality"
     516                            . " because config file ($config_file) is gone\n";
     517                    } elsif ($fault == 0 and $quality == 0) {
     518                            print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
     519                                . " because config file ($config_file) is missing\n";
     520                            $status = 0;
    396521                    } else {
    397                         print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
    398                             " because config file is missing\n";
    399                         $status = 0;
     522                            # config file is missing but this is a bad warp anyways so clean it
     523                            print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
     524                                . " even though config file ($config_file) is missing\n";
    400525                    }
    401526                }
    … …  
    413538
    414539        if ($status) {
    415             if ($skyfile->{quality} != 8007) {
     540            # XXX: what is special about quality == 8007?
     541            if ($skyfile->{quality} != 8007 || $check_all) {
    416542                my @files = ();
    417543
    … …  
    420546                addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
    421547                addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
    422 #            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     548                # these are rebuilt during update so we can delete them here
     549                addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
     550                addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id );
    423551                if ($mode eq "goto_purged") {
    424552                    # additional files to remove for 'purge' mode
    … …  
    426554                    addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
    427555                    addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
    428                     # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
    429 
    430                     # XXX: do we want to delete these?
     556                    addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id );
     557                    addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
     558
     559                    # XXX: do we want to delete these? trace file is empty
    431560                    # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
    432                     # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
    433                 }
    434             # actual command to delete the files
     561                }
     562                # actual command to delete the files
    435563                $status = &delete_files (\@files);
    436564            }
    437565        }
     566        bzip2_file("LOG.EXP", $path_base, $skycell_id);
     567        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
    438568
    439569        if ($status)  {
     570            my $update_skyfile = 1;
    440571            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
    441572            if ($mode eq "goto_purged") {
    442573                $command .= " -topurgedskyfile";
     574                if ($data_state eq 'purged') {
     575                    $update_skyfile = 0;
     576                }
    443577            }
    444578            elsif ($mode eq "goto_cleaned") {
    445579                $command .= " -tocleanedskyfile";
     580                if ($data_state eq 'cleaned') {
     581                    $update_skyfile = 0;
     582                }
    446583            }
    447584            elsif ($mode eq "goto_scrubbed") {
    448585                $command .= " -toscrubbedskyfile";
    449             }
    450             $command .= " -dbname $dbname" if defined $dbname;
    451 
    452             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    453                     run(command => $command, verbose => $verbose);
     586                if ($data_state eq 'scrubbed') {
     587                    $update_skyfile = 0;
     588                }
     589            }
     590            $command .= " -dbname $dbname" if defined $dbname;
     591
     592            if ($update_skyfile) {
     593                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     594                        run(command => $command, verbose => $verbose);
     595                unless ($success) {
     596                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     597                    &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
     598                }
     599
     600                set_destreak_goto_cleaned();
     601            }
     602
     603         } else {
     604            $num_errors++;
     605            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
     606            $command .= " -dbname $dbname" if defined $dbname;
     607
     608            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     609                run(command => $command, verbose => $verbose);
    454610            unless ($success) {
    455611                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    457613            }
    458614
    459             set_destreak_goto_cleaned();
    460 
    461          } else {
    462             my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    463             $command .= " -dbname $dbname" if defined $dbname;
    464 
    465             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     615            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
     616            $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state";
     617            $command .= " -dbname $dbname" if defined $dbname;
     618
     619            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    466620                run(command => $command, verbose => $verbose);
    467621            unless ($success) {
    … …  
    469623                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
    470624            }
    471 
    472             #            exit $PS_EXIT_UNKNOWN_ERROR;
    473         }
    474     }
     625        }
     626    }
     627    print "Cleanup completed for warp_id $stage_id.";
     628    print " num_errors: $num_errors" if $num_errors;
     629    print "\n";
    475630    exit 0;
    476631}
    … …  
    603758                &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
    604759            }
    605 #           exit $PS_EXIT_UNKNOWN_ERROR;
    606760        }
    607761    }
    … …  
    619773    my $skyfiles;                  # Array reference of component files
    620774    my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
     775    $command .= ' -all' if $check_all;
    621776    $command .= " -dbname $dbname" if defined $dbname;
    622     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     777    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
    623778    unless ($success) {
    624779        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    … …  
    646801        &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
    647802
     803    my $num_errors = 0;
    648804    my @files = ();
    649805    foreach my $skyfile (@{ $skyfiles }) {
    650806        my $path_base = $skyfile->{path_base};
    651807        my $skycell_id = $skyfile->{skycell_id};
     808        my $data_state = $skyfile->{data_state};
    652809
    653810        my $status = 1;
    … …  
    661818                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    662819
     820            if (0) {
    663821                unless ($ipprc->file_exists($config_file)) {
    664822                    if (file_gone($config_file)) {
    … …  
    672830                }
    673831            }
     832                unless ($ipprc->file_exists($config_file)) {
     833                    my $fault = $skyfile->{fault};
     834                    my $quality = $skyfile->{quality};
     835                    if (file_gone($config_file)) {
     836                        print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality"
     837                            . " because config file ($config_file) is gone\n";
     838                    } elsif ($fault == 0 and $quality == 0) {
     839                            print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
     840                                . " because config file ($config_file) is missing\n";
     841                            $status = 0;
     842                    } else {
     843                            # config file is missing but this is a bad diff anyways so clean it
     844                            print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
     845                                . " even though config file ($config_file) is missing\n";
     846                    }
     847                }
     848            }
    674849            elsif ($mode eq "goto_scrubbed") {
    675850                my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
    … …  
    712887
    713888            }
    714 #           print STDERR "MY FILES: @files\n";
    715889            $status = &delete_files(\@files);
    716890        }
    717 #       print STDERR "MY STATUS: $status\n";
     891
     892        bzip2_file("LOG.EXP", $path_base, $skycell_id);
     893        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
     894
    718895        if ($status) {
    719896            my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
     897            my $update_skyfile = 1;
    720898
    721899            if ($mode eq "goto_purged") {
    722900                $command .= " -topurgedskyfile";
     901                if ($data_state eq 'purged') {
     902                    $update_skyfile = 0;
     903                }
    723904            }
    724905            elsif ($mode eq "goto_cleaned") {
    725906                $command .= " -tocleanedskyfile";
     907                if ($data_state eq 'cleaned') {
     908                    $update_skyfile = 0;
     909                }
    726910            }
    727911            elsif ($mode eq "goto_scrubbed") {
    728912                $command .= " -toscrubbedskyfile";
    729             }
    730 
     913                if ($data_state eq 'scrubbed') {
     914                    $update_skyfile = 0;
     915                }
     916            }
     917
     918            $command .= " -dbname $dbname" if defined $dbname;
     919
     920            if ($update_skyfile) {
     921                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     922                    run(command => $command, verbose => $verbose);
     923                unless ($success) {
     924                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     925                    &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
     926                }
     927                set_destreak_goto_cleaned();
     928            }
     929
     930
     931        } else {
     932            $num_errors++;
     933            my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state";
    731934            $command .= " -dbname $dbname" if defined $dbname;
    732935
    … …  
    738941            }
    739942
    740             set_destreak_goto_cleaned();
    741 
    742         } else {
    743             my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
    744 
    745             $command .= " -dbname $dbname" if defined $dbname;
    746 
    747             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     943            $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
     944
     945            $command .= " -dbname $dbname" if defined $dbname;
     946
     947            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    748948                run(command => $command, verbose => $verbose);
    749949            unless ($success) {
    … …  
    751951                &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
    752952            }
    753 #           exit $PS_EXIT_UNKNOWN_ERROR;
    754         }
    755     }
     953        }
     954    }
     955    print "Cleanup completed for diff_id $stage_id.";
     956    print " num_errors: $num_errors" if $num_errors;
     957    print "\n";
    756958    exit 0;
    757959}
    … …  
    19172119
    19182120    foreach my $file (@$files) {
    1919         print STDERR "unlinking $stage $stage_id $file\n";
     2121        print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose;
    19202122
    19212123        my $error_code = $ipprc->kill_file($file);
    … …  
    20082210}
    20092211
    2010 # this gets set to 1 the first time we set the corresponding destreak run to be cleaned
    2011 my $ds_done = 0;
    20122212sub set_destreak_goto_cleaned {
    20132213
    … …  
    20272227}
    20282228
     2229sub bzip2_file {
     2230    my $filerule = shift;
     2231    my $path_base = shift;
     2232    my $component = shift;
     2233
     2234    my $filename = $ipprc->filename($filerule, $path_base, $component);
     2235    if (!$ipprc->file_exists($filename)) {
     2236        return 1;
     2237    }
     2238    if (my $resolved = $ipprc->file_resolve($filename)) {
     2239        my $bzip2_filename = $filename . '.bz2';
     2240        if ($ipprc->file_exists($bzip2_filename)) {
     2241            $ipprc->kill_file($bzip2_filename);
     2242        }
     2243        my $bzip2_file = $ipprc->file_create($bzip2_filename);
     2244        my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
     2245
     2246        my $command = "$bzip2 < $resolved > $bzip2_file";
     2247        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     2248            run(command => $command, verbose => $very_verbose);
     2249        if ($success) {
     2250            # success delete the original file
     2251            my $error_code = $ipprc->kill_file($filename);
     2252        } else {
     2253            # if bzip2 failed. Carry on but don't delete the existing file
     2254            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     2255            print STDERR "Failed to bzip2 $filename: $error_code\n";
     2256            return 0;
     2257        }
     2258    }
     2259    return 1;
     2260}
     2261
    20292262# XXX we currently do not set the error state in the db on my_die
    20302263sub my_die

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/magic_destreak.pl

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl

    r34738 r34839  
    134134# First check the expected file name where the stack_id is the FILE_ID
    135135my $file;
    136 if (!$singlefilter) {
     136# if (!$singlefilter) {
     137if (1) {
     138
    137139    $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id);
    138140    if (! $ipprc->file_resolve($file)) {
    139         # XXX: Beginning of section that can be removed eventually
    140 
    141         # no file with the expected name found
    142         # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID
    143         # but instead used FILE_ID = [0 .. num_filters-1]
    144141        print "\nfailed to resolve $file\n";
    145         if (!$filter) {
    146             &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
    147         }
    148         print "Trying old style FILE_ID\n";
    149         my $max_filters = 5;
    150         for (my $i=0; $i < $max_filters; $i++) {
    151             my $file_id = sprintf "%03d", $i;
    152             $file  = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
    153             my $resolved = $ipprc->file_resolve($file);
    154             if (!$resolved) {
    155                 # we fail here assumming that if file_id N doesn't exist, neither to N+1
    156                 print "\nfailed to resolve $file\n\n";
    157                 &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
     142        # file with proper file rule not found. Try older systems
     143        if ($singlefilter) {
     144            # input is from a single filter static sky run which used psphot instead of psphotStack
     145            # The file rule is different for psphot
     146            $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
     147            &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR)
     148                unless $ipprc->file_exists($file);
     149        } else {
     150            # XXX: Beginning of section that can be removed eventually
     151            # no file with the expected name found
     152            # assume that the input is from an early multifilter staticsky run that did not use stack_id as the FILE_ID
     153            # but instead used FILE_ID = [0 .. num_filters-1]
     154            if (!$filter) {
     155                &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
    158156            }
    159             # Check the filter id for this file
    160             my $command = "$fhead $resolved | grep FILTERID";
    161             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    162                 run(command => $command, verbose => 0);
    163             unless ($success) {
    164                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    165                 &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
     157            print "Trying old style FILE_ID\n";
     158            my $max_filters = 5;
     159            for (my $i=0; $i < $max_filters; $i++) {
     160                my $file_id = sprintf "%03d", $i;
     161                $file  = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
     162                my $resolved = $ipprc->file_resolve($file);
     163                if (!$resolved) {
     164                    # we fail here assumming that if file_id N doesn't exist, neither to N+1
     165                    print "\nfailed to resolve $file\n\n";
     166                    &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
     167                }
     168                # Check the filter id for this file
     169                my $command = "$fhead $resolved | grep FILTERID";
     170                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     171                    run(command => $command, verbose => 0);
     172                unless ($success) {
     173                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     174                    &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
     175                }
     176                # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
     177                my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
     178                my $this_filter = substr $filt, 1, 7;
     179
     180                # if it matches we're done
     181                last if $this_filter eq $filter;
     182
     183                # nope loop around to try the next one
     184                print "Input file for $filter is not $file ($this_filter)\n";
     185                $file = undef;
    166186            }
    167             # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
    168             my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
    169             my $this_filter = substr $filt, 1, 7;
    170 
    171             # if it matches we're done
    172             last if $this_filter eq $filter;
    173 
    174             # nope loop around to try the next one
    175             print "Input file for $filter is not $file ($this_filter)\n";
    176             $file = undef;
    177         }
     187        }
     188        # XXX: End of section that can be removed eventually
     189    }
     190    if ($file) {
    178191        print "\nInput file for $stack_id filter: $filter: $file\n";
    179 
    180         # XXX: End of section that can be removed eventually
    181     }
    182 } else {
    183     # input is from a single filter static sky run use a different file rule
    184     # XXX: can't we just make staticsky.pl use the same file rule?
    185     $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
    186     &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR)
    187         unless $ipprc->file_exists($file);
    188 }
    189 
    190 if (!$file) {
    191     &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR);
     192    } else {
     193        &my_die("Unable to find input file for stack_id $stack_id.", $skycal_id, $PS_EXIT_SYS_ERROR);
     194    }
    192195}
    193196
    … …  
    197200}
    198201
    199 # read the input header to find the number of detections and the number of detections
    200 # with extended model
     202# XXX: The following is for compatability with psphotStack cmfs created prior to adding NDET
     203# read the input header to find the number of detections
     204my $compatability_flags = "";
    201205my $n_detections = 0;
    202 my $n_extended = 0;
    203206{
     207    # NSTARS is buggy. It only counts sources with a model, and counts them twice.
     208    # We want the actual number of sources (real + matched). Use the length of the psf table
    204209    my $command = "echo $resolved | $fields -n SkyChip.psf NAXIS2";
    205210    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    211216        unless defined $n_detections;
    212217
    213     $command = "echo $resolved | $fields -n SkyChip.hdr NDET_EXT";
    214     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    215             run(command => $command, verbose => 0);
    216     (undef, $n_extended)  = split " ", join "", @$stdout_buf;
    217     chomp $n_extended;
    218     &my_die("Unable to find number of extended objects from $file: ", $skycal_id, $PS_EXIT_SYS_ERROR)
    219         unless defined $n_extended;
     218    $compatability_flags .= " -n_detections $n_detections";
     219
    220220}
    221221
    … …  
    257257        }
    258258
     259        # if ppStats didn't get n_detections use the compatability version extracted above
     260        $cmdflags .= $compatability_flags unless ($cmdflags =~ /-n_detections/);
     261
    259262        my $quality;
    260263        ($quality) =  $cmdflags =~ /-quality (\d+)/;
    … …  
    276279{
    277280    my $command = "$staticskytool -skycal_id $skycal_id";
    278     $command .= " -n_detections $n_detections";
    279     $command .= " -n_extended $n_extended";
    280281    $command .= " -addskycalresult -path_base $outroot";
    281282    $command .= " $cmdflags";

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl

    r28403 r34839  
    268268
    269269    my %tangents = ();
     270
     271    my %products = ('image' => "PPSTACK.UNCONV",
     272                    'mask'  => "PPSTACK.UNCONV.MASK",
     273                    'variance' => "PPSTACK.UNCONV.VARIANCE",
     274                    'exp'   => "PPSTACK.UNCONV.EXP",
     275                    'num'   => "PPSTACK.UNCONV.EXPNUM",
     276                    'bkg'   => "PPSTACK.OUTPUT.BKGMODEL"
     277        );
    270278   
    271279    foreach my $imfile (@$imfiles) {
    … …  
    281289
    282290        $projection_cell =~ s/^(.*)\..*$/$1/;
    283        
     291
    284292        unless (exists($tangents{$projection_cell})) {
    285293            # Make a temp file and fill, but be sure to save
    286             ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX",
    287                                                  UNLINK => !$save_temps);
    288             $tangents{$projection_cell}{FILE} = $tempFile;
    289             $tangents{$projection_cell}{NAME} = $tempName;
    290             if ($masks) {
    291                 my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX",
    292                                                      UNLINK => !$save_temps);
    293                 $tangents{$projection_cell}{MFILE} = $maskFile;
    294                 $tangents{$projection_cell}{MNAME} = $maskName;
    295             }           
    296         }
    297         print "$skycell_id $projection_cell\n";
    298         my $file = $ipprc->filename("PPSTACK.OUTPUT", $path_base, $skycell_id);
    299         print "$file $state $quality\n";
    300         my $f_fh = $tangents{$projection_cell}{FILE};
    301         print $f_fh "$file\n";
    302         if ($masks) {
    303             my $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
    304             print "$mask\n";
    305             my $m_fh = $tangents{$projection_cell}{MFILE};
    306             print $m_fh "$mask\n";
     294            foreach my $key (keys %products) {
     295                ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.$key.XXXX",
     296                                                  UNLINK => !$save_temps);
     297                $tangents{$projection_cell}{$key}{FILE} = $tempFile;
     298                $tangents{$projection_cell}{$key}{NAME} = $tempName;
     299            }
     300        }
     301        foreach my $key (keys %products) {
     302            print "$skycell_id $projection_cell\n";     
     303            my $file = $ipprc->filename($products{$key}, $path_base, $skycell_id);
     304            print "$file $state $quality\n";
     305            my $f_fh = $tangents{$projection_cell}{$key}{FILE};
     306            print $f_fh "$file\n";
    307307        }
    308308    }
    309309    foreach my $projection_cell (keys %tangents) {
    310         $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}";
    311         if ($masks) {
    312             $command .= " -masks $tangents{$projection_cell}{MNAME} ";
    313         }
    314         $command .= " ${outroot}.${projection_cell} ";
    315         print "$command\n";
    316         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
    317         unless ($success) {
    318             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    319             &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
    320         }
     310        ## Loop over results here.
     311        # Images
     312        # Masks
     313        # Variances
     314        # Nexptime
     315        # Nexp
     316        # Backgrounds
     317       
     318        foreach my $key (keys %products) {
     319            $command = "$ppSkycell -images $tangents{$projection_cell}{$key}{NAME}";
     320            $command .= " ${outroot}.${projection_cell}.${key} ";
     321            if ($key eq 'bkg') {
     322                $command .= " -Di BIN1 1 -Di BIN2 1 ";
     323            }
     324            elsif ($key eq 'image') {
     325                $command .= " -masks $tangents{$projection_cell}{mask}{NAME} ";
     326            }
     327            elsif ($key eq 'mask') {
     328                next; # This should be made with the images.
     329            }
     330            print "$command\n";
     331            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     332            unless ($success) {
     333                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     334                &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
     335            }
     336        }
     337
    321338        # Update database:
    322339        $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot";

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl

    r32562 r34839  
    229229    my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name
    230230
     231    my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base});
     232
    231233    &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image );
    232234    &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask );
    … …  
    240242    print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve;
    241243    print $listFile "\tSOURCES\tSTR\t" . $sources . "\n";
     244    print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel );
    242245
    243246    print $listFile "END\n\n";

  • branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl

    r33053 r34839  
    128128# Where do we get the astrometry source from?
    129129my $astromSource;               # The astrometry source
     130my $doBackground;               # Do we want to make background models?
    130131{
    131132    my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -";
    … …  
    139140        &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
    140141    $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
     142    $doBackground = metadataLookupBool($metadata, 'BACKGROUND.MODEL');   
    141143}
    142144
    … …  
    168170if ($do_stats) {
    169171    $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats;
     172}
     173my $outputBKGs;
     174if ($doBackground) {
     175    $outputBKGs = prepare_output ("PSWARP.OUTPUT.BKGMODEL", $outroot, $skycell_id, 1);
    170176}
    171177my $configuration;
    … …  
    209215my ($weightFile, $weightName) = tempfile( "$tempOutRoot.weight.list.XXXX", UNLINK => !$save_temps);
    210216my ($astromFile, $astromName) = tempfile( "$tempOutRoot.astrom.list.XXXX", UNLINK => !$save_temps);
    211 
     217my ($bkgFile, $bkgName);
     218if ($doBackground) {
     219    ($bkgFile, $bkgName) = tempfile( "$tempOutRoot.bkg.list.XXXX", UNLINK => !$save_temps);
     220}
    212221my $wrote_astrom = 0;
    213222foreach my $imfile (@$imfiles) {
    … …  
    235244    print $maskFile   "$mask\n";
    236245    print $weightFile "$weight\n";
     246    my $bkg;
     247    if ($doBackground) {
     248        $bkg    = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
     249        print $bkgFile "$bkg\n";
     250    }
    237251
    238252    if (!$wrote_astrom) {
    … …  
    245259close $weightFile;
    246260close $astromFile;
    247 
     261if ($doBackground) {
     262    close($bkgFile);
     263}
    248264# We need the recipe to determine if we care whether the PSF is generated or not
    249265my $recipe;
    … …  
    269285    $command .= " -variancelist $weightName";
    270286    $command .= " -astromlist $astromName";
     287    $command .= " -bkglist $bkgName" if ($doBackground);
    271288    $command .= " $outroot $skyFile";
    272289    $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";

  • branches/eam_branches/ipp-20121130/ippTasks

  • branches/eam_branches/ipp-20121130/ippTasks/Makefile.am

    r33030 r34839  
    4545        diffphot.pro \
    4646        lap.pro \
    47         vp.pro
     47        vp.pro \
     48        bg.regeneration.pro
    4849
    4950other_files = \

  • branches/eam_branches/ipp-20121130/ippTasks/addstar.pro

    r34265 r34839  
    212212  host         local
    213213  periods      -poll $LOADPOLL
    214   periods      -exec 300
     214  periods      -exec $LOADEXEC_ADD
    215215  periods      -timeout 300
    216216  npending     1
    … …  
    266266  host         local
    267267  periods      -poll $LOADPOLL
    268   periods      -exec 300
     268  periods      -exec $LOADEXEC_ADD
    269269  periods      -timeout 300
    270270  npending     1

  • branches/eam_branches/ipp-20121130/ippTasks/nightly_science.pro

    r32375 r34839  
    258258task              ns.dqstats.load
    259259  host            local
    260   periods         -poll 3600
     260  periods         -poll $LOADPOLL
    261261  periods         -exec $LOADEXEC
    262262  periods         -timeout 300

  • branches/eam_branches/ipp-20121130/ippTasks/science.cleanup.pro

    r34203 r34839  
    1313
    1414check.globals
     15
     16# if set tell cleanup script to check for files from all components regardless of the data_state
     17$CHECK_ALL_COMPONENTS = 0
     18macro set.check.all.components
     19    $CHECK_ALL_COMPONENTS = 1
     20end
     21macro clear.check.all.components
     22    $CHECK_ALL_COMPONENTS = 0
     23end
     24macro get.check.all.components
     25    echo CHECK_ALL_COMPONENTS = $CHECK_ALL_COMPONENTS
     26end
     27
    1528
    1629## chip.cleanup.pro : -*- sh -*-
    … …  
    130143    book getword chipPendingCleanup $pageName state -var CLEANUP_MODE
    131144    book getword chipPendingCleanup $pageName chip_id -var CHIP_ID
     145    book getword chipPendingCleanup $pageName exp_tag -var EXP_TAG
     146    book getword chipPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
    132147    book getword chipPendingCleanup $pageName dbname -var DBNAME
    133148
    134     # specify choice of local or remote host based on camera and chip (class_id)
     149    # specify choice of local or remote host based on camera and chip (in this case FPA)
    135150    set.host.for.camera $CAMERA FPA
     151
     152    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     153
     154    sprintf LOGFILE "%s/%s/%s.ch.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_ID
    136155
    137156    stdout $LOGDIR/chip.cleanup.log
    138157    stderr $LOGDIR/chip.cleanup.log
    139158
    140     # XXX is everything listed here needed?
    141     $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE
     159    $run = ipp_cleanup.pl --stage chip --stage_id $CHIP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     160    if ($CHECK_ALL_COMPONENTS)
     161        $run = $run --check-all
     162    end
    142163    add_standard_args run
    143164
    … …  
    282303    book getword camPendingCleanup $pageName state  -var CLEANUP_MODE
    283304    book getword camPendingCleanup $pageName cam_id -var CAM_ID
     305    book getword camPendingCleanup $pageName exp_tag -var EXP_TAG
     306    book getword camPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
    284307    book getword camPendingCleanup $pageName dbname -var DBNAME
    285308
    … …  
    287310    set.host.for.camera $CAMERA FPA
    288311
     312    # set the WORKDIR variable
     313    set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR
     314
     315    ## generate LOGFILE specific to this exposure (& cam_id)
     316    sprintf LOGFILE "%s/%s/%s.cm.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID
     317
    289318    stdout $LOGDIR/camera.cleanup.log
    290319    stderr $LOGDIR/camera.cleanup.log
    291320
    292     # XXX is everything listed here needed?
    293     $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE
     321    $run = ipp_cleanup.pl --stage camera --stage_id $CAM_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
    294322    add_standard_args run
    295323
    … …  
    591619    book getword warpPendingCleanup $pageName state -var CLEANUP_MODE
    592620    book getword warpPendingCleanup $pageName warp_id -var WARP_ID
     621    book getword warpPendingCleanup $pageName exp_tag -var EXP_TAG
     622    book getword warpPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE
    593623    book getword warpPendingCleanup $pageName dbname -var DBNAME
    594624
    595625    # specify choice of local or remote host based on camera and warp (class_id)
    596626    set.host.for.camera $CAMERA FPA
     627    set.workdir.by.camera $CAMERA $WARP_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     628
     629    sprintf LOGFILE "%s/%s/%s.wrp.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_ID
    597630
    598631    stdout $LOGDIR/warp.cleanup.log
    599632    stderr $LOGDIR/warp.cleanup.log
    600633
    601     # XXX is everything listed here needed?
    602     $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE
     634    $run = ipp_cleanup.pl --stage warp --stage_id $WARP_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     635    if ($CHECK_ALL_COMPONENTS)
     636        $run = $run --check-all
     637    end
    603638    add_standard_args run
    604639
    … …  
    748783    book getword diffCleanup $pageName state -var CLEANUP_MODE
    749784    book getword diffCleanup $pageName diff_id -var DIFF_ID
     785    book getword diffCleanup $pageName tess_id -var TESS_ID
     786    book getword diffCleanup $pageName workdir -var WORKDIR_TEMPLATE
    750787    book getword diffCleanup $pageName dbname -var DBNAME
    751788
    752789    # specify choice of local or remote host based on camera and diff (class_id)
    753790    set.host.for.camera $CAMERA FPA
     791    set.workdir.by.camera $CAMERA $DIFF_ID $WORKDIR_TEMPLATE $default_host WORKDIR
     792    sprintf LOGFILE "%s/%s/%s.dif.%s.cleanup.log" $WORKDIR $TESS_ID $TESS_ID $DIFF_ID
     793
    754794
    755795    stdout $LOGDIR/diff.cleanup.log
    756796    stderr $LOGDIR/diff.cleanup.log
    757797
    758     # XXX is everything listed here needed?
    759     $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE
     798    $run = ipp_cleanup.pl --stage diff --stage_id $DIFF_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
     799    if ($CHECK_ALL_COMPONENTS)
     800        $run = $run --check-all
     801    end
    760802    add_standard_args run
    761803
    … …  
    899941    book getword stackCleanup $pageName state -var CLEANUP_MODE
    900942    book getword stackCleanup $pageName stack_id -var STACK_ID
     943    book getword stackCleanup $pageName path_base -var PATH_BASE
    901944    book getword stackCleanup $pageName dbname -var DBNAME
    902945
    … …  
    907950    stderr $LOGDIR/stack.cleanup.log
    908951
    909     # XXX is everything listed here needed?
    910     $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE
     952    sprintf LOGFILE "%s.cleanup.log" PATH_BASE
     953    $run = ipp_cleanup.pl --stage stack --stage_id $STACK_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $LOGFILE
    911954    add_standard_args run
    912955

  • branches/eam_branches/ipp-20121130/ippTasks/skycal.pro

    r32983 r34839  
    88# test for required global variables
    99check.globals
     10
     11# skcal.pro should have a more restricted polling limit (to avoid stress with getstar)
     12if ($?POLL_LIMIT_SKYCAL == 0) set POLL_LIMIT_SKYCAL = 25
     13
     14macro set.skycal.poll
     15  if ($0 != 2)
     16    echo "USAGE:set.skycal.poll (value)"
     17    break
     18  end
     19 
     20  $POLL_LIMIT_SKYCAL = $1
     21end
     22
     23macro get.skycal.poll
     24  echo "skycal poll limit: $POLL_LIMIT_SKYCAL"
     25end
    1026
    1127### Initialise the books containing the tasks to do
    … …  
    7995      # save the DB name for the exit tasks
    8096      option $DB:$skycal_DB
    81       $run = $run -dbname $DB:$skycal_DB -limit 40
     97      $run = $run -dbname $DB:$skycal_DB
    8298      $skycal_DB ++
    8399      if ($skycal_DB >= $DB:n) set skycal_DB = 0
    … …  
    85101    add_poll_args run
    86102    add_poll_labels run
     103    $run = $run -limit $POLL_LIMIT_SKYCAL
    87104    command $run
    88105  end
    … …  
    123140
    124141  task.exec
     142    # if we are unable to run use "long" exectime
     143    periods -exec $RUNEXEC
    125144    book npages skycalRun -var N
    126145    if ($N == 0) break
    127146    if ($NETWORK == 0) break
    128147    if ($BURNTOOLING == 1) break
     148
    129149
    130150    # look for new entries in skycalRun (pantaskState == INIT)
    … …  
    147167
    148168    # set the host and workdir based on the skycell hash
    149     set.host.for.skycell $SKYCELL_ID
     169    # set.host.for.skycell $SKYCELL_ID
     170    host anyhost
    150171    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    151172
    … …  
    172193      echo command $run
    173194    end
     195    # since we have work to do shorten exec time
     196    periods -exec .1
    174197    command $run
    175198  end
    … …  
    197220  host         local
    198221
    199   periods      -poll 60.0
    200   periods      -exec 1800.0
     222  periods      -poll 10.0
     223  periods      -exec 1200.0
    201224  periods      -timeout 120.0
    202225  npending     1
    203   active false
     226  active true
    204227 
    205228  stdout NULL

  • branches/eam_branches/ipp-20121130/ippTasks/staticsky.pro

    r33957 r34839  
    7070
    7171  periods      -poll $LOADPOLL
    72   periods      -exec $LOADEXEC
    73   periods      -timeout 30
     72  periods      -exec 30
     73  periods      -timeout 60
    7474  npending     1
    7575
    … …  
    154154
    155155    # set the host and workdir based on the skycell hash
    156     set.host.for.skycell $SKYCELL_ID
     156    # set.host.for.skycell $SKYCELL_ID
     157    host anyhost
    157158    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
    158159

  • branches/eam_branches/ipp-20121130/ippTasks/summit.copy.pro

    r33168 r34839  
    223223    periods      -exec     10
    224224    periods      -poll     1
    225     periods      -timeout  20
     225    periods      -timeout  60
    226226    npending     1
    227227

  • branches/eam_branches/ipp-20121130/ippTasks/survey.pro

    r34350 r34839  
    1717 book create SURVEY_PUBLISH
    1818 book create SURVEY_STATICSKYSINGLE
     19 book create SURVEY_SKYCAL
    1920 $haveSurveyBooks = TRUE
    2021end
    … …  
    3132$SURVEY_WARP_BG_DB = 0
    3233$SURVEY_PUBLISH_DB = 0
     34$SURVEY_SKYCAL_DB = 0
    3335$SURVEY_STATICSKYSINGLE_DB = 0
    3436$SURVEY_EXEC = 120
    … …  
    7375    active true
    7476  end 
     77  task survey.skycal
     78    active true
     79  end
    7580end
    7681
    … …  
    110115  end
    111116  task survey.staticskysingle
     117    active false
     118  end
     119  task survey.skycal
    112120    active false
    113121  end
    … …  
    464472  book delpage SURVEY_STATICSKYSINGLE $1
    465473end
     474
     475macro survey.add.skycal
     476  if ($0 != 3)
     477    echo "USAGE: survey.add.skycal (label) (dist_group)"
     478    break
     479  end
     480  book newpage SURVEY_SKYCAL $1
     481  book setword SURVEY_SKYCAL $1 LABEL $1
     482  book setword SURVEY_SKYCAL $1 DIST_GROUP $2
     483  book setword SURVEY_SKYCAL $1 STATE PENDING
     484end
     485
     486macro survey.del.skycal
     487  if ($0 != 2)
     488    echo "USAGE: survey.del.skycal (label)"
     489    break
     490  end
     491  book delpage SURVEY_SKYCAL $1
     492end
     493
     494macro survey.show.skycal
     495  if ($0 != 1)
     496    echo "USAGE: survey.show.skyacl"
     497    break
     498  end
     499  book listbook SURVEY_SKYCAL
     500end
     501
    466502
    467503task survey.diff
    … …  
    14531489end
    14541490
     1491# Survey task for Skycal stage
     1492
     1493task survey.skycal
     1494  host local
     1495 
     1496  periods      -poll $SURVEY_POLL
     1497  periods      -exec $SURVEY_EXEC
     1498  periods      -timeout $SURVEY_TIMEOUT
     1499  npending     1
     1500
     1501  stdout $LOGDIR/survey.skycal.log
     1502  stderr $LOGDIR/survey.skycal.log
     1503
     1504  task.exec
     1505    book npages SURVEY_SKYCAL -var N
     1506    if ($N == 0)
     1507#      echo "No labels for processing"
     1508      break
     1509    endif
     1510
     1511    book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW
     1512    if ("$label" == "NULL")
     1513      # All labels have been done --- reset
     1514#      echo "Resetting labels"
     1515      for i 0 $N
     1516        book getpage SURVEY_SKYCAL $i -var label
     1517        book setword SURVEY_SKYCAL $label STATE NEW
     1518      end
     1519      book getpage SURVEY_SKYCAL 0 -var label -key STATE NEW
     1520
     1521      # Select different database
     1522      $SURVEY_SKYCAL_DB ++
     1523      if ($SURVEY_SKYCAL_DB >= $DB:n) set SURVEY_SKYCAL_DB = 0
     1524    end
     1525
     1526    book setword SURVEY_SKYCAL $label STATE DONE
     1527    book getword SURVEY_SKYCAL $label DIST_GROUP -var dist_group
     1528
     1529    # note: currently skycal uses the staticskyRun workdir
     1530
     1531    $run = staticskytool -defineskycalrun -select_label $label -set_dist_group $dist_group
     1532
     1533    if ($DB:n == 0)
     1534      option DEFAULT
     1535    else
     1536      $run = $run -dbname $DB:$SURVEY_SKYCAL_DB
     1537      option $DB:$SURVEY_SKYCAL_DB
     1538    end
     1539   
     1540    # echo $run
     1541    command $run
     1542  end
     1543
     1544  # success
     1545  task.exit    0
     1546#    echo "Success"
     1547  end
     1548
     1549  # locked list
     1550  task.exit    default
     1551    showcommand failure
     1552  end
     1553
     1554  task.exit    crash
     1555    showcommand crash
     1556  end
     1557
     1558  # operation times out?
     1559  task.exit    timeout
     1560    showcommand timeout
     1561  end
     1562end

  • branches/eam_branches/ipp-20121130/ippToPsps

  • branches/eam_branches/ipp-20121130/ippToPsps/config/OB/tables.vot

    r34615 r34839  
    2727        <DESCRIPTION>flag indicating odd properties</DESCRIPTION>
    2828      </FIELD>
    29       <FIELD name="varFlag" arraysize="1" datatype="unsignedByte" unit="unknown">
    30         <DESCRIPTION>variable in difference image detections</DESCRIPTION>
     29      <FIELD name="qualityFlag" arraysize="1" datatype="unsignedByte" unit="unknown">
     30        <DESCRIPTION>subset of objInfoFlag - flags related to goodness of object</DESCRIPTION>
    3131      </FIELD>
    3232      <FIELD name="surveyID" arraysize="1" datatype="unsignedByte" unit="unknown">
    … …  
    4545        <DESCRIPTION> Declination error</DESCRIPTION>
    4646      </FIELD>
     47      <FIELD name="cx" arraysize="1" datatype="double" unit="unknown">
     48        <DESCRIPTION>Cartesian x</DESCRIPTION>
     49      </FIELD>
     50      <FIELD name="cy" arraysize="1" datatype="double" unit="unknown">
     51        <DESCRIPTION>Cartesian y</DESCRIPTION>
     52      </FIELD>
     53      <FIELD name="cz" arraysize="1" datatype="double" unit="unknown">
     54        <DESCRIPTION>Cartesian z</DESCRIPTION>
     55      </FIELD>
    4756      <FIELD name="lambda" arraysize="1" datatype="double" unit="degree">
    4857        <DESCRIPTION> ecliptic longitude</DESCRIPTION>
    … …  
    284293      <FIELD name="consistencyFlag" arraysize="1" datatype="short" unit="unknown">
    285294        <DESCRIPTION>Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation</DESCRIPTION>
     295      </FIELD>
     296      <FIELD name="Random" arraysize="1" datatype="double" unit="unknown">
     297        <DESCRIPTION>Random number between 0 and 1</DESCRIPTION>
    286298      </FIELD>
    287299      <FIELD name="dataRelease" arraysize="1" datatype="unsignedByte" unit="unknown">

  • branches/eam_branches/ipp-20121130/ippToPsps/jython/objectbatch.py

    r34723 r34839  
    134134                   ," + filter[1] + "80pct = MAG_80/1000 \
    135135                   ," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \
    136                    ," + filter[1] + "StackDetectID = STACK_DETECT_ID + " + str(filterID)
     136                   ," + filter[1] + "StackDetectID = STACK_DETECT_ID"
    137137
    138138            #self.logger.info(sql)
    … …  
    236236               ,ippObjID \
    237237               ,objInfoFlag \
    238                ,varFlag \
     238               ,qualityFlag \
    239239               ,surveyID \
    240240               ,ra \
    … …  
    246246               ,sgSep \
    247247               ,dataRelease \
     248               ,Random \
    248249               ) \
    249250               SELECT \
    … …  
    251252               ,CAT_ID*1000000000 + OBJ_ID \
    252253               ,FLAGS \
    253                ,0 \
     254               ,FLAGS >> 24 & 0xFF \
    254255               ," + str(self.surveyID) + " \
    255256               ,RA \
    … …  
    257258               ,RA_ERR \
    258259               ,DEC_ERR \
    259                ,NMEASURE \
     260               ,0 \
    260261               ,PSF_QF_PERF \
    261262               ,STARGAL_SEP \
    262263               , " + str(self.config.dataRelease) + "\
     264               , RAND() \
    263265               FROM " + cptTableName
    264266

  • branches/eam_branches/ipp-20121130/ippToPsps/jython/stackbatch.py

    r34661 r34839  
    55
    66import stilts
     7import re
    78from java.lang import *
    89from java.sql import *
    … …  
    143144    '''
    144145    def updateApFlxs(self, prefix, psfCondition):
     146        #properly -999 these to start with:
    145147
    146148        sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \
    147         "+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \
    148         "+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \
    149         "+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \
    150         "+prefix+"flxR1Fill=b.APER_FILL_1, \
    151         "+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \
    152         "+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \
    153         "+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \
    154         "+prefix+"flxR2Fill=b.APER_FILL_2, \
    155         "+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \
    156         "+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \
    157         "+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \
    158         "+prefix+"flxR3Fill=b.APER_FILL_3, \
    159         "+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \
    160         "+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \
    161         "+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \
    162         "+prefix+"flxR4Fill=b.APER_FILL_4, \
    163         "+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \
    164         "+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \
    165         "+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \
    166         "+prefix+"flxR5Fill=b.APER_FILL_5, \
    167         "+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \
    168         "+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \
    169         "+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \
    170         "+prefix+"flxR6Fill=b.APER_FILL_6, \
    171         "+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \
    172         "+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \
    173         "+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \
    174         "+prefix+"flxR7Fill=b.APER_FILL_7, \
    175         "+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \
    176         "+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \
    177         "+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \
    178         "+prefix+"flxR8Fill=b.APER_FILL_8, \
    179         "+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \
    180         "+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \
    181         "+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \
    182         "+prefix+"flxR9Fill=b.APER_FILL_9, \
    183         "+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \
    184         "+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \
    185         "+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \
    186         "+prefix+"flxR10Fill=b.APER_FILL_10 \
    187         WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
    188 
    189         self.scratchDb.execute(sql)
     149        "+prefix+"flxR1=-999, \
     150        "+prefix+"flxR1Err=-999, \
     151        "+prefix+"flxR1Std=-999, \
     152        "+prefix+"flxR1Fill=-999, \
     153        "+prefix+"flxR2=-999, \
     154        "+prefix+"flxR2Err=-999, \
     155        "+prefix+"flxR2Std=-999, \
     156        "+prefix+"flxR2Fill=-999, \
     157        "+prefix+"flxR3=-999, \
     158        "+prefix+"flxR3Err=-999, \
     159        "+prefix+"flxR3Std=-999, \
     160        "+prefix+"flxR3Fill=-999, \
     161        "+prefix+"flxR4=-999, \
     162        "+prefix+"flxR4Err=-999, \
     163        "+prefix+"flxR4Std=-999, \
     164        "+prefix+"flxR4Fill=-999, \
     165        "+prefix+"flxR5=-999, \
     166        "+prefix+"flxR5Err=-999, \
     167        "+prefix+"flxR5Std=-999, \
     168        "+prefix+"flxR5Fill=-999, \
     169        "+prefix+"flxR6=-999, \
     170        "+prefix+"flxR6Err=-999, \
     171        "+prefix+"flxR6Std=-999, \
     172        "+prefix+"flxR6Fill=-999, \
     173        "+prefix+"flxR7=-999, \
     174        "+prefix+"flxR7Err=-999, \
     175        "+prefix+"flxR7Std=-999, \
     176        "+prefix+"flxR7Fill=-999, \
     177        "+prefix+"flxR8=-999, \
     178        "+prefix+"flxR8Err=-999, \
     179        "+prefix+"flxR8Std=-999, \
     180        "+prefix+"flxR8Fill=-999, \
     181        "+prefix+"flxR9=-999, \
     182        "+prefix+"flxR9Err=-999, \
     183        "+prefix+"flxR9Std=-999, \
     184        "+prefix+"flxR9Fill=-999, \
     185        "+prefix+"flxR10=-999, \
     186        "+prefix+"flxR10Err=-999, \
     187        "+prefix+"flxR10Std=-999, \
     188        "+prefix+"flxR10Fill=-999 \
     189        WHERE a.ippDetectID=b.IPP_IDET "
     190        #self.logger.infoPair("sql", sql)
     191        self.scratchDb.execute(sql)
     192
     193
     194        #we have variable numbers of these columns. find out which are in use
     195        sql = "DESCRIBE SkyChip_xrad 'aper_fill_%'"
     196        rs = self.scratchDb.executeQuery(sql)
     197        rs.first()
     198        columns = []
     199        column = rs.getString(1)
     200        columns.append(column)
     201        while (rs.next()):
     202            column = rs.getString(1)
     203            columns.append(column)
     204        rs.close()
     205
     206        #use regex to leave just the numbers
     207
     208        numbers = []
     209        for column in columns:
     210            number = re.sub("APER_FILL_","",column)
     211            if (int(number) >= 1 and int(number) <=10):
     212                numbers.append(number)
     213                self.logger.infoPair("Aperture Numbers",number)
     214
     215        #generate the sql to do the necessary ops on the columns   
     216        sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET "
     217        cnt =0
     218        for number in numbers:
     219            if (cnt>0):
     220                sql=sql+" , "
     221            sql=sql+prefix+"flxR"+number+"=b.APER_FLUX_"+number +" / " + str(self.expTime) + ", "
     222            sql=sql+prefix+"flxR"+number+"Err=b.APER_FLUX_ERR_"+number +" / " + str(self.expTime) + ", "
     223            sql=sql+prefix+"flxR"+number+"Std=b.APER_FLUX_STDEV_"+number +" / " + str(self.expTime) + ", "
     224            sql=sql+prefix+"flxR"+number+"Fill=b.APER_FILL_"+number + " "
     225            cnt=cnt+1
     226           
     227        sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition     
     228        self.scratchDb.execute(sql)
     229
     230        #self.logger.infoPair("sql",sql)
     231
     232        #sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \
     233        #"+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \
     234        #"+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \
     235        #"+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \
     236        #"+prefix+"flxR1Fill=b.APER_FILL_1, \
     237        #"+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \
     238        ##"+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \
     239        #"+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \
     240        #"+prefix+"flxR2Fill=b.APER_FILL_2, \
     241        #"+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \
     242        #"+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \
     243        #"+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \
     244        #"+prefix+"flxR3Fill=b.APER_FILL_3, \
     245        #"+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \
     246        #"+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \
     247        #"+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \
     248        #"+prefix+"flxR4Fill=b.APER_FILL_4, \
     249        #"+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \
     250        #"+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \
     251        #"+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \
     252        #"+prefix+"flxR5Fill=b.APER_FILL_5, \
     253        #"+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \
     254        #"+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \
     255        #"+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \
     256        #"+prefix+"flxR6Fill=b.APER_FILL_6, \
     257        #"+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \
     258        #"+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \
     259        #"+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \
     260        #"+prefix+"flxR7Fill=b.APER_FILL_7, \
     261        #"+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \
     262        #"+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \
     263        #"+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \
     264        #"+prefix+"flxR8Fill=b.APER_FILL_8, \
     265        #"+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \
     266        #"+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \
     267        #"+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \
     268        #"+prefix+"flxR9Fill=b.APER_FILL_9, \
     269        #"+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \
     270        #"+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \
     271        #"+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \
     272        #"+prefix+"flxR10Fill=b.APER_FILL_10 \
     273        #WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition
     274
     275        #self.scratchDb.execute(sql)
    190276
    191277    '''
    … …  
    395481               ,KRON_FLUX_ERR / " + str(self.expTime) + " \
    396482               ,MOMENTS_R1 * 2.5 \
    397                ,PSF_CHISQ \
     483               ,PSF_CHISQ / PSF_NDOF \
    398484               ,N_FRAMES \
    399485               , '" + self.dateStr + "' \

  • branches/eam_branches/ipp-20121130/ippToPsps/src/Dvo.c

    r34629 r34839  
    2121    if (this->mysql) mysql_close(this->mysql);
    2222
    23     free(this);
     23    // libohana redirects free and gets confused by thigns allocated by calloc
     24    // so call the real free funtion through this warpper
     25    real_free(this);
    2426}
    2527
    … …  
    9496                PhotCode *code = GetPhotcodebyCode(meas->photcode);
    9597                sprintf(sql,
    96                         "INSERT INTO dvoDetection (imageID, ippDetectID, detectID, ippObjID, objID, flags, zp, zpErr, airMass, expTime, ra, dec_, raErr, decErr) VALUES (%d, %u, %lu, %lu, %lu, %u, %f, %f, %f, %f, %lf, %lf, %f %f)",
     98                        "INSERT INTO dvoDetection (imageID, ippDetectID, detectID, ippObjID, objID, flags, zp, zpErr, airMass, expTime, ra, dec_, raErr, decErr) VALUES (%d, %u, %lu, %lu, %lu, %u, %f, %f, %f, %f, %lf, %lf, %f, %f)",
    9799                        image->imageID,    // imageID
    98100                        meas->detID,   // ippDetectID
    99101                        meas->extID,   // detectID
    100102                        (uint64_t)ave->catID*1000000000 + (uint64_t)ave->objID, // ippObjID
    101                         dvoDetections[i].ave.extID,   // objID
    102                         meas->dbFlags,                // flags
     103                        ave->extID,   // objID
     104                        meas->dbFlags,  // flags
    103105                        code->C * 0.001 + code->K * (meas->airmass - 1) - meas->Mcal,   // zp
    104                         meas->dMcal,                  // zpErr
     106                        meas->dMcal,    // zpErr
    105107                        meas->airmass,
    106108                        pow(10.0, 0.4 * meas->dt),    // expTime
    107109                        ave->R - meas->dR / 3600.,    // ra
    108110                        ave->D - meas->dR / 3600.,    // dec
    109                         meas->dXccd * meas->pltscale, // estimate of raErr
    110                         meas->dYccd * meas->pltscale // estimate of decErr
     111                        meas->dXccd * 0.01 * fabs(meas->pltscale), // estimate of raErr
     112                        meas->dYccd * 0.01 * fabs(meas->pltscale) // estimate of decErr
    111113                        );
    112114

  • branches/eam_branches/ipp-20121130/ippTools/configure.ac

    r34081 r34839  
    11AC_PREREQ(2.61)
    22
    3 AC_INIT([ipptools], [1.1.71], [ipp-support@ifa.hawaii.edu])
     3AC_INIT([ipptools], [1.1.73], [ipp-support@ifa.hawaii.edu])
    44AC_CONFIG_SRCDIR([autogen.sh])
    55

  • branches/eam_branches/ipp-20121130/ippTools/share/Makefile.am

    r34265 r34839  
    8888        chiptool_listrun.sql \
    8989        chiptool_pendingcleanupimfile.sql \
     90        chiptool_pendingcleanupimfile_all.sql \
    9091        chiptool_pendingcleanuprun.sql \
    9192        chiptool_pendingimfile.sql \
    … …  
    172173        difftool_pendingcleanuprun.sql \
    173174        difftool_pendingcleanupskyfile.sql \
     175        difftool_pendingcleanupskyfile_all.sql \
    174176        difftool_revertdiffskyfile_delete.sql \
    175177        difftool_revertdiffskyfile_updated.sql \
    … …  
    391393        stacktool_sassskyfile.sql \
    392394        stacktool_tosum.sql \
     395        stacktool_tobkg.sql \
    393396        stacktool_tosummary.sql \
    394397        stacktool_addsummary.sql \
    … …  
    423426        warptool_pendingcleanuprun.sql \
    424427        warptool_pendingcleanupskyfile.sql \
     428        warptool_pendingcleanupskyfile_all.sql \
    425429        warptool_revertoverlap.sql \
    426430        warptool_revertwarped_delete.sql \

  • branches/eam_branches/ipp-20121130/ippTools/share/camtool_find_pendingimfile.sql

  • branches/eam_branches/ipp-20121130/ippTools/share/camtool_pendingcleanuprun.sql

    r25324 r34839  
    22    camRun.cam_id,
    33    rawExp.camera,
     4    rawExp.exp_tag,
     5    camRun.workdir,
    46    camRun.state
    57FROM camRun

  • branches/eam_branches/ipp-20121130/ippTools/share/chiptool_pendingcleanuprun.sql

    r22751 r34839  
    1 SELECT
     1SELECT DISTINCT
    22    chipRun.chip_id,
    33    rawExp.camera,
    4     chipRun.state
     4    rawExp.exp_tag,
     5    chipRun.state,
     6    chipRun.workdir
    57FROM chipRun
    68JOIN rawExp

  • branches/eam_branches/ipp-20121130/ippTools/share/chiptool_setimfiletoupdate.sql

  • branches/eam_branches/ipp-20121130/ippTools/share/difftool_pendingcleanuprun.sql

    r32218 r34839  
    66            rawExp.camera,
    77            diffRun.state,
     8            diffRun.workdir,
     9            diffRun.tess_id,
    810            diffRun.label
    911        FROM diffRun
    … …  
    3537            rawExp.camera,
    3638            diffRun.state,
     39            diffRun.workdir,
     40            diffRun.tess_id,
    3741            diffRun.label
    3842        FROM diffRun
    … …  
    6468            rawExp.camera,
    6569            diffRun.state,
     70            diffRun.workdir,
     71            diffRun.tess_id,
    6672            diffRun.label
    6773        FROM diffRun
    … …  
    9399            rawExp.camera,
    94100            diffRun.state,
     101            diffRun.workdir,
     102            diffRun.tess_id,
    95103            diffRun.label
    96104        FROM diffRun

  • branches/eam_branches/ipp-20121130/ippTools/share/pstamptool_revertdependent.sql

    r29286 r34839  
    44SET pstampDependent.fault = 0
    55-- fault count hook %s
    6 WHERE pstampDependent.state = 'new'
     6WHERE pstampRequest.state ='run'
     7    AND pstampJob.state ='run'
     8    AND pstampDependent.state = 'new'
    79    AND (pstampDependent.fault > 0)

  • branches/eam_branches/ipp-20121130/ippTools/share/pxadmin_create_tables.sql

    r34750 r34839  
    21192119      n_detections INT,
    21202120      n_extended INT,
     2121      n_forced INT,
    21212122      zpt_obs FLOAT,
    21222123      zpt_stdev FLOAT,

  • branches/eam_branches/ipp-20121130/ippTools/share/stacktool_inputskyfile.sql

    r32937 r34839  
    44    rawExp.exp_id,
    55    rawExp.exp_name,
     6    rawExp.exp_time,
    67    rawExp.object,
    78    rawExp.dateobs,

  • branches/eam_branches/ipp-20121130/ippTools/share/stacktool_pendingcleanuprun.sql

    r24186 r34839  
    88    stackRun.stack_id,
    99    rawExp.camera,
    10     stackRun.state
     10    stackRun.state,
     11    stackSumSkyfile.path_base
    1112FROM stackRun
     13JOIN stackSumSkyfile using(stack_id)
    1214JOIN stackInputSkyfile
    1315    USING(stack_id)

  • branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_defineskycalrun.sql

    r32960 r34839  
    22    sky_id,
    33    stack_id,
     4    skycell_id,
     5    filter,
    46    staticskyRun.workdir,
    57    staticskyRun.label,
    … …  
    1113    JOIN stackRun USING(stack_id)
    1214    JOIN stackSumSkyfile USING(stack_id)
     15    JOIN skycell USING(tess_id, skycell_id)
    1316    LEFT JOIN skycalRun USING(sky_id, stack_id)
    1417WHERE staticskyRun.state = 'full'

  • branches/eam_branches/ipp-20121130/ippTools/share/staticskytool_pendingskycalrun.sql

    r34743 r34839  
    66    staticskyResult.path_base,
    77    rawExp.camera,
    8     0 AS singlefilter
     8    staticskyResult.num_inputs = 1 AS singlefilter
    99FROM skycalRun
    1010JOIN staticskyRun USING (sky_id)

  • branches/eam_branches/ipp-20121130/ippTools/share/warptool_pendingcleanuprun.sql

    r23418 r34839  
    11SELECT
    22    warpRun.warp_id,
     3    warpRun.workdir,
    34    rawExp.camera,
     5    rawExp.exp_tag,
    46    warpRun.state
    57FROM warpRun

  • branches/eam_branches/ipp-20121130/ippTools/share/warptool_towarped.sql

  • branches/eam_branches/ipp-20121130/ippTools/src

  • branches/eam_branches/ipp-20121130/ippTools/src/chiptool.c

    r34501 r34839  
    13111311    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false);
    13121312    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     1313    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    13131314    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    13141315
    … …  
    13191320    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    13201321
    1321     psString query = pxDataGet("chiptool_pendingcleanupimfile.sql");
     1322    char *sql_file = all ? "chiptool_pendingcleanupimfile_all.sql" : "chiptool_pendingcleanupimfile.sql";
     1323    psString query = pxDataGet(sql_file);
    13221324    if (!query) {
    13231325        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");

  • branches/eam_branches/ipp-20121130/ippTools/src/chiptoolConfig.c

    r34501 r34839  
    296296    psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0,          "search by chip ID", 0);
    297297    psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id",                 0,            "search by exp_id", NULL);
     298    psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-all",  0,            "list all components regardless of data_state", false);
    298299    psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    299300    psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);

  • branches/eam_branches/ipp-20121130/ippTools/src/difftool.c

    r34538 r34839  
    25612561
    25622562    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false);
     2563    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    25632564    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    25642565    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    … …  
    25702571    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
    25712572
    2572     psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
     2573    char * sql_file  = all ? "difftool_pendingcleanupskyfile_all.sql": "difftool_pendingcleanupskyfile.sql" ;
     2574    psString query = pxDataGet(sql_file);
    25732575    if (!query) {
    25742576        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    … …  
    27402742    PS_ASSERT_PTR_NON_NULL(config, false);
    27412743
    2742     PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
    2743     PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
    2744 
    2745     psMetadata *where = psMetadataAlloc();
    2746     PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");
    2747     PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id",   "==");
    2748 
    2749     if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) {
    2750         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
     2744    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
     2745
     2746    if (!state) {
     2747        psMetadata *where = psMetadataAlloc();
     2748        PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");
     2749        PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id",   "==");
     2750
     2751        PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
     2752        PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
     2753
     2754        if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) {
     2755            psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
     2756            psFree (where);
     2757            return false;
     2758        }
    27512759        psFree (where);
    2752         return false;
    2753     }
    2754     psFree (where);
     2760    } else {
     2761      if (strcmp(state,"error_cleaned") == 0) {
     2762        change_skyfile_data_state(config,"error_cleaned", "goto_cleaned");
     2763      }
     2764      else if (strcmp(state, "error_scrubbed") == 0) {
     2765        change_skyfile_data_state(config,"error_scrubbed", "goto_scrubbed");
     2766      }
     2767      else if (strcmp(state, "error_purged") == 0) {
     2768        change_skyfile_data_state(config,"error_purged", "goto_purged");
     2769      }
     2770      else {
     2771        psError(PS_ERR_UNKNOWN, false, "unhandled state given");
     2772        return(false);
     2773      }
     2774    }
    27552775
    27562776    return true;

  • branches/eam_branches/ipp-20121130/ippTools/src/difftoolConfig.c

    r34443 r34839  
    395395    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
    396396    psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,          "search by diff ID", 0);
     397    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all",  0,            "list all skycells regardless of data_state", false);
    397398    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    398399    psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    … …  
    411412    psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
    412413    psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     414
    413415    // -updatediffskyfile
    414416    psMetadata *updatediffskyfileArgs = psMetadataAlloc();
    … …  
    416418    psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,   "search by skycell ID", NULL);
    417419    psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0,        "set fault code (required)", 0);
    418     psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0,      "set quality", 0);
     420    psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0,  "set quality", 0);
     421    psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-set_state", 0,    "set data_state state", NULL);
    419422
    420423    // -tocleanedskyfile

  • branches/eam_branches/ipp-20121130/ippTools/src/magictool.c

  • branches/eam_branches/ipp-20121130/ippTools/src/stacktool.c

    r34731 r34839  
    3737static bool inputskyfileMode(pxConfig *config);
    3838static bool tosumMode(pxConfig *config);
     39static bool tobkgMode(pxConfig *config);
    3940static bool addsumskyfileMode(pxConfig *config);
    4041static bool sumskyfileMode(pxConfig *config);
    … …  
    7677        MODECASE(STACKTOOL_MODE_INPUTSKYFILE,          inputskyfileMode);
    7778        MODECASE(STACKTOOL_MODE_TOSUM,                 tosumMode);
     79        MODECASE(STACKTOOL_MODE_TOBKG,                 tobkgMode);
    7880        MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE,         addsumskyfileMode);
    7981        MODECASE(STACKTOOL_MODE_SUMSKYFILE,            sumskyfileMode);
    … …  
    952954        // negative simple so the default is true
    953955        if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) {
     956            psError(PS_ERR_UNKNOWN, false, "failed to print array");
     957            psFree(output);
     958            return false;
     959        }
     960    }
     961
     962    psFree(output);
     963
     964    return true;
     965}
     966
     967static bool tobkgMode(pxConfig *config)
     968{
     969    PS_ASSERT_PTR_NON_NULL(config, false);
     970
     971    psMetadata *where = psMetadataAlloc();
     972    PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
     973    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
     974
     975    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     976    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     977
     978    psString query = pxDataGet("stacktool_tobkg.sql");
     979    if (!query) {
     980        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
     981        return false;
     982    }
     983
     984    psString whereClause = psStringCopy(""); // WHERE conditions to add
     985    if (psListLength(where->list)) {
     986        psString new = psDBGenerateWhereConditionSQL(where, NULL);
     987        psStringAppend(&whereClause, "\nAND %s", new);
     988        psFree(new);
     989    }
     990    psFree(where);
     991
     992    psStringAppend(&query, "\nORDER by priority DESC, stack_id");
     993
     994    // treat limit == 0 as "no limit"
     995    if (limit) {
     996        psString limitString = psDBGenerateLimitSQL(limit);
     997        psStringAppend(&query, " %s", limitString);
     998        psFree(limitString);
     999    }
     1000
     1001    if (!p_psDBRunQueryF(config->dbh, query, whereClause)) {
     1002        psError(PS_ERR_UNKNOWN, false, "database error");
     1003        psFree(query);
     1004        return false;
     1005    }
     1006    psFree(query);
     1007
     1008    psArray *output = p_psDBFetchResult(config->dbh);
     1009    if (!output) {
     1010        psErrorCode err = psErrorCodeLast();
     1011        switch (err) {
     1012            case PS_ERR_DB_CLIENT:
     1013                psError(PXTOOLS_ERR_SYS, false, "database error");
     1014            case PS_ERR_DB_SERVER:
     1015                psError(PXTOOLS_ERR_PROG, false, "database error");
     1016            default:
     1017                psError(PXTOOLS_ERR_PROG, false, "unknown error");
     1018        }
     1019
     1020        return false;
     1021    }
     1022    if (!psArrayLength(output)) {
     1023        psTrace("stacktool", PS_LOG_INFO, "no rows found");
     1024        psFree(output);
     1025        return true;
     1026    }
     1027
     1028    if (psArrayLength(output)) {
     1029        // negative simple so the default is true
     1030        if (!ippdbPrintMetadatas(stdout, output, "stackBkgSkyfile", !simple)) {
    9541031            psError(PS_ERR_UNKNOWN, false, "failed to print array");
    9551032            psFree(output);
    … …  
    16811758    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
    16821759    PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
     1760    PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
    16831761
    16841762    psMetadata *where = psMetadataAlloc();
    16851763    PXOPT_COPY_S64(config->args, where, "-stack_id",   "stack_id",   "==");
    16861764
    1687     if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault, quality)) {
     1765    if (background_model) {
     1766      psMetadata *values = psMetadataAlloc();
     1767      PXOPT_COPY_S16(config->args, values, "-set_background_model", "background_model", "==");
     1768      long rows = psDBUpdateRows(config->dbh,"stackSumSkyfile", where, values);
     1769      psFree(values);
     1770      if (!rows) {
     1771        // This maybe should rollback and error if rows != 1
     1772        psError(PS_ERR_UNKNOWN, true, "no rows changed");
     1773        return false;
     1774      }
     1775    }
     1776    else {   
     1777      if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault, quality)) {
    16881778        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
    16891779        psFree (where);
    16901780        return false;
     1781      }
    16911782    }
    16921783    psFree (where);

  • branches/eam_branches/ipp-20121130/ippTools/src/stacktool.h

    r28375 r34839  
    3131    STACKTOOL_MODE_INPUTSKYFILE,
    3232    STACKTOOL_MODE_TOSUM,
     33    STACKTOOL_MODE_TOBKG,
    3334    STACKTOOL_MODE_ADDSUMSKYFILE,
    3435    STACKTOOL_MODE_SUMSKYFILE,

  • branches/eam_branches/ipp-20121130/ippTools/src/stacktoolConfig.c

    r34731 r34839  
    150150    psMetadataAddBool(tosumArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    151151
     152    // -tobkg
     153    psMetadata *tobkgArgs = psMetadataAlloc();
     154    psMetadataAddS64(tobkgArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
     155    psMetadataAddStr(tobkgArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     156    psMetadataAddU64(tobkgArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     157    psMetadataAddBool(tobkgArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     158   
    152159    // -addsumskyfile
    153160    psMetadata *addsumskyfileArgs = psMetadataAlloc();
    … …  
    280287    psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-fault", 0,            "set fault code (required)", 0);
    281288    psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-set_quality", 0,            "set quality", 0);
    282 
     289    psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-set_background_model", 0,   "set background model", 0);
     290   
    283291    // -exportrun
    284292    psMetadata *exportrunArgs = psMetadataAlloc();
    … …  
    304312    PXOPT_ADD_MODE("-inputskyfile",    "", STACKTOOL_MODE_INPUTSKYFILE,    inputskyfileArgs);
    305313    PXOPT_ADD_MODE("-tosum",           "", STACKTOOL_MODE_TOSUM,          tosumArgs);
     314    PXOPT_ADD_MODE("-tobkg",           "", STACKTOOL_MODE_TOBKG,          tobkgArgs);
    306315    PXOPT_ADD_MODE("-addsumskyfile",   "", STACKTOOL_MODE_ADDSUMSKYFILE,   addsumskyfileArgs);
    307316    PXOPT_ADD_MODE("-sumskyfile",      "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE,      sumskyfileArgs);

  • branches/eam_branches/ipp-20121130/ippTools/src/staticskytool.c

    r34733 r34839  
    132132    pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "stackRun.data_group",       "LIKE");
    133133    pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
    134 #ifdef notdef
    135     PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min",      "skycell.glat",              ">=");
    136     PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max",      "skycell.glat",              "<=");
    137     PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min",   "skycell.radeg/15",          ">=");
    138     PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max",   "skycell.radeg/15",          "<=");
    139     PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min",     "skycell.radeg",             ">=");
    140     PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max",     "skycell.radeg",             "<=");
    141     PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min",    "skycell.decdeg",             ">=");
    142     PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max",    "skycell.decdeg",             "<=");
    143 #endif
    144134    if (!pxskycellAddWhere(config, whereMD)) {
    145135        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
    … …  
    686676    psStringAppend(&query, "\nGROUP BY sky_id");
    687677    if (num_filters) {
    688         psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters);
     678        psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters);
    689679    }
    690680       
    … …  
    10491039    pxAddLabelSearchArgs(config, whereMD, "-select_data_group",    "staticskyRun.data_group",       "LIKE");
    10501040    pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
     1041    if (!pxskycellAddWhere(config, whereMD)) {
     1042        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
     1043        return false;
     1044    }
    10511045
    10521046    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
    … …  
    11281122        psString sky_label =  psMetadataLookupStr(&status, row, "label");
    11291123        psString sky_data_group =  psMetadataLookupStr(&status, row, "data_group");
     1124        psString sky_dist_group =  psMetadataLookupStr(&status, row, "dist_group");
    11301125
    11311126        // create a staticskyRun
    … …  
    11381133                                label ? label : sky_label,
    11391134                                data_group ? data_group : sky_data_group,
    1140                                 dist_group,
     1135                                dist_group ? dist_group : sky_dist_group,
    11411136                                reduction,
    11421137                                registered,
    … …  
    11591154
    11601155    psMetadata *where = psMetadataAlloc();
    1161     PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycal_id",   "==");
    1162     PXOPT_COPY_STR(config->args, where, "-label",   "label",    "==");
    1163     PXOPT_COPY_STR(config->args, where, "-state",   "state",    "==");
     1156    PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id",   "==");
     1157    PXOPT_COPY_S64(config->args, where, "-sky_id",   "sky_id",   "==");
     1158    PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id",   "==");
     1159    PXOPT_COPY_STR(config->args, where, "-label",   "skycalRun.label",    "==");
     1160    PXOPT_COPY_STR(config->args, where, "-state",   "skycalRun.state",    "==");
     1161    if (!pxskycellAddWhere(config, where)) {
     1162        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
     1163        return false;
     1164    }
    11641165    if (!psListLength(where->list)) {
    11651166        psFree(where);
    … …  
    11681169    }
    11691170
    1170     psString query = psStringCopy("UPDATE skycalRun");
     1171    psString query = psStringCopy("UPDATE skycalRun JOIN stackRun using(stack_id) join skycell using(tess_id, skycell_id)");
    11711172
    11721173    // pxUpdateRun gets parameters from config->args and updates
    … …  
    12761277    PXOPT_LOOKUP_STR(hostname, config->args,    "-hostname", false, false);
    12771278    PXOPT_LOOKUP_S32(n_astrom, config->args,    "-n_astrom", false, false);
    1278     PXOPT_LOOKUP_S32(n_detections, config->args,    "-n_detections", false, false);
    1279     PXOPT_LOOKUP_S32(n_extended, config->args,    "-n_extended", false, false);
     1279    PXOPT_LOOKUP_S32(n_detections, config->args,"-n_detections", false, false);
     1280    PXOPT_LOOKUP_S32(n_extended, config->args,  "-n_extended", false, false);
     1281    PXOPT_LOOKUP_S32(n_forced, config->args,    "-n_forced", false, false);
    12801282
    12811283    PXOPT_LOOKUP_STR(path_base, config->args,   "-path_base", false, false);
    … …  
    13181320        n_detections,
    13191321        n_extended,
     1322        n_forced,
    13201323        zpt_obs,
    13211324        zpt_stdev,

  • branches/eam_branches/ipp-20121130/ippTools/src/staticskytoolConfig.c

    r34733 r34839  
    126126    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
    127127    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
    128     psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0);
     128    psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0);
    129129    pxskycellAddArguments(resultArgs);
    130130    psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
    … …  
    166166    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
    167167    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
     168    pxskycellAddArguments(defineskycalrunArgs);
    168169    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
    169170    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
    … …  
    181182    psMetadata *updateskycalrunArgs = psMetadataAlloc();
    182183    psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0);
     184    psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0,    "search by sky ID", 0);
     185    psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-stack_id", 0,  "search by stack ID", 0);
    183186    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
    184187    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
     188    pxskycellAddArguments(updateskycalrunArgs);
    185189    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL);
    186190    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
    … …  
    215219    psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_detections", 0,        "define number of detections", 0);
    216220    psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_extended", 0,          "define number of extended detections", 0);
     221    psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_forced", 0,            "define number of forced detections", 0);
    217222
    218223    psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);

  • branches/eam_branches/ipp-20121130/ippTools/src/warptool.c

    r34731 r34839  
    18381838    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    18391839    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     1840    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    18401841
    18411842    psMetadata *where = psMetadataAlloc();
    … …  
    18431844    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
    18441845
    1845     psString query = pxDataGet("warptool_pendingcleanupskyfile.sql");
     1846    char * sql_file = all ? "warptool_pendingcleanupskyfile_all.sql" : "warptool_pendingcleanupskyfile.sql";
     1847    psString query = pxDataGet(sql_file);
    18461848    if (!query) {
    18471849        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
    … …  
    20572059        PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
    20582060        PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false);
     2061
    20592062        if (magicked) {
    20602063            psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked);
    … …  
    20632066        if (quality) {
    20642067          psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.quality = %"PRId16, quality);
     2068        }
     2069        PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
     2070        if (background_model) {
     2071          psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.background_model = %"PRId16, background_model);
    20652072        }
    20662073    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
    … …  
    21232130    // warp_id, skycell_id, fault are required
    21242131    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
    2125 
    2126     if (!state) {
     2132    PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false);
     2133    if (background_model) {
     2134      // CZW 2012-12-06: I'm unclear why we don't use this form for all updates?
     2135      psMetadata *where = psMetadataAlloc();
     2136      psMetadata *values = psMetadataAlloc();
     2137      PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
     2138      PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
     2139      PXOPT_COPY_S16(config->args, values, "-set_background_model", "background_model", "==");
     2140      long rows = psDBUpdateRows(config->dbh,"warpSkyfile", where, values);
     2141      psFree(values);
     2142      psFree(where);
     2143      if (!rows) {
     2144        // This maybe should rollback and error if rows != 1
     2145        psError(PS_ERR_UNKNOWN, true, "no rows changed");
     2146        return false;
     2147      }
     2148    }
     2149    else if (!state) {
    21272150      psMetadata *where = psMetadataAlloc();
    21282151      PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");

  • branches/eam_branches/ipp-20121130/ippTools/src/warptoolConfig.c

    r34731 r34839  
    364364    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
    365365    psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,          "search by warp ID (required)", 0);
     366    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all",  0,            "list all skycells regardless of data_state", false);
    366367    psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    367368    psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    … …  
    409410    psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-set_quality",  0,"new quality value", 0);
    410411    psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0,   "set state", 0);
    411 
     412    psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-set_background_model", 0, "set the background_model value", 0);
    412413    // -exportrun
    413414    psMetadata *exportrunArgs = psMetadataAlloc();

  • branches/eam_branches/ipp-20121130/magic/censorObjects/src/censorLoop.c

    r33626 r34839  
    2929
    3030
    31 
    32 
    3331    // select the current recipe
    3432    psMetadata *recipe  = psMetadataLookupPtr (NULL, config->recipes, CENSOR_RECIPE);
    … …  
    5856    // check on the mask file at this level if possible
    5957    badMaskFile = censorBadFileByView(badMaskFile, inMask, config, view);
     58
     59    bool checkInputOnly = psMetadataLookupBool(NULL, config->arguments, "CHECK.INPUT.ONLY");
    6060
    6161    // files associated with the science image
    … …  
    116116                if (!readout->data_exists) { continue; }
    117117
    118                 if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE;
     118                if (!checkInputOnly) {
     119                    if (!censorReadout (readout, config, view, badMaskFile)) ESCAPE;
     120                }
    119121
    120122                if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
Note: See TracChangeset for help on using the changeset viewer.