IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Dec 3, 2010, 2:35:32 AM (16 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20101103/ippScripts/scripts
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/camera_exp.pl

    r28043 r29906  
    7070my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $cam_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    7171
    72 my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    73 
    7472if (not defined $run_state) { $run_state = 'new'; }
    75 if ($run_state eq 'update') {
    76     $logDest .= '.update';
     73
     74my_die ("$run_state is an invalid value for run-state", $cam_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update');
     75
     76
     77my $replicateOutputs = 1;
     78
     79my $logDest;
     80my $traceDest;
     81if ($run_state eq 'new') {
     82    $logDest = prepare_output("LOG.EXP", $outroot, undef, 0);
     83    $traceDest = prepare_output("TRACE.EXP", $outroot, undef, 0);
     84} else {
     85    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, undef, 0);
     86    $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, undef, 0);
    7787}
    7888
    … …  
    188198    print $list4File ($chipMask . "\n");
    189199
    190     push @outMasks, $ipprc->filename("PSASTRO.OUTPUT.MASK", $outroot, $class_id) if $produceMasks;
     200    push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks;
    191201}
    192202close $list1File;
    … …  
    199209
    200210# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs
    201 my $jpeg1      = $ipprc->filename("PPIMAGE.JPEG1",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    202 my $jpeg2      = $ipprc->filename("PPIMAGE.JPEG2",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    203 my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",     $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    204 my $fpaStats   = $ipprc->filename("PSASTRO.STATS",      $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    205 my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    206 my $configuration = $ipprc->filename("PSASTRO.CONFIG",  $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
    207 
    208 if ($run_state eq 'update') {
    209     $traceDest .= '.update';
    210     $fpaStats .= '.update';
    211 }
    212 
    213 # convert supplied DVO database name to UNIX filename
    214 my $dvodbReal;
    215 if (defined $dvodb) {
    216     $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO
    217     $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal );
    218 }
    219 
    220 #my $dtime_addstar = 0;
     211my $jpeg1      = prepare_output("PPIMAGE.JPEG1",      $outroot, undef, 1);
     212my $jpeg2      = prepare_output("PPIMAGE.JPEG2",      $outroot, undef, 1);
     213my $fpaObjects = prepare_output("PSASTRO.OUTPUT",     $outroot, undef, 1);
     214my $configuration = prepare_output("PSASTRO.CONFIG",  $outroot, undef, 1);
     215
     216my $do_stats;
     217my $fpaStats;
     218if ($run_state eq 'new') {
     219    $do_stats = 1;
     220    $fpaStats = prepare_output("PSASTRO.STATS",      $outroot, undef, 1);
     221} else {
     222    $do_stats = 0;
     223}
    221224
    222225unless ($no_op) {
    … …  
    236239            &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code);
    237240        }
    238         &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1);
     241        check_output($jpeg1, $replicateOutputs);
    239242    }
    240243
    … …  
    251254            &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code);
    252255        }
    253         &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2);
     256        check_output($jpeg2, $replicateOutputs);
    254257    }
    255258
    … …  
    265268        $command .= " -dbname $dbname" if defined $dbname;
    266269
    267         my $do_stats;
    268270        if ($run_state eq 'new') {
    269             $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS";
    270271            $command .= " -dumpconfig $configuration";
    271             $do_stats = 1;
    272272        } elsif ($run_state eq 'update') {
    273273            $command .= " -ipprc $configuration";
    … …  
    275275            &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR);
    276276        }
     277        $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS" if $do_stats;
    277278
    278279        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    286287        my $quality;            # Quality flag
    287288        if ($do_stats) {
     289            check_output($fpaStats, $replicateOutputs);
     290
    288291            my $fpaStatsReal = $ipprc->file_resolve($fpaStats);
    289             &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $fpaStatsReal );
    290292
    291293            # parse stats from metadata
    … …  
    306308
    307309        if (!$quality) {
    308             &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($fpaObjects);
     310            check_output($fpaObjects, $replicateOutputs);
    309311
    310312            foreach my $outMask (@outMasks) {
    311                 &my_die("Unable to find expected output file: $outMask", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outMask);
     313                check_output($outMask, $replicateOutputs);
    312314            }
    313315
    314316            if ($run_state eq 'new') {
    315                 &my_die("Couldn't find expected output file: $configuration", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
     317                check_output($configuration, $replicateOutputs);
    316318            }
    317319        }
    318 
    319         # run addstar on the output fpaObjects (if a DVO database is defined)
    320 #         if (defined $dvodbReal and ($run_state eq 'new')) {
    321 
    322 #             ## XXX the camera analysis can either save the full set of
    323 #             ## detections, or just the image metadata, in the dvodb
    324 
    325 #             ## get the addstar recipe for this camera and CAMERA reduction
    326 #             $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -";
    327 #             ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    328 #                 run(command => $command, verbose => $verbose);
    329 #             unless ($success) {
    330 #                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    331 #                 &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR);
    332 #             }
    333 #             my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    334 #                 &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR);
    335 
    336 #             ## allow the dvodb to save only images, or the full detection set
    337 #             my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY');
    338 
    339 #             # XXX this construct requires the user to have a valid .ptolemyrc
    340 #             # XXX which in turn points at ippconfig/dvo.site
    341 #             # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default)
    342 #             # XXX this needs to be converted to addstar_client...
    343 
    344 #             my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar
    345 #             my $command;
    346 #             $command  = "$addstar -D CAMERA $camdir -update";
    347 #             $command .= " -image" if $imagesOnly;
    348 #             $command .= " -D CATDIR $dvodbReal";
    349 
    350 #             my $realFile = $ipprc->file_resolve($fpaObjects);
    351 #             $command .= " $realFile";
    352 
    353 #             my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
    354 
    355 #             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    356 #                 run(command => $command, verbose => $verbose);
    357 #             unless ($success) {
    358 #                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    359 #                 &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code);
    360 #             }
    361 #             $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start);   # MJD of starting script
    362 #         }
    363320    }
    364321}
    … …  
    374331    $fpaCommand .= " -hostname $host" if defined $host;
    375332    $fpaCommand .= " -dtime_script $dtime_script";
    376 #    $fpaCommand .= " -dtime_addstar $dtime_addstar";
    377333} else {
    378334    $fpaCommand .= " -updaterun -set_state full";
    … …  
    393349}
    394350
     351exit 0;
     352
     353sub prepare_output
     354{
     355    my $filerule = shift;
     356    my $outroot  = shift;
     357    my $class_id = shift;
     358    my $delete = shift;
     359    $delete = 0 if !defined $delete;
     360
     361    my $error;
     362    my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error)
     363                    or &my_die("failed to prepare output file for: $filerule", $cam_id, $error);
     364
     365    return $output;
     366}
     367
     368sub check_output
     369{
     370    my $file = shift;
     371    my $replicate = shift;
     372
     373    if (!defined $file) {
     374        return;
     375    }
     376
     377    &my_die("Couldn't find expected output file: $file",  $cam_id, $PS_EXIT_SYS_ERROR) unless
     378        $ipprc->file_exists($file);
     379
     380    if ($replicate and (file_scheme($file) eq 'neb')) {
     381        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $cam_id, $PS_EXIT_SYS_ERROR);
     382    }
     383}
    395384
    396385sub my_die

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/chip_imfile.pl

    r28899 r29906  
    6666
    6767pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    68 pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state",
     68pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state --dbname",
    6969           -exitval => 3) unless
    7070    defined $exp_id and
    … …  
    7575    defined $camera and
    7676    defined $outroot and
     77    defined $dbname and
    7778    defined $run_state;
    7879
    7980my_die ("$run_state is an invalid value for run-state", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update');
    8081
    81 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
    82 
    83 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    84 $logDest .= ".update" if $run_state eq "update";
     82my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR );
     83
     84my $neb;
     85my $scheme = file_scheme($outroot);
     86if ($scheme and $scheme eq 'neb') {
     87    $neb = $ipprc->nebulous();
     88}
     89
     90my ($logDest, $traceDest);
     91if ($run_state eq 'new') {
     92    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, 0);
     93    $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
     94} else {
     95    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
     96    $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
     97}
    8598
    8699if ($redirect) {
    … …  
    90103    print STDOUT "FULL COMMAND: $0 @ARGS\n\n";
    91104}
     105
    92106
    93107# Recipes to use based on reduction class
    … …  
    114128
    115129## these names are used in ppImage, and thus may be URIs
    116 my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    117 my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    118 my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    119 my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    120 my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    121 my $outputStats   = $ipprc->filename("PPIMAGE.STATS",       $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    122 my $traceDest     = $ipprc->filename("TRACE.IMFILE",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    123 my $configuration = $ipprc->filename("PPIMAGE.CONFIG",      $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
    124 
    125 if ($run_state eq 'update') {
    126     $outputStats .= '.update';
    127     $traceDest .= '.update';
     130my $outputImage   = prepare_output("PPIMAGE.CHIP",          $outroot, $class_id, 1 );
     131my $outputMask    = prepare_output("PPIMAGE.CHIP.MASK",     $outroot, $class_id, 1);
     132my $outputWeight  = prepare_output("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id, 1);
     133my $pattern       = prepare_output("PPIMAGE.PATTERN",       $outroot, $class_id, 1);
     134my $backmdl       = prepare_output("PSPHOT.BACKMDL",        $outroot, $class_id, 1);
     135
     136my $configuration;
     137my $outputSources;
     138my $outputPsf;
     139my $outputStats;
     140my $outputBin1;
     141my $outputBin2;
     142my $dump_config = 1;
     143my $do_binned_images = 1;
     144if ($run_state eq 'new') {
     145    # prepare the files that are only created for a new run
     146    $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
     147    $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
     148} else {
     149    $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id)
     150            or &my_die("Missing entry from camera config: PPIMAGE.CONFIG", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     151    if ($ipprc->file_exists($configuration)) {
     152        $dump_config = 0;
     153    } else {
     154        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
     155
     156        # XXX: should we create a new config dump file?
     157        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
     158        my $gone = 0;
     159        if (storage_object_exists($configuration, \$gone)) {
     160            if ($gone) {
     161                rename_gone_file($configuration);
     162                $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
     163                # if we dump the config we need to insure that the config dump represents
     164                # the full processing
     165            } else {
     166                # file is temporarily not available. Don't dump config.
     167                $dump_config = 0;
     168            }
     169        }
     170    }
     171
    128172    # make sure that any lingering destreak backup files are gone
    129173    $ipprc->delete_destreak_backup_file($outputImage)
    … …  
    134178        or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
    135179}
     180if ($do_binned_images) {
     181    $outputBin1    = prepare_output("PPIMAGE.BIN1",          $outroot, $class_id, 1);
     182    $outputBin2    = prepare_output("PPIMAGE.BIN2",          $outroot, $class_id, 1);
     183}
    136184
    137185my $cmdflags;
    … …  
    152200    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
    153201        &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     202
     203    my $do_photom = metadataLookupBool($recipeData, 'PHOTOM');
     204    if ($do_photom and ($run_state eq 'update')) {
     205        # If previous photometry outputs are ok skip running photometry
     206        if ($dump_config || rerun_photometry($outroot, $class_id)) {
     207            carp "Will rerun photometry\n";
     208        } else {
     209            $do_photom = 0;
     210        }
     211    }
     212    if ($do_photom) {
     213        $outputSources = prepare_output("PSPHOT.OUTPUT",   $outroot, $class_id, 1);
     214        $outputPsf     = prepare_output("PSPHOT.PSF.SAVE", $outroot, $class_id, 1);
     215    }
     216
    154217
    155218    ## XXX make the feature more general?
    … …  
    357420    }
    358421
     422    $command  = "$ppImage -file $uri $outroot";
     423    if ($dump_config) {
     424        $command .= " -recipe PPIMAGE $recipe_ppImage";
     425        $command .= " -dumpconfig $configuration";
     426    } else {
     427        $command .= " -ipprc $configuration";
     428    }
     429    if ($do_photom) {
     430        $command .= " -recipe PSPHOT $recipe_psphot";
     431    } else {
     432        $command .= " -Db PPIMAGE:PHOTOM FALSE";
     433    }
    359434    if ($run_state eq "new") {
    360         $command  = "$ppImage -file $uri $outroot";
    361         $command .= " -recipe PPIMAGE $recipe_ppImage";
    362         $command .= " -recipe PSPHOT $recipe_psphot";
    363         $command .= " -threads $threads" if defined $threads;
    364         $command .= " -dbname $dbname" if defined $dbname;
    365         $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
    366         $command .= " -source_id $source_id" if defined $source_id;
    367         $command .= " -dumpconfig $configuration";
    368         $command .= " -tracedest $traceDest -log $logDest";
    369         $do_stats = 1;
    370     } else {
    371         $command  = "$ppImage -file $uri $outroot";
    372         $command .= " -ipprc $configuration";
    373         $command .= " -threads $threads" if defined $threads;
    374         $command .= " -dbname $dbname" if defined $dbname;
    375         $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
    376         $command .= " -source_id $source_id" if defined $source_id;
    377         $command .= " -tracedest $traceDest -log $logDest";
    378         $command .= " -Db PPIMAGE:PHOTOM FALSE";
    379     }
    380     if ($do_stats) {
    381435        $command .= " -recipe PPSTATS CHIPSTATS";
    382436        $command .= " -stats $outputStats";
    383     }
     437        $do_stats = 1;
     438    }
     439    if (!$do_binned_images) {
     440        $command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE";
     441    }
     442    $command .= " -threads $threads" if defined $threads;
     443    $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
     444    $command .= " -source_id $source_id" if defined $source_id;
     445    $command .= " -tracedest $traceDest -log $logDest";
     446    $command .= " -dbname $dbname" if defined $dbname;
    384447
    385448    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    394457    my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
    395458    my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.VARIANCE.FITS');
     459    my $outputBackmdlExpect = metadataLookupBool($recipeData, 'BACKGROUND');
     460    my $outputPatternExpect = (metadataLookupBool($recipeData, 'PATTERN.ROW') or metadataLookupBool($recipeData, 'PATTERN.CELL')) ;
    396461
    397462    my $quality;                # Quality flag
    … …  
    416481
    417482    if (!$quality) {
    418         &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputImageExpect or $ipprc->file_exists($outputImage);
    419         &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputMaskExpect or $ipprc->file_exists($outputMask);
    420         &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputWeightExpect or $ipprc->file_exists($outputWeight);
    421         &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
    422         &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
    423         if ($run_state eq 'new') {
    424             &my_die("Couldn't find expected output file: $configuration", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
    425         }
    426     }
    427 
     483        my $replicateImages = 0;
     484        check_output($outputImage, $replicateImages) if $outputImageExpect;
     485        check_output($outputMask, $replicateImages) if $outputMaskExpect;
     486        check_output($outputWeight, $replicateImages) if $outputWeightExpect;
     487        if ($do_binned_images) {
     488            check_output($outputBin1, 1);
     489            check_output($outputBin2, 1);
     490        }
     491        check_output($configuration, 1) if $dump_config;
     492        check_output($backmdl, 1) if $outputBackmdlExpect;
     493        check_output($pattern, 1) if $outputPatternExpect;
     494        if ($do_photom) {
     495            check_output($outputSources, 1);
     496            check_output($outputPsf, 1);
     497        }
     498        # XXX: Do we want to replicate the stats, logs, trace file?
     499    }
    428500}
    429501
    … …  
    462534}
    463535
     536exit 0;
     537
     538# check whether psphot outputs should be regenerated.
     539# Whether we need to or not is a somewhat complicated question.
     540sub rerun_photometry
     541{
     542    my $outroot = shift;
     543    my $class_id = shift;
     544    my $outputSources = $ipprc->filename('PSPHOT.OUTPUT', $outroot, $class_id)
     545                    or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     546
     547    my $make_sources = 0;
     548    my $sources_available = 0;
     549    if ($ipprc->file_exists($outputSources)) {
     550        $sources_available = 1;
     551    } else {
     552        carp "WARNING: photometry sources file $outputSources is not available";
     553        my $gone;
     554        if (storage_object_exists($outputSources, \$gone)) {
     555            # check whether the file is permanantely or temporarily gone
     556            if ($gone) {
     557                carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
     558                rename_gone_file($outputSources);
     559                $make_sources = 1;
     560            }
     561        } else {
     562            # storage object must have been deleted
     563            $make_sources = 1;
     564        }
     565    }
     566
     567    my $make_psf = 0;
     568    my $psf_available = 0;
     569    my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
     570                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     571    if ($ipprc->file_exists($outputPsf)) {
     572        $psf_available = 1;
     573    } else {
     574        carp "PSF file $outputPsf is missing";
     575        my $gone = 0;
     576        if (storage_object_exists($outputPsf, \$gone)) {
     577            # object exists, but no instances are available. If they are permanently gone
     578            # rename the storage object
     579            if ($gone) {
     580                carp "WARNING: PSF storage object exists but all instances are permanently gone";
     581                rename_gone_file($outputPsf);
     582                $make_psf = 1;
     583            }
     584        } else {
     585            $make_psf = 1;
     586        }
     587    }
     588
     589    if ($sources_available && $psf_available) {
     590        return 0;
     591    }
     592
     593    # if either of the files are gone rerun photometry unless the other file is temporarily not available
     594
     595    if (!$sources_available && !$make_sources) {
     596        # destreak will die if the sources is not available
     597        &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     598    }
     599    if (!$psf_available && !$make_psf) {
     600        # warp updates need the psf file
     601        &my_die("PSPHOT.PSF.SAVE is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     602    }
     603
     604    return $make_psf || $make_sources;
     605}
     606
     607# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
     608# does not exist and one that all of the instances have been lost.
     609# XXXX This should be implemented properly in Nebulous
     610# For now uses Bill's script 'whichnode' which queries the nebulous database directly
     611
     612my $whichnode;
     613sub storage_object_exists
     614{
     615    my $file = shift;
     616    my $ref_all_gone = shift;
     617
     618    my $exists = $neb->storage_object_exists($file);
     619    if (!$exists) {
     620        return 0;
     621    }
     622
     623    if (!$whichnode) {
     624        $whichnode = can_run('whichnode') or
     625            &my_die("Can't find whichnode",  $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     626    }
     627
     628    my $command = "$whichnode $file";
     629
     630    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     631        run(command => $command, verbose => $verbose);
     632    unless ($success) {
     633        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     634        &my_die("Unable to perform whichnode: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     635    }
     636
     637    my @lines = split "\n", (join "", @$stdout_buf);
     638
     639    if (scalar @lines == 0) {
     640        # no output the file is really and truely gone
     641        # XXX: this is now caught above
     642        print STDERR "storage object for $file does not exist\n";
     643        return 0;
     644    }
     645
     646    my $numGone = 0;
     647    my $numNotGone = 0;
     648    foreach my $line (@lines) {
     649        chomp $line;
     650
     651        # output lines are either
     652        #   "volume available"
     653        # or
     654        #   "volume not available"
     655
     656        my ($volume, $answer, undef) = split " ", $line;
     657        # our hack is if the volume has an X in the name it's gone
     658        if ($volume =~ /X/) {
     659            print STDERR "$file is on $volume which is gone\n";
     660            $numGone++;
     661        } elsif ($answer eq 'available') {
     662            $numNotGone++;
     663        } elsif ($answer eq 'not') {
     664            print STDERR "$file is on $volume which is not available\n";
     665            $numNotGone++;
     666        } else {
     667            print STDERR "unexpected output from whichnode: $line\n";
     668        }
     669    }
     670    # if there are any instances that are not on a gone volume set all_gone to 0
     671    if ($numNotGone == 0 and $numGone > 0) {
     672        $$ref_all_gone = 1;
     673    } else {
     674        $$ref_all_gone = 0;
     675    }
     676
     677    # storage object exists so return true
     678    return 1;
     679}
     680sub rename_gone_file
     681{
     682    # check whether the only instance of file is on a lost volume
     683    # XXX: we don't have a proper interface for this.
     684    # For now try to use Bill's hack: the script 'whichnode'
     685
     686    my $file = shift;
     687    $neb->move($file, "$file.gone") or
     688                    &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     689}
     690
     691# Prepare to write to an output file
     692#   Lookup the filename in the rules.
     693#   Make sure that if file exists and is a nebulous file that there is only one instance
     694#   Deal with files that have been lost.
     695sub prepare_output
     696{
     697    my $filerule = shift;
     698    my $outroot  = shift;
     699    my $class_id = shift;
     700    my $delete = shift;
     701    $delete = 0 if !defined $delete;
     702
     703    my $error;
     704    my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error)
     705                    or &my_die("failed to prepare output file for: $filerule", $exp_id, $chip_id, $class_id, $error);
     706    return $output;
     707}
     708
     709sub check_output
     710{
     711    my $file = shift;
     712    my $replicate = shift;
     713
     714    if (!defined $file) {
     715        return;
     716    }
     717
     718    &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
     719
     720    if ($replicate and (file_scheme($file) eq 'neb')) {
     721        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     722    }
     723}
    464724
    465725sub my_die

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_cleanup.pl

    r29839 r29906  
    19841984sub file_gone
    19851985{
    1986     # if $check_for_gone check whether the only instance of file is on a lost volumen
     1986    # if $check_for_gone check whether the only instance of file is on a lost volume
    19871987    # XXX: we don't have a proper interface for this.
    19881988    # For now try to use Bill's hack the script 'whichnode'
    … …  
    19901990
    19911991    my $file = shift;
     1992
     1993    if (file_scheme($file) ne 'neb') {
     1994        return 0;
     1995    }
    19921996
    19931997    if (!$whichnode) {
    … …  
    20212025            print STDERR "$file is on $volume which is gone\n";
    20222026            $numGone++;
     2027        } elsif ($answer eq 'available') {
     2028            $numNotGone++;
    20232029        } elsif ($answer eq 'not') {
    20242030            print STDERR "$file is on $volume which is not available\n";

  • branches/eam_branches/ipp-20101103/ippScripts/scripts/ipp_inject_fileset.pl

    r24201 r29906  
    2424# Parse the command-line arguments
    2525my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help);
     26my $exp_name;
    2627GetOptions('camera|i=s'     => \$camera,    # user-supplied camera name
    2728           'telescope|t=s'  => \$telescope, # user-supplied telescope name
     29           'exp_name|=s'    => \$exp_name,  # user-supplied exp_name name
    2830           'workdir|w=s'    => \$workdir,   # working directory for output files
    2931           'reduction=s'    => \$reduction, # user-supplied camera name
    … …  
    7981}
    8082
    81 # use the first file name as the exp_name (strip off .fits)
     83# if not supplied, use the first file name as the exp_name (strip off .fits)
    8284my $num = 0;
    83 my $exp_name;
    8485foreach my $file ( @ARGV ) {
    8586    # check for file existence
    … …  
    8990        # strip off the extension
    9091        my ( $vol, $path, $name ) = File::Spec->splitpath( $file );
    91         ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/;
     92        ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts|flt)(|.gz)/;
    9293        print "exp_name : $exp_name.\n";
    9394    }
Note: See TracChangeset for help on using the changeset viewer.