IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:41:49 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/tap_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/tap_branches

  • branches/tap_branches/ippScripts/scripts/dist_component.pl

    r25677 r27838  
    88use Sys::Hostname;
    99my $host = hostname();
     10my $date = `date`;
    1011print "\n\n";
    11 print "Starting script $0 on $host\n\n";
     12print "Starting script $0 on $host at $date\n\n";
    1213
    1314use vars qw( $VERSION );
    … …  
    2829use Pod::Usage qw( pod2usage );
    2930
    30 # filerules that are not included in 'cleaned' distribution bundles
    31 # I could simplify the function below by using a two level hash but
    32 # that would be less clear I think
    33 my %chip_cleaned = ( 'PPIMAGE.CHIP' => 'image',
    34                      'PPIMAGE.CHIP.MASK' => 'mask',
    35                      'PPIMAGE.CHIP.VARIANCE' => 'variance' );
    36 my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' );
    37 my %fake_cleaned;
    38 my %warp_cleaned = ( 'PSWARP.OUTPUT' => 'image',
    39                      'PSWARP.OUTPUT.MASK' => 'mask',
    40                      'PSWARP.OUTPUT.VARIANCE' => 'variance' );
    41 my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image',
    42                      'PPSUB.OUTPUT.MASK' => 'mask',
    43                      'PPSUB.OUTPUT.VARIANCE' => 'variance' );
    44 my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
    45                       'PPSTACK.OUTPUT.MASK' => 'mask',
    46                       'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
    47 
    48 
    4931# Look for programs we need
    5032my $missing_tools;
    5133my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
    52 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
     34my $dist_make_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
    5335if ($missing_tools) {
    5436    warn("Can't find required tools.");
    … …  
    5739
    5840# Parse the command-line arguments
    59 my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
     41my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean, $alt_path_base);
    6042my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality);
    6143my ($dbname, $save_temps, $verbose, $no_update, $logfile);
    … …  
    7456           'no_magic'       => \$no_magic,   # magic is not required for this distribution run
    7557           'magicked'       => \$magicked,   # magicked state for this component
     58           'alt_path_base=s'=> \$alt_path_base,  # path to alternate inputs
    7659           'outdir=s'       => \$outdir,     # "directory" for outputs
    7760           'clean'          => \$clean,      # create clean distribution
    … …  
    10083}
    10184
    102 $ipprc->define_camera($camera);
    103 
    10485my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    10586
    106 # making a clean bundle of raw images doesn't make sense
     87my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
     88close $rf;
    10789
    108 # create the output directories if it is not a nebulous path and it doesn't exist
    109 if (index($outdir, "neb://") != 0) {
    110     if (! -e $outdir ) {
    111         my $code = system "mkdir -p $outdir";
    112         &my_die("cannot create output directory $outdir", $dist_id, $component,
    113                 $code >> 8) if $code;
    114     }
    115 }
    116 
    117 # Get the list of data products for this component
    118 # note: We my_die in get_file_list if something goes wrong.
    119 
    120 my $file_list = get_file_list($stage, $component, $path_base, $clean);
    121 
    122 if (($stage ne 'raw') and ($stage ne 'fake')) {
    123     # If the file list is empty it is an error because we should at least get a config dump file
    124     # except for fake stage which doesn't do anything yet
    125     &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
    126 }
    127 
    128 # set up directory for temporary files
    129 
    130 my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
    131 if (-e $tmpdir) {
    132     if (-d $tmpdir) {
    133         my $rc = system "rm -r $tmpdir";
    134         &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
    135     } else {
    136         unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
    137     }
    138 }
    139 mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component,
    140                        $PS_EXIT_UNKNOWN_ERROR);
    141 
    142 #
    143 # we need to run set masked pixels to NAN in the image and variance images
    144 # unless
    145 #   1. we are building a clean distribution bundle
    146 #   2. magic is not required for this distRun
    147 #   3. the processing for the component produced no images (warp or diff with bad quality for example)
    148 my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
    149 
    150 my ($image, $mask, $variance);
    151 
    152 # foreach my $file_rule (keys %$file_list) {
    153 my $num_files = 0;
    154 foreach my $file (@$file_list) {
    155     # check whether this file rule refers to an image, mask, or variance fits image
    156     my $file_rule = $file->{file_rule};
    157     my $image_type = get_image_type($stage, $file_rule);
    158 
    159     # if this is an image and we are building a clean bundle or if quality is bad skip this rule
    160     next if $image_type and ($clean or $poor_quality);
    161 
    162     # if magic is required, don't ship jpegs or binned fits images
    163     next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/)
    164             or ($file_rule =~ /.JPEG2/));
    165 
    166     if ($stage eq "diff") {
    167         next if $file_rule =~ /CONV/;
    168     }
    169 
    170     my $file_name = $file->{name};
    171     my $base = basename($file_name);
    172     my $path = $ipprc->file_resolve($file_name);
    173 
    174     if (!$path) {
    175         # skip this file if $poor_quality
    176         # this is for compatability with older runs which don't have the files list in the
    177         # config dump.
    178         # Once we give up on supporting that we can remove the next line. (If the file is in the list
    179         # it must exist)
    180         next if $poor_quality;
    181 
    182         &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR);
    183     }
    184 
    185     # open the file to make sure it exists (and to work around the failed mount phenomena)
    186     my $fh = open_with_retries($path);
    187     close $fh;
    188 
    189     # we need to pre-process the image before adding to the bundle. Save the path names.
    190     # the images will be created below
    191     $num_files++;
    192     if ($image_type && $nan_masked_pixels) {
    193         # save the
    194         if ($image_type eq 'image') {
    195             $image = $file_name;
    196         } elsif ($image_type eq 'mask') {
    197             $mask = $file_name;;
    198         } elsif ($image_type eq 'variance') {
    199             $variance = $file_name;;
    200         } else {
    201             &my_die("invalid image type found: $image_type", $dist_id, $component,
    202                        $PS_EXIT_PROG_ERROR);
    203         }
    204     } else {
    205         # create a symbolic link from the file in the nebulous repository
    206         # in the temporary directory
    207         symlink $path, "$tmpdir/$base";
    208     }
    209 }
    210 
    211 if ($nan_masked_pixels) {
    212     # One last check as to whether magic has been applied to the inputs
    213 
    214     # Note: the sql for disttool -pendingcomponent won't select a component that
    215     # requires magic and hasn't been magicked, but we check again here
    216 
    217     if (!($magicked or $no_magic)) {
    218         &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
    219     }
    220 
    221     &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
    222     &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
    223     &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
    224 
    225     my $class_id;
    226     # run streaksrelease to set masked pixels to NAN
    227     if ($stage eq "raw") {
    228         $class_id = $component;
    229         # we can use the chip mask because disttool demands that magic have been run
    230         # and so the camera mask and the chip mask are the same
    231         $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
    232         my $fh = open_with_retries($mask);
    233         close $fh;
    234     } elsif ($stage eq "chip") {
    235         $class_id = $component;
    236     }
    237 
    238     my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
    239     $command .= " -class_id $class_id" if $class_id;
    240     $command .= " -mask $mask" if $mask;
    241     $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask);
    242     $command .= " -weight $variance" if $variance;
    243     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    244         run(command => $command, verbose => $verbose);
    245     unless ($success) {
    246         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    247         &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
    248     }
    249 }
    250 
    251 
    252 my $file_name;
    253 my $bytes;
    254 my $md5sum;
    255 
    256 if ($num_files) {
    257     # create the tarfile
    258     # XXX TODO: create a file rule for the tar file name
    259     my $tbase = basename($path_base);
    260     $tbase .= ".$component" if $component;
    261     $file_name = "$tbase.tgz";
    262     my $tarfile = "$outdir/$file_name";
    263 
    264     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
     90my ($file_name, $bytes, $md5sum);
     91{
     92    my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id";
     93    $command .= " --results_file $rf_name";
     94    $command .= " --component $component --path_base $path_base --outdir $outdir";
     95    $command .= " --chip_path_base $chip_path_base" if $chip_path_base;
     96    $command .= " --state $run_state" if defined $run_state;
     97    $command .= " --data_state $data_state" if defined $data_state;
     98    $command .= " --poor_quality" if defined $poor_quality;
     99    $command .= " --no_magic" if defined $no_magic;
     100    $command .= " --magicked" if defined $magicked;
     101    $command .= " --alt_path_base $alt_path_base" if defined $alt_path_base;
     102    $command .= " --clean" if defined $clean;
     103    $command .= " --save-temps" if defined $save_temps;
     104    $command .= " --dbname $dbname" if defined $dbname;
     105    $command .= " --verbose" if defined $verbose;
    265106
    266107    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    268109    unless ($success) {
    269110        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    270         &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
     111        &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code);
    271112    }
    272113
    273     # tell the module not to die on error
    274     $Digest::MD5::File::NOFATALS = 1;
    275     $md5sum = file_md5_hex($tarfile);
    276     &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
     114    open $rf, "<$rf_name" or &my_die("failed to open results file $rf_name", $dist_id, $component, $outdir, $PS_EXIT_UNKNOWN_ERROR);
    277115
    278     $bytes = -s $tarfile;
     116    my @lines = (<$rf>);
    279117
    280     delete_tmpdir($tmpdir);
    281 } else {
    282     # no files for this component
    283     $file_name = "none";
    284     $bytes = 0;
    285     $md5sum = "0";
     118    my $metadata = $mdcParser->parse (join "", @lines) or
     119        &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR);
     120
     121    my $results = parse_md_list($metadata);
     122
     123    if ((scalar @$results) != 1) {
     124        my $n = scalar @$results;
     125        &my_die("Unexected number of results from dist_make_bundle.pl: $n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR);
     126    }
     127    my $result = $results->[0];
     128
     129    $file_name = $result->{name};
     130    &my_die("undefined file name from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $file_name;
     131
     132    $bytes = $result->{bytes};
     133    &my_die("undefined file size from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $bytes;
     134
     135    $md5sum = $result->{md5sum};
     136    &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum;
    286137}
     138
    287139{
    288     my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component";
     140    my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir";
    289141    $command .= " -name $file_name -bytes $bytes -md5sum $md5sum";
    290142    $command .= " -dbname $dbname" if defined $dbname;
    … …  
    295147        unless ($success) {
    296148            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    297             &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
     149            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code);
    298150        }
    299151    } else {
    … …  
    305157
    306158### Pau.
    307 
    308 # return the image type (image, mask, or variance) if this file rule refers to
    309 # one of the big fits files that are not included in a clean distribution
    310 sub get_image_type {
    311     my $stage = shift;
    312     my $rule = shift;
    313     my $type;
    314 
    315     if ($stage eq "raw") {
    316         if ($rule eq "RAW.IMAGE") {
    317             $type = 'image';
    318         }
    319     } elsif ($stage eq "chip") {
    320         $type = $chip_cleaned{$rule};
    321     } elsif ($stage eq "camera") {
    322         $type = $camera_cleaned{$rule};
    323     } elsif ($stage eq "fake") {
    324         $type = $fake_cleaned{$rule};
    325     } elsif ($stage eq "warp") {
    326         $type = $warp_cleaned{$rule};
    327     } elsif ($stage eq "diff") {
    328         $type = $diff_cleaned{$rule};
    329     } elsif ($stage eq "stack") {
    330         $type = $stack_cleaned{$rule};
    331     } else {
    332         &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    333     }
    334     return $type;
    335 }
    336 
    337 sub open_with_retries {
    338     my $name = shift;
    339    
    340     my $tries = 1;
    341     my $max_tries = 5;
    342     my $opened = 0;
    343     while (!$opened && ($tries <= $max_tries)) {
    344         $opened = open(IN, "<$name");
    345         if (!$opened) {
    346             print STDERR "WARNING failed to open $name re-try $tries\n";
    347             $tries++;
    348             sleep 5;
    349         }
    350     }
    351    
    352     &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
    353                     $PS_EXIT_DATA_ERROR) if (!$opened);
    354 
    355     return *IN;
    356 }
    357 
    358 sub get_file_list {
    359     my $stage = shift;
    360     my $component = shift;
    361     my $path_base = shift;
    362     my $clean = shift;
    363 
    364     my @file_list;
    365     if ($stage eq "raw") {
    366         # XXX: TODO for now disttool sets path_base is set to the uri of the file
    367         # eventually rawImfile will have a path_base that we'll need to add '.fits'
    368         # XXX do we want to distribute the registration log files?
    369         if (!$clean) {
    370             my %file;
    371             $file{file_rule} = "RAW.IMAGE";
    372             $file{name} = $path_base;
    373             push @file_list, \%file;
    374         }
    375         return \@file_list;
    376     }
    377 
    378     # we get the list of output data products for this run from the config dump file that
    379     # is created when the run is done
    380     my $config_file_rule;
    381     if ($stage eq "chip") {
    382         $config_file_rule = "PPIMAGE.CONFIG";
    383     } elsif ($stage eq "camera") {
    384         $config_file_rule = "PSASTRO.CONFIG";
    385     } elsif ($stage eq 'fake') {
    386         # XXX: fake is a no op now return an emtpy list
    387         return \@file_list;
    388     } elsif ($stage eq "warp") {
    389         $config_file_rule = "PSWARP.CONFIG";
    390     } elsif ($stage eq "diff") {
    391         $config_file_rule = "PPSUB.CONFIG";
    392     } elsif ($stage eq "stack") {
    393         $config_file_rule = "PPSTACK.CONFIG";
    394     } else {
    395         &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    396     }
    397     my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
    398                 &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
    399                     $PS_EXIT_CONFIG_ERROR);
    400 
    401     my %config_file_hash;
    402 
    403     # add the configuration file to the list
    404     $config_file_hash{file_rule} = $config_file_rule;
    405     $config_file_hash{name} = $config_file;
    406     push @file_list, \%config_file_hash;
    407 
    408     my $resolved = $ipprc->file_resolve($config_file);
    409 
    410     &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
    411                     $PS_EXIT_CONFIG_ERROR) if (!$resolved);
    412 
    413     # we don't use the mdc parser because the perl parser is way is too slow for complicated config
    414     # files like this
    415     my $in = open_with_retries($resolved);
    416 
    417     my $line;
    418     while ($line = <$in>) {
    419         if ($line =~ /FILES.OUTPUT/) {
    420             # print "found FILES.OUTPUT\n";
    421             last;
    422         }
    423     }
    424     &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component,
    425                     $PS_EXIT_CONFIG_ERROR) if (!$line);
    426 
    427     while ($line = <$in>) {
    428         chomp $line;
    429         my ($key, $type, $val) = split " ", $line;
    430         # skip blank lines
    431         next if !$key;
    432         # we're done when we find END
    433         last if $key eq "END";
    434         # skip multi and other lines
    435         next if ($type ne "STR");
    436 
    437         # printf "%-32.32s   %s\n", $key, $val;
    438 
    439         &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
    440                 $PS_EXIT_CONFIG_ERROR) if (!$val);
    441 
    442         my %file;
    443         $file{file_rule} = $key;
    444         $file{name} = $val;
    445         push @file_list, \%file;
    446     }
    447     close $in;
    448 
    449     return \@file_list;
    450 }
    451 
    452 
    453 
    454 
    455 sub file_check
    456 {
    457     my $file = shift;           # Name of file
    458     &my_die("Unable to find output file: $file", $dist_id, $component, $PS_EXIT_SYS_ERROR) unless
    459         $ipprc->file_exists($file);
    460 }
    461 
    462159sub my_die
    463160{
    464161    my $msg = shift;            # Warning message on die
    465     my $dist_id = shift;    # Magic DS identifier
    466     my $component = shift;      # class_id or skycell_id
     162    my $dist_id = shift;        # distRun.dist_id
     163    my $component = shift;      # class_id, skycell_id, or exposure
     164    my $outdir = shift;         # output directory
    467165    my $exit_code = shift;      # Exit code to add
    468166
    … …  
    472170    $command   .= " -dist_id $dist_id";
    473171    $command   .= " -component $component";
     172    $command   .= " -outdir $outdir";
    474173    $command   .= " -fault $exit_code";
    475174    $command   .= " -dbname $dbname" if defined $dbname;
    … …  
    486185    }
    487186
    488     delete_tmpdir();
    489 
    490187    carp($msg);
    491188    exit $exit_code;
    492189}
    493190
    494 sub delete_tmpdir
    495 {
    496     if (!$save_temps and $tmpdir and -e $tmpdir) {
    497         system "rm -r $tmpdir";
    498     }
    499 }
    500 
    501191__END__
Note: See TracChangeset for help on using the changeset viewer.