Index: trunk/ippScripts/scripts/dist_component.pl =================================================================== --- trunk/ippScripts/scripts/dist_component.pl (revision 23693) +++ trunk/ippScripts/scripts/dist_component.pl (revision 23702) @@ -27,241 +27,4 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); - -# Look for programs we need -my $missing_tools; -my $disttool = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1); -my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Parse the command-line arguments -my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean); -my ($outroot, $run_state, $data_state, $magicked); -my ($dbname, $save_temps, $verbose, $no_update, $logfile); - -GetOptions( - 'dist_id=s' => \$dist_id, # distribution run identifier - 'camera=s' => \$camera, # camera for evaluating file rules - 'stage=s' => \$stage, # raw, chip, warp, or diff - 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id - 'component=s' => \$component, # the class_id or skycell_id - 'path_base=s' => \$path_base, # path_base of the input - 'chip_path_base=s'=> \$chip_path_base, # path base for camera stage (to enable us to find the mask filefor raw images) - 'state=s' => \$run_state, # state of the run - 'data_state=s' => \$data_state, # data_state for this component - 'magicked' => \$magicked, # magicked state for this component - 'outroot=s' => \$outroot, # "directory" for outputs - 'clean' => \$clean, # create clean distribution - 'save-temps' => \$save_temps, # Save temporary files? - 'dbname=s' => \$dbname, # Database name - 'verbose' => \$verbose, # Print stuff? - 'no-update' => \$no_update, # Don't update the database? - 'logfile=s' => \$logfile, - ) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outroot", - -exitval => 3) unless - defined $dist_id and - defined $camera and - defined $stage and - defined $stage_id and - defined $component and - defined $path_base and - defined $outroot; - -$ipprc->redirect_output($logfile) if $logfile; - -if (($stage eq 'raw') and !defined $chip_path_base) { - pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3); -} - -$ipprc->define_camera($camera); - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -# making a clean bundle of raw images doesn't make sense -if (($stage eq "raw") and $clean) { - # well I suppose we could ship the registration log file. naw. - &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); -} - -# create the output directories if it is not a nebulous path and it doesn't exist -if (index($outroot, "neb://") != 0) { - if (! -e $outroot ) { - my $code = system "mkdir -p $outroot"; - &my_die("cannot create output directory $outroot", $dist_id, $component, - $code >> 8) if $code; - } -} - -my $file_list = get_file_list($stage, $component, $path_base); - -# we die in get_file_list if something goes wrong. -# If the file list is empty it is an error because we should at least get a config dump file - -&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar keys %$file_list); - -# set up directory for temporary files - -my $tmpdir = "$outroot/tmpdir.$dist_id.$component.$$"; -if (-e $tmpdir) { - if (-d $tmpdir) { - my $rc = system "rm -r $tmpdir"; - &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc; - } else { - unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR); - } -} -mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component, - $PS_EXIT_UNKNOWN_ERROR); - -# XXX get this from camera and a recipe file -# or this with the quality flag -my $need_to_nan_masked = !$clean && ($camera eq "GPC1"); - -my ($image, $mask, $variance); - -my @base_list; -foreach my $file_rule (keys %$file_list) { - # if this is an image, mask, or variance image and we are building a clean distribution bundle, - # skip it - my $image_type = get_image_type($stage, $file_rule); - next if $clean && $image_type; - - my $file_name = $file_list->{$file_rule}; - my $base = basename($file_name); - my $path = $ipprc->file_resolve($file_name); - - &my_die("failed to resolve $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR) if !$path; - - # open the file to make sure it exists (and to work around the failed mount phenomena) - my $fh = open_with_retries($file_name); - close $fh; - - if ($image_type && $need_to_nan_masked) { - # save the - if ($image_type eq 'image') { - $image = $path - } elsif ($image_type eq 'mask') { - $mask = $path; - } elsif ($image_type eq 'variance') { - $variance = $path; - } else { - &my_die("invalid image type found: $image_type", $dist_id, $component, - $PS_EXIT_PROG_ERROR); - } - } else { - # just symlink the existing file into the temporary directory - symlink $path, "$tmpdir/$base"; - } - push @base_list, $base; -} - -my $run_streaksrelease; -if ($need_to_nan_masked) { - # make sure that the needed images exist. - # XXX: they won't be found for a warp or diff with bad quality - # we probably should be passing in the component's quality flag - if (0) { - if (($stage ne "warp") and ($stage ne "diff")) { - &my_die("no image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image; - &my_die("no mask image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask; - &my_die("no variance image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance; - } - } - $run_streaksrelease = 1 if $image; -} - -if ($run_streaksrelease) { - # Here is where we determine whether or not a GPC1 image can be released. - # XXX: actually the sql for disttool -pendingcomponent won't pass a component that - # requires magic and hasn't been magicked, but we check again here - # XXX: get this from a recipe - - if ($camera eq "GPC1" and !$magicked) { - &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR); - } - - my $class_id; - # run streaksrelease to set masked pixels to NAN - if ($stage eq "raw") { - $class_id = $component; - # we can use the chip mask because disttool demands that magic have been run - # and so the camera mask and the chip mask are the same - $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component); - my $fh = open_with_retries($mask); - close $fh; - } elsif ($stage eq "chip") { - $class_id = $component; - } - - my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir"; - $command .= " -class_id $class_id" if $class_id; - $command .= " -mask $mask" if $mask; - $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask); - $command .= " -weight $variance" if $variance; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); - } -} - - -my $file_name; -my $bytes; -my $md5sum; -# create the tarfile -{ - # XXX TODO: create a file rule for the tar file name - my $tbase = basename($path_base); - $tbase .= ".$component" if $component; - $file_name = "$tbase.tgz"; - my $tarfile = "$outroot/$file_name"; - - my $command = "tar -C $tmpdir -czhf $tarfile ."; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); - } - - # tell the module not to die on error - $Digest::MD5::File::NOFATALS = 1; - $md5sum = file_md5_hex($tarfile); - &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum; - - my @finfo = stat($tarfile); - &my_die("unable to stat $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !@finfo; - $bytes = $finfo[7]; - - delete_tmpdir($tmpdir); -} -{ - my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component"; - $command .= " -name $file_name -bytes $bytes -md5sum $md5sum"; - $command .= " -dbname $dbname" if defined $dbname; - - unless (defined $no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); - } - } else { - print "skipping command $command\n"; - } -} - -exit 0; - -### Pau. # filerules that are not included in 'cleaned' distribution bundles @@ -283,4 +46,242 @@ 'PPSTACK.OUTPUT.VARIANCE' => 'variance' ); + +# Look for programs we need +my $missing_tools; +my $disttool = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1); +my $streaksrelease = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean); +my ($outroot, $run_state, $data_state, $magicked, $no_magic, $no_images); +my ($dbname, $save_temps, $verbose, $no_update, $logfile); + +GetOptions( + 'dist_id=s' => \$dist_id, # distribution run identifier + 'camera=s' => \$camera, # camera for evaluating file rules + 'stage=s' => \$stage, # raw, chip, warp, or diff + 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id + 'component=s' => \$component, # the class_id or skycell_id + 'path_base=s' => \$path_base, # path_base of the input + 'chip_path_base=s'=> \$chip_path_base, # path base for camera stage (to enable us to find the mask filefor raw images) + 'state=s' => \$run_state, # state of the run + 'data_state=s' => \$data_state, # data_state for this component + 'no_images' => \$no_images, # the processing for this component did not produced images + 'no_magic' => \$no_magic, # magic is not required for this distribution run + 'magicked' => \$magicked, # magicked state for this component + 'outroot=s' => \$outroot, # "directory" for outputs + 'clean' => \$clean, # create clean distribution + 'save-temps' => \$save_temps, # Save temporary files? + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'logfile=s' => \$logfile, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outroot", + -exitval => 3) unless + defined $dist_id and + defined $camera and + defined $stage and + defined $stage_id and + defined $component and + defined $path_base and + defined $outroot; + +$ipprc->redirect_output($logfile) if $logfile; + +if (($stage eq 'raw') and !defined $chip_path_base) { + pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3); +} + +$ipprc->define_camera($camera); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# making a clean bundle of raw images doesn't make sense +if (($stage eq "raw") and $clean) { + # well I suppose we could ship the registration log file. naw. + &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR); +} + +# create the output directories if it is not a nebulous path and it doesn't exist +if (index($outroot, "neb://") != 0) { + if (! -e $outroot ) { + my $code = system "mkdir -p $outroot"; + &my_die("cannot create output directory $outroot", $dist_id, $component, + $code >> 8) if $code; + } +} + +# Get the list of data products for this component +# note: We my_die in get_file_list if something goes wrong. +my $file_list = get_file_list($stage, $component, $path_base); + +# If the file list is empty it is an error because we should at least get a config dump file +# XXX: is this true? what about fake? + +&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar keys %$file_list); + +# set up directory for temporary files + +my $tmpdir = "$outroot/tmpdir.$dist_id.$component"; +if (-e $tmpdir) { + if (-d $tmpdir) { + my $rc = system "rm -r $tmpdir"; + &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc; + } else { + unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR); + } +} +mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component, + $PS_EXIT_UNKNOWN_ERROR); + +# +# we need to run set masked pixels to NAN in the image and variance images +# unless +# 1. we are building a clean distribution bundle +# 2. magic is not required for this distRun +# 3. the processing for the component produced no images (warp or diff with bad quality for example) +my $nan_masked_pixels = ! ($clean || $no_magic || $no_images); + +my ($image, $mask, $variance); + +my @base_list; +foreach my $file_rule (keys %$file_list) { + # check whether this file rule refers to an image, mask, or variance fits image + my $image_type = get_image_type($stage, $file_rule); + + # if we are building a clean bundle skip this rule + next if $clean && $image_type; + + my $file_name = $file_list->{$file_rule}; + my $base = basename($file_name); + my $path = $ipprc->file_resolve($file_name); + + &my_die("failed to resolve $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR) if !$path; + + # open the file to make sure it exists (and to work around the failed mount phenomena) + my $fh = open_with_retries($path); + close $fh; + + # we need to pre-process the image before adding to the bundle. Save the path names. + if ($image_type && $nan_masked_pixels) { + # save the + if ($image_type eq 'image') { + $image = $path + } elsif ($image_type eq 'mask') { + $mask = $path; + } elsif ($image_type eq 'variance') { + $variance = $path; + } else { + &my_die("invalid image type found: $image_type", $dist_id, $component, + $PS_EXIT_PROG_ERROR); + } + } else { + # create a symbolic link from the file in the nebulous repositorythe into the temporary directory + symlink $path, "$tmpdir/$base"; + } + push @base_list, $base; +} + +if ($nan_masked_pixels) { + # One last check as to whether magic has been applied to the inputs + + # Note: the sql for disttool -pendingcomponent won't select a component that + # requires magic and hasn't been magicked, but we check again here + + if (!($magicked or $no_magic)) { + &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR); + } + + &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image; + &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask; + &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance; + + + my $class_id; + # run streaksrelease to set masked pixels to NAN + if ($stage eq "raw") { + $class_id = $component; + # we can use the chip mask because disttool demands that magic have been run + # and so the camera mask and the chip mask are the same + $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component); + my $fh = open_with_retries($mask); + close $fh; + } elsif ($stage eq "chip") { + $class_id = $component; + } + + my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir"; + $command .= " -class_id $class_id" if $class_id; + $command .= " -mask $mask" if $mask; + $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask); + $command .= " -weight $variance" if $variance; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); + } +} + + +my $file_name; +my $bytes; +my $md5sum; +# create the tarfile +{ + # XXX TODO: create a file rule for the tar file name + my $tbase = basename($path_base); + $tbase .= ".$component" if $component; + $file_name = "$tbase.tgz"; + my $tarfile = "$outroot/$file_name"; + + my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile ."; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); + } + + # tell the module not to die on error + $Digest::MD5::File::NOFATALS = 1; + $md5sum = file_md5_hex($tarfile); + &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum; + + my @finfo = stat($tarfile); + &my_die("unable to stat $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !@finfo; + $bytes = $finfo[7]; + + delete_tmpdir($tmpdir); +} +{ + my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component"; + $command .= " -name $file_name -bytes $bytes -md5sum $md5sum"; + $command .= " -dbname $dbname" if defined $dbname; + + unless (defined $no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code); + } + } else { + print "skipping command $command\n"; + } +} + +exit 0; + +### Pau. + # return the image type (image, mask, or variance) if this file rule refers to # one of the big fits files that are not included in a clean distribution @@ -340,6 +341,7 @@ my %file_list; if ($stage eq "raw") { - # XXX: TODO for now path_base is set to the uri of the file - # eventually we'll have a path_base that we'll need to resolve + # XXX: TODO for now disttool sets path_base is set to the uri of the file + # eventually rawImfile will have a path_base that we'll need to add '.fits' + # XXX do we want to distribute the registration log files? $file_list{"RAW.IMAGE"} = $path_base; return \%file_list; @@ -402,5 +404,5 @@ $PS_EXIT_CONFIG_ERROR) if (!$val); - $file_list{key} = $val; + $file_list{$key} = $val; } close $in; @@ -511,5 +513,5 @@ STAGE STR RAW # there is isn't really a file rule for raw images - DUMMY BOOL F + RAW.IMAGE BOOL F END @@ -518,7 +520,7 @@ STAGE STR CHIP PPIMAGE.CONFIG BOOL T -# PPIMAGE.CHIP BOOL F -# PPIMAGE.CHIP.MASK BOOL F -# PPIMAGE.CHIP.VARIANCE BOOL F + PPIMAGE.CHIP BOOL F + PPIMAGE.CHIP.MASK BOOL F + PPIMAGE.CHIP.VARIANCE BOOL F PPIMAGE.BIN1 BOOL T PPIMAGE.BIN2 BOOL T @@ -544,9 +546,9 @@ PROD_LIST METADATA STAGE STR CAM_CHIP -# PSASTRO.OUTPUT.MASK BOOL F + PSASTRO.OUTPUT.MASK BOOL F END PROD_LIST METADATA STAGE STR FAKE -# PPSIM.OUTPUT BOOL F + PPSIM.OUTPUT BOOL F END # list of data products for a gpc1 warpSkyfile (pswarp) @@ -554,7 +556,7 @@ STAGE STR WARP PSWARP.CONFIG BOOL T -# PSWARP.OUTPUT BOOL F -# PSWARP.OUTPUT.MASK BOOL F -# PSWARP.OUTPUT.VARIANCE BOOL F + PSWARP.OUTPUT BOOL F + PSWARP.OUTPUT.MASK BOOL F + PSWARP.OUTPUT.VARIANCE BOOL F PSWARP.OUTPUT.SOURCES BOOL T PSPHOT.BACKMDL.MEF BOOL T @@ -569,7 +571,7 @@ STAGE STR DIFF PPSUB.CONFIG BOOL T -# PPSUB.OUTPUT BOOL F -# PPSUB.OUTPUT.MASK BOOL F -# PPSUB.OUTPUT.VARIANCE BOOL F + PPSUB.OUTPUT BOOL F + PPSUB.OUTPUT.MASK BOOL F + PPSUB.OUTPUT.VARIANCE BOOL F PPSUB.OUTPUT.KERNELS BOOL T PPSUB.OUTPUT.JPEG1 BOOL T @@ -584,7 +586,7 @@ STAGE STR STACK PPSTACK.CONFIG BOOL T -# PPSTACK.OUTPUT BOOL F -# PPSTACK.OUTPUT.MASK BOOL F -# PPSTACK.OUTPUT.VARIANCE BOOL F + PPSTACK.OUTPUT BOOL F + PPSTACK.OUTPUT.MASK BOOL F + PPSTACK.OUTPUT.VARIANCE BOOL F PPSTACK.TARGET.PSF BOOL T PSPHOT.OUT.CMF.MEF BOOL T