Index: trunk/ippScripts/scripts/dist_advancerun.pl =================================================================== --- trunk/ippScripts/scripts/dist_advancerun.pl (revision 30412) +++ trunk/ippScripts/scripts/dist_advancerun.pl (revision 30489) @@ -23,5 +23,4 @@ use PS::IPP::Config 1.01 qw( :standard ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -29,9 +28,10 @@ # Parse the command-line arguments -my ($dist_id, $stage, $stage_id, $outdir, $clean); +my ($dist_id, $stage, $stage_id, $outdir, $clean, $camera); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); GetOptions( - 'dist_id=s' => \$dist_id,# Magic destreak run identifier + 'dist_id=s' => \$dist_id,# Magic destreak run identifier + 'camera=s' => \$camera, 'stage=s' => \$stage, # raw, chip, warp, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id @@ -53,4 +53,6 @@ defined $stage_id and defined $outdir; + +my $ipprc = PS::IPP::Config->new($camera); # IPP configuration $ipprc->redirect_output($logfile) if $logfile; @@ -134,9 +136,46 @@ } -# XXX should we create a file rule for this? -my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc"; - -{ - my $command = "$tool_cmd $exportarg -outfile $dbinfo_file"; + +# work around the fact that $ipprc->file_create does not actually create a file on disk +# unless the scheme is nebulous +sub create_file { + my $rule = shift; + my $path_base = shift; + my $ref = shift; + + my $file; + + my $error; + $file = $ipprc->prepare_output($rule, $path_base, undef, 1, \$error) + or &my_die("Unable to prepare outut for $rule", $dist_id, $PS_EXIT_SYS_ERROR); + + my $scheme = file_scheme($file); + $scheme = "" if !$scheme; + + my $resolved; + if ($scheme) { + $ipprc->file_create($file) + or &my_die("Unable to create $file", $dist_id, $PS_EXIT_SYS_ERROR); + + $resolved = $ipprc->file_resolve($file) + or &my_die("Unable to resolve $file", $dist_id, $PS_EXIT_SYS_ERROR); + + if ($scheme eq 'neb') { + &my_die("$resolved not found", $dist_id, $PS_EXIT_SYS_ERROR) unless ($resolved and -e $resolved); + } + } else { + $resolved = $file; + } + $$ref = $resolved; + + return $file +} + +my $dbinfo_root = "$outdir/dbinfo.$stage.$stage_id"; +my $resolved; +my $dbinfo_file = create_file("DIST.OUTPUT.DBINFO", $dbinfo_root, \$resolved); + +{ + my $command = "$tool_cmd $exportarg -outfile $resolved"; $command .= " -clean" if ((defined $clean) and ($stage ne "raw")); $command .= " -dbname $dbname" if defined $dbname; @@ -149,5 +188,8 @@ } } -my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc"; + +my $dirinfo_root = "$outdir/dirinfo.$stage.$stage_id"; +my $dirinfo = create_file("DIST.OUTPUT.DIRINFO", $dirinfo_root, \$resolved); + { my $command = "$tool_cmd $list_mode"; @@ -168,6 +210,6 @@ &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR); - open MANIFEST, ">$dirinfo" or - &my_die("Unable to open dirinfo file $dirinfo", $dist_id, $PS_EXIT_UNKNOWN_ERROR); + open MANIFEST, ">$resolved" or + &my_die("Unable to open dirinfo file $resolved", $dist_id, $PS_EXIT_UNKNOWN_ERROR); my $destdir; @@ -197,5 +239,4 @@ &my_die("unable to find path", $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path; my $component_dir = find_componentdir($destdir, $path); -# print MANIFEST "$component METADATA\n"; print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n"; } @@ -217,5 +258,4 @@ } } - exit 0; Index: trunk/ippScripts/scripts/dist_bundle.pl =================================================================== --- trunk/ippScripts/scripts/dist_bundle.pl (revision 30412) +++ trunk/ippScripts/scripts/dist_bundle.pl (revision 30489) @@ -16,5 +16,5 @@ use IPC::Cmd 0.36 qw( can_run run ); -use File::Temp qw( tempfile ); +use File::Temp qw( tempfile tempdir ); use File::Basename qw( basename ); use Digest::MD5::File qw( file_md5_hex ); @@ -71,5 +71,5 @@ # Parse the command-line arguments my ($camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $alt_path_base, $clean); -my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix); +my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -88,5 +88,5 @@ 'magicked' => \$magicked, # magicked state for this component 'alt_path_base=s'=> \$alt_path_base, # path to alternate inputs - 'outdir=s' => \$outdir, # "directory" for outputs + 'outroot=s' => \$outroot, # outroot 'prefix=s' => \$prefix, # "prefix" to apply to filenames 'clean' => \$clean, # create clean distribution @@ -99,5 +99,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outdir", +pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outroot", -exitval => 3) unless defined $camera and @@ -106,5 +106,5 @@ defined $component and defined $path_base and - defined $outdir; + defined $outroot; $ipprc->redirect_output($logfile) if $logfile; @@ -116,12 +116,5 @@ $ipprc->define_camera($camera); -# create the output directories if it is not a nebulous path and it doesn't exist -if (index($outdir, "neb://") != 0) { - if (! -e $outdir ) { - my $code = system "mkdir -p $outdir"; - &my_die("cannot create output directory $outdir", $component, - $code >> 8) if $code; - } -} +$ipprc->outroot_prepare($outroot); # Get the list of data products for this component @@ -138,15 +131,9 @@ # set up directory for temporary files -my $tmpdir = "$outdir/tmpdir.$component.$$"; -if (-e $tmpdir) { - if (-d $tmpdir) { - my $rc = system "rm -r $tmpdir"; - &my_die("cannot rm $tmpdir return code: $rc", $component, $PS_EXIT_UNKNOWN_ERROR) if $rc; - } else { - unlink $tmpdir or &my_die("cannot delete $tmpdir", $component, $PS_EXIT_UNKNOWN_ERROR); - } -} -mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $component, - $PS_EXIT_UNKNOWN_ERROR); +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); +$temproot = "/tmp" if !defined $temproot; +&my_die("directory for TEMP.DIR $temproot does not exist", $component, $PS_EXIT_CONFIG_ERROR) if ! -e $temproot; + +my $tmpdir = tempdir("$temproot/dist.XXXX", CLEANUP => !$save_temps); # @@ -349,9 +336,36 @@ if ($num_files) { # create the tarfile - # XXX TODO: create a file rule for the tar file name - my $tbase = basename($path_base); - $tbase .= ".$component" if $component; - $file_name = ($prefix ? $prefix : "") . "$tbase.tgz"; - my $tarfile = "$outdir/$file_name"; +# my $tbase = basename($path_base); +# $tbase .= ".$component" if $component; +# $file_name = ($prefix ? $prefix : "") . "$tbase.tgz"; + my $error; + my $output_tarfile; + my $rule; + if ($stage eq 'chip' or $stage eq 'chip_bg' or $stage eq 'raw') { + $rule = "DIST.OUTPUT.CHIP.BUNDLE"; + } else { + $rule = "DIST.OUTPUT.BUNDLE"; + } + $output_tarfile = $ipprc->prepare_output($rule, $outroot, $component, 1, \$error) + or &my_die("Failed to prepare output tarfile: $error", $component, $error); + $file_name = basename($output_tarfile); + + + my $scheme = file_scheme($output_tarfile); + + my $tarfile; + if ($scheme) { + $ipprc->file_create($output_tarfile) + or &my_die("Failed to create $output_tarfile: $error", $component, $error); + $tarfile = $ipprc->file_resolve($output_tarfile); + + if ($scheme eq 'neb') { + &my_die("output file $output_tarfile not found", $component, $PS_EXIT_SYS_ERROR) + unless ($tarfile and -e $tarfile); + } + } else { + # no scheme the filename and the resolved filename are the same + $tarfile = $output_tarfile; + } my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile ."; Index: trunk/ippScripts/scripts/dist_component.pl =================================================================== --- trunk/ippScripts/scripts/dist_component.pl (revision 30412) +++ trunk/ippScripts/scripts/dist_component.pl (revision 30489) @@ -21,8 +21,5 @@ use PS::IPP::Metadata::Config; use PS::IPP::Metadata::List qw( parse_md_list ); - use PS::IPP::Config 1.01 qw( :standard ); - -my $ipprc = PS::IPP::Config->new(); # IPP configuration use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -77,5 +74,10 @@ defined $outdir; +my $ipprc = PS::IPP::Config->new($camera); # IPP configuration $ipprc->redirect_output($logfile) if $logfile; + +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); + +$temproot = "/tmp" if !defined $temproot; if (($stage eq 'raw') and !$clean and !defined $chip_path_base) { @@ -85,6 +87,9 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps); +my ($rf, $rf_name) = tempfile("$temproot/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps); close $rf; + +my $basename = basename($path_base); +my $outroot = "$outdir/$basename"; my ($file_name, $bytes, $md5sum); @@ -92,5 +97,5 @@ my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id"; $command .= " --results_file $rf_name"; - $command .= " --component $component --path_base $path_base --outdir $outdir"; + $command .= " --component $component --path_base $path_base --outroot $outroot"; $command .= " --chip_path_base $chip_path_base" if $chip_path_base; $command .= " --state $run_state" if defined $run_state; @@ -136,4 +141,5 @@ &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum; } + { Index: trunk/ippScripts/scripts/dist_defineruns.pl =================================================================== --- trunk/ippScripts/scripts/dist_defineruns.pl (revision 30412) +++ trunk/ippScripts/scripts/dist_defineruns.pl (revision 30489) @@ -42,5 +42,5 @@ # Parse the command-line arguments -my ($stage, $stage_limit, $dist_root, $no_magic); +my ($stage, $stage_limit, $dist_root, $workdir, $no_magic); my ($dbname, $save_temps, $verbose, $no_update, $logfile); my @labels; @@ -49,6 +49,7 @@ 'stage=s' => \$stage, # stage to queue 'label=s' => \@labels, # labels + 'workdir=s' => \$workdir, # workdir 'stage_limit=s' => \$stage_limit,# maximum number of runs queued for each stage - 'dist_root=s' => \$dist_root, # root of distribution work area +# 'dist_root=s' => \$dist_root, # root of distribution work area 'no_magic' => \$no_magic, # queue runs without requiring magic (for testing only) 'dbname=s' => \$dbname, # Database name @@ -66,15 +67,19 @@ $ipprc->redirect_output($logfile) if $logfile; -if (!$dist_root) { - $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT"); - &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root; +if (!$workdir) { + print "workdir not supplied will use DISTRIBUTION_ROOT\n"; + # old method where we set workdir based on a config file + if (!$dist_root) { + $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT"); + &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root; + } + + my ($day, $month, $year) = (gmtime)[3,4,5]; + my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day; + + $workdir = $dist_root . "/$datestr"; + + print "workdir is $workdir\n"; } - -my ($day, $month, $year) = (gmtime)[3,4,5]; -my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day; - -my $workdir = "$dist_root/$datestr"; - -print "workdir is $workdir\n"; # if stage is not supplied as an argument, loop over all stages @@ -90,5 +95,5 @@ my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label"; $command .= " -no_magic" if $no_magic; - $command .= " -dry_run" if $no_update; + $command .= " -pretend" if $no_update; $command .= " -limit $stage_limit" if $stage_limit; $command .= " -set_label $label"; Index: trunk/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- trunk/ippScripts/scripts/dist_make_fileset.pl (revision 30412) +++ trunk/ippScripts/scripts/dist_make_fileset.pl (revision 30489) @@ -96,5 +96,5 @@ # make sure that the database info file for this run exists my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc"; -if (! -e "$dbinfo_file" ) { +if (! $ipprc->file_exists($dbinfo_file) ) { &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -103,5 +103,5 @@ # make sure that the dirinfo file for this run exists my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc"; -if (! -e "$dirinfo_file" ) { +if (!$ipprc->file_exists($dirinfo_file)) { &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -277,9 +277,13 @@ $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - print STDERR "Unable to perform $command error_code: $error_code\n"; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR "Unable to perform $command error_code: $error_code\n"; + } + } else { + print STDERR "skipping $command\n"; } exit $fault;