Index: /tags/ipp-20101215/DataStoreServer/scripts/dsreg =================================================================== --- /tags/ipp-20101215/DataStoreServer/scripts/dsreg (revision 30489) +++ /tags/ipp-20101215/DataStoreServer/scripts/dsreg (revision 30490) @@ -338,8 +338,13 @@ ## then copy or symlink the files into place foreach my $ref (@files) { - my $src = (defined $abspath ? '' : "$datapath/") . "$ref->{file}"; + my $src_path = (defined $abspath ? '' : "$datapath/") . "$ref->{file}"; my $filename = fileparse($ref->{file}, ()); my $dest = "$fileset_dir/$filename"; + my $src = $ipprc->file_resolve($src_path); + if (!$src) { + print STDERR "failed to resolve file $src_path\n"; + exit $PS_EXIT_SYS_ERROR; + } if ( !-e $src or !-r $src) { print STDERR "source file $src does not exist or is not accessible\n"; Index: /tags/ipp-20101215/ippScripts/scripts/dist_advancerun.pl =================================================================== --- /tags/ipp-20101215/ippScripts/scripts/dist_advancerun.pl (revision 30489) +++ /tags/ipp-20101215/ippScripts/scripts/dist_advancerun.pl (revision 30490) @@ -23,5 +23,4 @@ use PS::IPP::Config 1.01 qw( :standard ); -my $ipprc = PS::IPP::Config->new(); # IPP configuration use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -29,9 +28,10 @@ # Parse the command-line arguments -my ($dist_id, $stage, $stage_id, $outdir, $clean); +my ($dist_id, $stage, $stage_id, $outdir, $clean, $camera); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); GetOptions( - 'dist_id=s' => \$dist_id,# Magic destreak run identifier + 'dist_id=s' => \$dist_id,# Magic destreak run identifier + 'camera=s' => \$camera, 'stage=s' => \$stage, # raw, chip, warp, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id @@ -53,4 +53,6 @@ defined $stage_id and defined $outdir; + +my $ipprc = PS::IPP::Config->new($camera); # IPP configuration $ipprc->redirect_output($logfile) if $logfile; @@ -134,9 +136,46 @@ } -# XXX should we create a file rule for this? -my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc"; - -{ - my $command = "$tool_cmd $exportarg -outfile $dbinfo_file"; + +# work around the fact that $ipprc->file_create does not actually create a file on disk +# unless the scheme is nebulous +sub create_file { + my $rule = shift; + my $path_base = shift; + my $ref = shift; + + my $file; + + my $error; + $file = $ipprc->prepare_output($rule, $path_base, undef, 1, \$error) + or &my_die("Unable to prepare outut for $rule", $dist_id, $PS_EXIT_SYS_ERROR); + + my $scheme = file_scheme($file); + $scheme = "" if !$scheme; + + my $resolved; + if ($scheme) { + $ipprc->file_create($file) + or &my_die("Unable to create $file", $dist_id, $PS_EXIT_SYS_ERROR); + + $resolved = $ipprc->file_resolve($file) + or &my_die("Unable to resolve $file", $dist_id, $PS_EXIT_SYS_ERROR); + + if ($scheme eq 'neb') { + &my_die("$resolved not found", $dist_id, $PS_EXIT_SYS_ERROR) unless ($resolved and -e $resolved); + } + } else { + $resolved = $file; + } + $$ref = $resolved; + + return $file +} + +my $dbinfo_root = "$outdir/dbinfo.$stage.$stage_id"; +my $resolved; +my $dbinfo_file = create_file("DIST.OUTPUT.DBINFO", $dbinfo_root, \$resolved); + +{ + my $command = "$tool_cmd $exportarg -outfile $resolved"; $command .= " -clean" if ((defined $clean) and ($stage ne "raw")); $command .= " -dbname $dbname" if defined $dbname; @@ -149,5 +188,8 @@ } } -my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc"; + +my $dirinfo_root = "$outdir/dirinfo.$stage.$stage_id"; +my $dirinfo = create_file("DIST.OUTPUT.DIRINFO", $dirinfo_root, \$resolved); + { my $command = "$tool_cmd $list_mode"; @@ -168,6 +210,6 @@ &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR); - open MANIFEST, ">$dirinfo" or - &my_die("Unable to open dirinfo file $dirinfo", $dist_id, $PS_EXIT_UNKNOWN_ERROR); + open MANIFEST, ">$resolved" or + &my_die("Unable to open dirinfo file $resolved", $dist_id, $PS_EXIT_UNKNOWN_ERROR); my $destdir; @@ -197,5 +239,4 @@ &my_die("unable to find path", $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path; my $component_dir = find_componentdir($destdir, $path); -# print MANIFEST "$component METADATA\n"; print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n"; } @@ -217,5 +258,4 @@ } } - exit 0; Index: /tags/ipp-20101215/ippScripts/scripts/dist_bundle.pl =================================================================== --- /tags/ipp-20101215/ippScripts/scripts/dist_bundle.pl (revision 30489) +++ /tags/ipp-20101215/ippScripts/scripts/dist_bundle.pl (revision 30490) @@ -16,5 +16,5 @@ use IPC::Cmd 0.36 qw( can_run run ); -use File::Temp qw( tempfile ); +use File::Temp qw( tempfile tempdir ); use File::Basename qw( basename ); use Digest::MD5::File qw( file_md5_hex ); @@ -71,5 +71,5 @@ # Parse the command-line arguments my ($camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $alt_path_base, $clean); -my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix); +my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -88,5 +88,5 @@ 'magicked' => \$magicked, # magicked state for this component 'alt_path_base=s'=> \$alt_path_base, # path to alternate inputs - 'outdir=s' => \$outdir, # "directory" for outputs + 'outroot=s' => \$outroot, # outroot 'prefix=s' => \$prefix, # "prefix" to apply to filenames 'clean' => \$clean, # create clean distribution @@ -99,5 +99,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outdir", +pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outroot", -exitval => 3) unless defined $camera and @@ -106,5 +106,5 @@ defined $component and defined $path_base and - defined $outdir; + defined $outroot; $ipprc->redirect_output($logfile) if $logfile; @@ -116,12 +116,5 @@ $ipprc->define_camera($camera); -# create the output directories if it is not a nebulous path and it doesn't exist -if (index($outdir, "neb://") != 0) { - if (! -e $outdir ) { - my $code = system "mkdir -p $outdir"; - &my_die("cannot create output directory $outdir", $component, - $code >> 8) if $code; - } -} +$ipprc->outroot_prepare($outroot); # Get the list of data products for this component @@ -138,15 +131,9 @@ # set up directory for temporary files -my $tmpdir = "$outdir/tmpdir.$component.$$"; -if (-e $tmpdir) { - if (-d $tmpdir) { - my $rc = system "rm -r $tmpdir"; - &my_die("cannot rm $tmpdir return code: $rc", $component, $PS_EXIT_UNKNOWN_ERROR) if $rc; - } else { - unlink $tmpdir or &my_die("cannot delete $tmpdir", $component, $PS_EXIT_UNKNOWN_ERROR); - } -} -mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $component, - $PS_EXIT_UNKNOWN_ERROR); +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); +$temproot = "/tmp" if !defined $temproot; +&my_die("directory for TEMP.DIR $temproot does not exist", $component, $PS_EXIT_CONFIG_ERROR) if ! -e $temproot; + +my $tmpdir = tempdir("$temproot/dist.XXXX", CLEANUP => !$save_temps); # @@ -349,9 +336,36 @@ if ($num_files) { # create the tarfile - # XXX TODO: create a file rule for the tar file name - my $tbase = basename($path_base); - $tbase .= ".$component" if $component; - $file_name = ($prefix ? $prefix : "") . "$tbase.tgz"; - my $tarfile = "$outdir/$file_name"; +# my $tbase = basename($path_base); +# $tbase .= ".$component" if $component; +# $file_name = ($prefix ? $prefix : "") . "$tbase.tgz"; + my $error; + my $output_tarfile; + my $rule; + if ($stage eq 'chip' or $stage eq 'chip_bg' or $stage eq 'raw') { + $rule = "DIST.OUTPUT.CHIP.BUNDLE"; + } else { + $rule = "DIST.OUTPUT.BUNDLE"; + } + $output_tarfile = $ipprc->prepare_output($rule, $outroot, $component, 1, \$error) + or &my_die("Failed to prepare output tarfile: $error", $component, $error); + $file_name = basename($output_tarfile); + + + my $scheme = file_scheme($output_tarfile); + + my $tarfile; + if ($scheme) { + $ipprc->file_create($output_tarfile) + or &my_die("Failed to create $output_tarfile: $error", $component, $error); + $tarfile = $ipprc->file_resolve($output_tarfile); + + if ($scheme eq 'neb') { + &my_die("output file $output_tarfile not found", $component, $PS_EXIT_SYS_ERROR) + unless ($tarfile and -e $tarfile); + } + } else { + # no scheme the filename and the resolved filename are the same + $tarfile = $output_tarfile; + } my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile ."; Index: /tags/ipp-20101215/ippScripts/scripts/dist_component.pl =================================================================== --- /tags/ipp-20101215/ippScripts/scripts/dist_component.pl (revision 30489) +++ /tags/ipp-20101215/ippScripts/scripts/dist_component.pl (revision 30490) @@ -21,8 +21,5 @@ use PS::IPP::Metadata::Config; use PS::IPP::Metadata::List qw( parse_md_list ); - use PS::IPP::Config 1.01 qw( :standard ); - -my $ipprc = PS::IPP::Config->new(); # IPP configuration use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -77,5 +74,10 @@ defined $outdir; +my $ipprc = PS::IPP::Config->new($camera); # IPP configuration $ipprc->redirect_output($logfile) if $logfile; + +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); + +$temproot = "/tmp" if !defined $temproot; if (($stage eq 'raw') and !$clean and !defined $chip_path_base) { @@ -85,6 +87,9 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps); +my ($rf, $rf_name) = tempfile("$temproot/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps); close $rf; + +my $basename = basename($path_base); +my $outroot = "$outdir/$basename"; my ($file_name, $bytes, $md5sum); @@ -92,5 +97,5 @@ my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id"; $command .= " --results_file $rf_name"; - $command .= " --component $component --path_base $path_base --outdir $outdir"; + $command .= " --component $component --path_base $path_base --outroot $outroot"; $command .= " --chip_path_base $chip_path_base" if $chip_path_base; $command .= " --state $run_state" if defined $run_state; @@ -136,4 +141,5 @@ &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum; } + { Index: /tags/ipp-20101215/ippScripts/scripts/dist_defineruns.pl =================================================================== --- /tags/ipp-20101215/ippScripts/scripts/dist_defineruns.pl (revision 30489) +++ /tags/ipp-20101215/ippScripts/scripts/dist_defineruns.pl (revision 30490) @@ -42,5 +42,5 @@ # Parse the command-line arguments -my ($stage, $stage_limit, $dist_root, $no_magic); +my ($stage, $stage_limit, $dist_root, $workdir, $no_magic); my ($dbname, $save_temps, $verbose, $no_update, $logfile); my @labels; @@ -49,6 +49,7 @@ 'stage=s' => \$stage, # stage to queue 'label=s' => \@labels, # labels + 'workdir=s' => \$workdir, # workdir 'stage_limit=s' => \$stage_limit,# maximum number of runs queued for each stage - 'dist_root=s' => \$dist_root, # root of distribution work area +# 'dist_root=s' => \$dist_root, # root of distribution work area 'no_magic' => \$no_magic, # queue runs without requiring magic (for testing only) 'dbname=s' => \$dbname, # Database name @@ -66,15 +67,19 @@ $ipprc->redirect_output($logfile) if $logfile; -if (!$dist_root) { - $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT"); - &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root; +if (!$workdir) { + print "workdir not supplied will use DISTRIBUTION_ROOT\n"; + # old method where we set workdir based on a config file + if (!$dist_root) { + $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT"); + &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root; + } + + my ($day, $month, $year) = (gmtime)[3,4,5]; + my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day; + + $workdir = $dist_root . "/$datestr"; + + print "workdir is $workdir\n"; } - -my ($day, $month, $year) = (gmtime)[3,4,5]; -my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day; - -my $workdir = "$dist_root/$datestr"; - -print "workdir is $workdir\n"; # if stage is not supplied as an argument, loop over all stages @@ -90,5 +95,5 @@ my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label"; $command .= " -no_magic" if $no_magic; - $command .= " -dry_run" if $no_update; + $command .= " -pretend" if $no_update; $command .= " -limit $stage_limit" if $stage_limit; $command .= " -set_label $label"; Index: /tags/ipp-20101215/ippScripts/scripts/dist_make_fileset.pl =================================================================== --- /tags/ipp-20101215/ippScripts/scripts/dist_make_fileset.pl (revision 30489) +++ /tags/ipp-20101215/ippScripts/scripts/dist_make_fileset.pl (revision 30490) @@ -96,5 +96,5 @@ # make sure that the database info file for this run exists my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc"; -if (! -e "$dbinfo_file" ) { +if (! $ipprc->file_exists($dbinfo_file) ) { &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -103,5 +103,5 @@ # make sure that the dirinfo file for this run exists my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc"; -if (! -e "$dirinfo_file" ) { +if (!$ipprc->file_exists($dirinfo_file)) { &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -277,9 +277,13 @@ $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - print STDERR "Unable to perform $command error_code: $error_code\n"; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR "Unable to perform $command error_code: $error_code\n"; + } + } else { + print STDERR "skipping $command\n"; } exit $fault; Index: /tags/ipp-20101215/ippTasks/dist.pro =================================================================== --- /tags/ipp-20101215/ippTasks/dist.pro (revision 30489) +++ /tags/ipp-20101215/ippTasks/dist.pro (revision 30490) @@ -95,4 +95,5 @@ end +# This is now only used if we are not using a nebulous outroot macro get.host.for.component if ($0 != 3) @@ -122,4 +123,5 @@ end +# This is now only used if we are not using a nebulous outroot macro set.dist.workdir.by.component if ($0 != 5) @@ -139,5 +141,5 @@ book getword ipphosts distribution count -var count if ("$count" == "NULL") - echo "ERROR: distributin hosts list is empty" + echo "ERROR: distribution hosts list is empty" break end @@ -289,22 +291,22 @@ end - - set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR - if ("$OUTDIR" == "NULL") - echo ERROR failed to set workdir for $COMPONENT - break - end - substr $COMPONENT 0 3 COMP_HEAD if ("$COMP_HEAD" == "sky") set.host.for.skycell $COMPONENT + set.workdir.by.skycell $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR else - # assume component is a class_id, if not we will default to anyhost + # assume component is a class_id, if it is not we will default to + # host "anyhost" + # and volume "any" set.host.for.camera $CAMERA $COMPONENT + set.workdir.by.camera $CAMERA $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR + end + + if ("$OUTDIR" == "NULL") + echo ERROR failed to set workdir for $COMPONENT + break end sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT -# stdout $logfile -# stderr $logfile $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile @@ -401,4 +403,8 @@ book getpage distToAdvance 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break + + # XXX the sql for disttool -toadvance does not include the camera: + # hack around this for now + $CAMERA = GPC1 book setword distToAdvance $pageName pantaskState RUN @@ -414,20 +420,27 @@ end - host anyhost - - # using $DIST_ID as the "component" works fine here since we only look - # at the last two digits. But make sure that there 2 digits - $fake_component = $STAGE_ID + 10 - set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR + substr $OUTDIR_TEMPLATE 0 3 SCHEME + if ("$SCHEME" == "neb") + # XXX This does not work if workdir is not in nebulous + # and $default_host == "any" because /data/any does not exist + # use "dummy" as component. Result will be 'any' + # set.workdir.by.camera $CAMERA dummy $OUTDIR_TEMPLATE $default_host OUTDIR + set.workdir.by.camera GPC1 dummy $OUTDIR_TEMPLATE $default_host OUTDIR + else + # using $DIST_ID as the "component" works fine here since we only look + # at the last two digits. But make sure that there 2 digits + $fake_component = $STAGE_ID + 10 + set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR + end + if ("$OUTDIR" == "NULL") echo ERROR failed to set workdir for $DIST_ID break end + host anyhost sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID -# stdout $logfile -# stderr $logfile - - $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile + + $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --camera $CAMERA --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile add_standard_args run Index: /tags/ipp-20101215/ippTasks/rcserver.pro =================================================================== --- /tags/ipp-20101215/ippTasks/rcserver.pro (revision 30489) +++ /tags/ipp-20101215/ippTasks/rcserver.pro (revision 30490) @@ -136,10 +136,8 @@ sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID - stdout $logfile - stderr $logfile book setword rcPendingFS $pageName pantaskState RUN - $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR + $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR --logfile $logfile add_standard_args run Index: /tags/ipp-20101215/ippTasks/survey.pro =================================================================== --- /tags/ipp-20101215/ippTasks/survey.pro (revision 30489) +++ /tags/ipp-20101215/ippTasks/survey.pro (revision 30490) @@ -287,9 +287,10 @@ # user functions to manipulate dist labels macro survey.add.dist - if ($0 != 2) - echo "USAGE: survey.add.dist (label)" + if ($0 != 3) + echo "USAGE: survey.add.dist (label) (workdir)" break end book newpage SURVEY_DIST $1 + book setword SURVEY_DIST $1 WORKDIR $2 book setword SURVEY_DIST $1 STATE PENDING end @@ -896,8 +897,14 @@ end + book getword SURVEY_DIST $label WORKDIR -var workdir_base + $year = `date +%Y` + $month = `date +%m` + $day = `date +%d` + $workdir = $workdir_base/$label/$year/$month/$day + book setword SURVEY_DIST $label STATE DONE # note workdir is set by the script based on site.config - $run = dist_defineruns.pl --label $label + $run = dist_defineruns.pl --label $label --workdir $workdir if ($DB:n == 0) @@ -908,5 +915,7 @@ end -# echo $run + if ($VERBOSE > 1 ) + echo $run + end command $run end