Index: trunk/ippScripts/scripts/dist_make_fileset.pl
===================================================================
--- trunk/ippScripts/scripts/dist_make_fileset.pl	(revision 23838)
+++ trunk/ippScripts/scripts/dist_make_fileset.pl	(revision 23838)
@@ -0,0 +1,244 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Digest::MD5::File qw( file_md5_hex );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+# Look for programs we need
+my $missing_tools;
+my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
+my $dsreg   = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $product_root, $ds_dbhost, $ds_dbname);
+my ($dbname, $save_temps, $verbose, $no_update, $logfile);
+
+GetOptions(
+           'dist_id=s'      => \$dist_id,    # distribution run identifier
+           'dist_dir=s'     => \$dist_dir,   # directory containing dist run outputs
+           'target_id=s'    => \$target_id,  # 
+           'stage=s'        => \$stage,      # raw, chip, camera, fake, warp, stack, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, etc.
+           'prod_id=s'      => \$prod_id,    # id for the product
+           'product_name=s' => \$product_name,  # location of the data store directory for this product
+           'product_root=s' => \$product_root,  # location of the data store directory for this product
+           'ds_dbhost=s'    => \$ds_dbhost,  # database host for the datastore database
+           'ds_dbname=s'    => \$ds_dbname,  # database name for the datastore database
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --product_root --ds_dbhost --ds_dbname",
+           -exitval => 3) unless
+    defined $dist_id and
+    defined $dist_dir and
+    defined $target_id and
+    defined $stage and
+    defined $stage_id and
+    defined $prod_id and
+    defined $product_name and
+    defined $product_root and
+    defined $ds_dbhost and
+    defined $ds_dbname;
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $fs_tag = get_fileset_tag($ipprc, $stage, $stage_id, $dbname);
+
+&my_die("failed to lookup fileset tag", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR) if !$fs_tag;
+
+my $fileset_name = "$fs_tag.$stage.$stage_id.$dist_id.$prod_id";
+
+print "$fileset_name\n";
+
+
+# make sure that the database info file for this run exists
+my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc";
+if (! -e "$dist_dir/$dbinfo_file" ) {
+    &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+print "dbinfo file $dbinfo_file exists\n";
+
+# open the dsreg file list
+my ($listFile, $listFileName) = tempfile("/tmp/$stage.$stage_id.list.XXXX", UNLINK => !$save_temps );
+
+# add the dbinfo file to the list
+print $listFile "$dbinfo_file|||text|\n";
+
+my $components;
+{
+    my $command = "$disttool -processedcomponent -dist_id $dist_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    }
+
+    my $metadata = $mdcParser->parse (join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $dist_id, $prod_id, $PS_EXIT_PROG_ERROR);
+
+    $components = parse_md_list($metadata);
+
+    my $num_components = scalar @$components;
+
+    print "distRun $dist_id has $num_components components\n";
+}
+
+foreach my $component (@$components) {
+    my $size = $component->{bytes};
+    # skip zero size components. They are placeholders to complete processing
+    next if $size == 0;
+    my $md5sum = $component->{md5sum};
+    my $name = $component->{name};
+
+    # XXX: if tarfile is not always the right type we need to add a type to distComponent
+    print $listFile "$name|$size|$md5sum|tgz|\n";
+}
+
+close $listFile;
+
+{
+    # XXX: need to chose an appropriate file set type
+    my $command = "$dsreg --add $fileset_name --product $product_name --type notset --list $listFileName";
+
+    # the data store will refer to the distribution bundle via symlinks back to distRun.outdir
+    $command .= " --datapath $dist_dir --link";
+
+    # set the product specific columns in product list
+    $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $fs_tag";
+
+    $command .= " --dbname $ds_dbname";    # XXX: notyet --dbhost $ds_dbhost
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    }
+}
+
+# XXX: disttool -addrcdsfileset
+
+# XXX todo: add rcRun's for the destinations
+
+
+exit 0;
+
+sub getDBHandle {
+    my $ipprc = shift;
+    my $dbname = shift;
+    if (!$dbname) {
+        $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME");
+    }
+    my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER");
+    my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER");
+    my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD");
+
+    die "database configuration not set up" unless defined($dbserver) and defined($dbuser)
+        and defined($dbpassword) and defined($dbname);
+
+    my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+        or die "Cannot connect to database.\n";
+
+    return $dbh;
+}
+
+
+sub get_fileset_tag {
+    my $ipprc = shift;
+    my $stage = shift;
+    my $stage_id = shift;
+    my $dbname = shift;
+
+    if ($stage eq 'stack') {
+        return "stack.$stage_id";
+    }
+
+    #
+    # we are a long ways away from the rawExp in the pipline. Rather than do some exotic
+    # queries in the disttool, we look up the exp_name in the database using DBI
+    #
+    my $dbh = getDBHandle($ipprc, $dbname);
+
+    my $query; 
+
+    if ($stage eq 'raw') {
+        $query = "SELECT exp_name FROM rawExp WHERE exp_id = $stage_id";
+    } elsif ($stage eq 'chip') {
+        $query = "SELECT exp_name FROM chipRun JOIN rawExp USING(exp_id) WHERE chip_id = $stage_id";
+    } elsif ($stage eq 'camera') {
+        $query = "SELECT exp_name FROM camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"
+                    . " WHERE cam_id = $stage_id";
+    } elsif ($stage eq 'fake') {
+        $query = "SELECT exp_name FROM fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id)"
+                    . " JOIN rawExp USING(exp_id) WHERE fake_id = $stage_id";
+    } elsif ($stage eq 'warp') {
+        $query = "SELECT exp_name FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id)"
+                    . " JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE warp_id = $stage_id";
+    } elsif ($stage eq 'diff') {
+        $query = "SELECT exp_name FROM diffRun JOIN rawExp USING(exp_id) WHERE diff_id = $stage_id";
+    } else {
+        &my_die("$stage is invalid value for stage\n");
+    }
+
+    my $stmt = $dbh->prepare($query);
+    $stmt->execute();
+    my $ref = $stmt->fetchrow_hashref();
+
+    my $tag = $ref->{exp_name};
+
+    return $tag;
+}
+
+sub my_die {
+    my $msg = shift;
+    my $dist_id = shift;
+    my $prod_id = shift;
+    my $fault = shift;
+
+    # TODO: disttool -adddsfileset -prod_id $prod_id -dist_id $dist_id -fault $fault
+    print STDERR "$msg\n";
+
+    exit $fault;
+}
+
+__END__