Index: trunk/ippScripts/scripts/dist_advancerun.pl
===================================================================
--- trunk/ippScripts/scripts/dist_advancerun.pl	(revision 24104)
+++ trunk/ippScripts/scripts/dist_advancerun.pl	(revision 24125)
@@ -16,5 +16,5 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
-use File::Basename qw( basename );
+use File::Basename qw( dirname);
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -72,27 +72,45 @@
 
 my $tool_cmd;
+my $list_mode;
+my $component_key;
 if ($stage eq "raw") {
     $tool_cmd = "$regtool -exp_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "chip") {
     $tool_cmd = "$chiptool -chip_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "camera") {
     $tool_cmd = "$camtool -cam_id";
+    $list_mode = "-processedexp";
+    $component_key = "";
 } elsif ($stage eq "fake") {
     $tool_cmd = "$faketool -fake_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "warp") {
     $tool_cmd = "$warptool -warp_id";
+    $list_mode = "-warped";
+    $component_key = "skycell_id";
 } elsif ($stage eq "stack") {
     $tool_cmd = "$stacktool -stack_id";
+    $list_mode = "-sumskyfile";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";
+    $list_mode = "-diffskyfile";
+    $component_key = "skycell_id";
 } else {
     &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
 }
 
+$tool_cmd .= " $stage_id";
+
 # XXX should we create a file rule for this?
-my $outfile = "$outdir/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $command = "$tool_cmd $stage_id -exportrun -outfile $outfile";
+my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
+
+{
+    my $command = "$tool_cmd -exportrun -outfile $dbinfo_file";
     $command .= " -clean" if defined $clean;
     $command .= " -dbname $dbname" if defined $dbname;
@@ -105,8 +123,7 @@
     }
 }
-
-# set distRun.stage = 'full'
-{
-    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+{
+    my $command = "$tool_cmd $list_mode";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -117,4 +134,58 @@
         &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
     }
+    if (@$stdout_buf == 0) {
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $components = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    open MANIFEST, ">$dirinfo" or
+        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $destdir;
+    foreach my $c (@$components) {
+        if (!$destdir) {
+            my $workdir = $c->{workdir};
+            if ($workdir) {
+                $destdir = stripvolume($workdir, $stage);
+            } elsif ($stage eq 'raw') {
+                $destdir = 'none';
+            } else {
+                &my_die("workdir not found for open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+            }
+            print MANIFEST "destdir METADATA\n";
+            print MANIFEST "\t" , "destdir", "\tSTR\t", $destdir, "\n";
+            print MANIFEST "END\n\n";
+            print MANIFEST "components METADATA\n";
+        }
+        my $component = $c->{$component_key} ? $c->{$component_key} : "exposure";
+        my $path;
+        if ($stage eq 'raw') {
+            $path = $c->{uri};
+        } else {
+            $path = $c->{path_base};
+        }
+        &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
+        my $component_dir = find_componentdir($destdir, $path); 
+#        print MANIFEST "$component METADATA\n";
+        print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
+    }
+    print MANIFEST "END\n\n";
+    close MANIFEST or 
+        &my_die("Unable to close dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+{
+    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
 }
 
@@ -124,4 +195,66 @@
 ### Pau.
 
+sub stripvolume
+{
+    my $path = shift;
+    my $stage = shift;
+    my @segments;
+
+    # workdir isn't what we want for raw stage
+    return "none" if ($stage and ($stage eq 'raw'));
+
+    my $scheme = file_scheme($path);
+    my $tail;
+    if ($scheme) {
+        # strip off scheme://
+        $tail = substr($path, length($scheme) + 3);
+    } elsif (substr($path, 0, 1) eq '/') {
+        $tail = substr($path, 1);
+        $scheme = "";
+    }
+    # remove any leading / that are left
+    while ((substr($tail, 0, 1) eq '/')) {
+        $tail = substr($tail, 1);
+    }
+
+    if (($scheme eq 'neb') or ($scheme eq 'path')) {
+        my $volume;
+        ($volume, @segments) = split '/', $tail;
+
+    } elsif (!$scheme or ($scheme eq 'file')) {
+
+        # XXX Here we're assuming the /data/ipp??? structure. This won't be true when data is forwarded
+        # by remote sites. We need a way to configure this
+        my $volume;
+
+        # data/ippxxx/dirs
+        (undef, $volume, @segments) = split '/', $tail;
+    } else {
+        die( "unexpected workdir value: $path\n");
+    }
+
+    return caturi(@segments);
+}
+
+sub find_componentdir
+{
+    my $destdir = shift;
+    my $path = shift;
+
+    my $result;
+    if ($destdir eq 'none') {
+        $result = stripvolume($path);
+    } else {
+        # find location of destdir in the path
+        my $i = index($path, $destdir);
+
+        $result = substr($path, $i + length($destdir) + 1);
+
+        while (substr($result, 0, 1) eq '/') {
+            $result = substr($result, 1);
+        }
+    }
+    return dirname($result);
+}
 
 sub my_die
Index: trunk/ippScripts/scripts/dist_make_fileset.pl
===================================================================
--- trunk/ippScripts/scripts/dist_make_fileset.pl	(revision 24104)
+++ trunk/ippScripts/scripts/dist_make_fileset.pl	(revision 24125)
@@ -91,5 +91,12 @@
     &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
 }
-print "dbinfo file $dbinfo_file exists\n";
+print "dbinfo file $dbinfo_file exists\n" if $verbose;
+
+# make sure that the dirinfo file for this run exists
+my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc";
+if (! -e "$dist_dir/$dirinfo_file" ) {
+    &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+print "dirinfo file $dirinfo_file exists\n" if $verbose;
 
 # open the dsreg file list
@@ -97,6 +104,7 @@
 
 # add the dbinfo file to the list
-# XXX: change type from text to dbinfo once we descide on types
+# XXX: change type from text to dbinfo and dirinfo once we define the types
 print $listFile "$dbinfo_file|||text|dbinfo|\n";
+print $listFile "$dirinfo_file|||text|dirinfo|\n";
 
 my $components;
Index: trunk/ippScripts/scripts/receive_file.pl
===================================================================
--- trunk/ippScripts/scripts/receive_file.pl	(revision 24104)
+++ trunk/ippScripts/scripts/receive_file.pl	(revision 24125)
@@ -17,6 +17,8 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
 use File::Temp qw( tempfile );
+use File::Basename qw( basename );
 use Carp;
 
@@ -36,5 +38,5 @@
 
 # Parse the command-line arguments
-my ( $file_id, $source, $product, $fileset, $fileset_id, $file, $component, $bytes, $md5sum, $workdir, $dbname, $verbose, $no_update, $save_temps );
+my ( $file_id, $source, $product, $fileset, $fileset_id, $file, $component, $bytes, $md5sum, $workdir, $dirinfo_uri, $dbname, $verbose, $no_update, $save_temps );
 
 GetOptions(
@@ -49,4 +51,5 @@
            'md5sum=s'          => \$md5sum, # md5sum for file from data store
            'workdir=s'         => \$workdir, # Working directory for output
+           'dirinfo=s'    => \$dirinfo_uri, # file containing the destination directories for this component
            'dbname=s'          => \$dbname,    # Database name
            'verbose'           => \$verbose,   # Print to stdout
@@ -56,5 +59,5 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --file_id --source --product --fileset --file --workdir",
+pod2usage( -msg => "Required options: --file_id --source --product --fileset --file --component --workdir --bytes --md5sum --dirinfo",
            -exitval => $PS_EXIT_CONFIG_ERROR) unless
     defined $file_id and
@@ -66,10 +69,18 @@
     defined $bytes and
     defined $md5sum and
+    defined $dirinfo_uri and
     defined $workdir;
 
 $tempdir .= "/$file_id";
+
+&my_die( "dirinfo is NULL for $component", $file_id, $PS_EXIT_CONFIG_ERROR )
+    if (($dirinfo_uri eq "NULL") and ($component ne "dirinfo"));
 
 my $ipprc = PS::IPP::Config->new() or
     &my_die( "Unable to set up", $file_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+
 
 # Retrieve file
@@ -87,41 +98,161 @@
 my $mjd_copy = DateTime->now->mjd;   # MJD of finishing copy
 
+# figure out which dirinfo file to read
+my $dirinfo_file_to_read = $component eq "dirinfo" ? $filename : $dirinfo_uri;
+
+# process it
+my ($destdir, $components, $dirinfo_lines) = read_dirinfo_file($dirinfo_file_to_read, $file_id);
+
+# select a directory for the dirinfo and dbinfo files
+# XXX: perhaps this directory should be set by the script and passed in
+# rather than computed here.
+
+my ($day, $month, $year) = (localtime)[3,4,5];
+my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+my $dir_for_info_files = caturi($workdir, $datestr, $fileset);
+
 # Deal with file
-if ($file =~ m|^dbinfo\.\S+\.mdc$|) {
-    # Load into database
-
-    my $target = "$workdir/$file"; # Target destination for file
-    my $fixName = $ipprc->file_create( $target ); # Target for move
-    my $fixFile;
-
-    open $fixFile, ">$fixName" or &my_die( "can't open $fixName\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
-
-    # Need to fix paths to point to new workdir
-    open my $inFile, $filename or &my_die( "Can't open $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); # Input file
-    my $workdir_old;            # Old workdir
-    while (<$inFile>) {
-        # XXX This is a global approach to fixing the path: it should fix anything and everything, but won't
-        # work if there are multiple workdirs in a file and the bits are all mixed up.  To cover that case,
-        # we should fix each of the elements (workdir, uri, path_base) separately.
-        if (m|^\s*workdir\s+STR\s+(\S+)|) {
-            $workdir_old = $1;
-            $workdir_old =~ s|\@HOST\@|\\S+|;
-        }
-        if (defined $workdir_old) {
-            s|$workdir_old|$workdir|;
-        }
-        print $fixFile $_;
-    }
-    close($inFile);
-    close($fixFile);
-#{
-    my $command = "$receivetool -updatefileset -fileset_id $fileset_id -dbinfo_uri $fixName"; # Command to execute
+if ($component eq 'dirinfo') {
+    # save the dirinfo file contents into the $workdir
+
+    $dirinfo_uri = caturi($dir_for_info_files, basename($filename));
+    print "dirinfo_uri: $dirinfo_uri\n" if $verbose;
+
+    my $resolved = $ipprc->file_resolve($dirinfo_uri, 'create');
+    &my_die( "failed to resolve $dirinfo_uri\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    open OUT, ">$resolved" 
+        or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    print OUT @$dirinfo_lines
+        or &my_die( "failed to write $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    close OUT
+        or &my_die( "failed to close $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    
+    # update the fileset to allow processing of other files
+    my $command = "$receivetool -updatefileset -fileset_id $fileset_id";
+    $command .= " -set_state new -dirinfo $dirinfo_uri";
     $command .= " -dbname $dbname" if defined $dbname;
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    &my_die( "Unable to set dbinfo_uri for $fileset_id to  $fixName\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
-#}
-
-} elsif ($file =~ m|.*\.tgz$|) {
+    &my_die( "Unable to update fileset $fileset_id to\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+
+} elsif ($component eq "dbinfo") {
+
+    open INFILE, $filename or &my_die( "Can't open $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my @lines = (<INFILE>);
+
+    my $dbinfo = join "", @lines;
+
+    close INFILE;
+
+    my $dbinfo_uri = caturi($dir_for_info_files, basename($filename));
+    print "dbinfo_uri: $dbinfo_uri\n" if $verbose;
+
+    my $resolved = $ipprc->file_resolve($dbinfo_uri, 'create');
+    &my_die( "failed to resolve $dbinfo_uri\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    open OUT, ">$resolved" 
+        or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # We process the dbinfo file (the exported run from the distribution) line by line
+    # Rather than read it as an mdc and interptet it, we do our substitutions directly
+    # This is much faster. Parsing a mdc file for a chip run takes several seconds.
+    # we are very strict on the formatting
+
+    # first line tells us the run type. From this we get the stage
+    #
+    my $line = $lines[0];
+    my ($runType, $multi) = split " ", $line;
+    &my_die( "unexpected first line found in $filename: $line\n", $file_id, $PS_EXIT_UNKNOWN_ERROR)
+        if !$runType or ($multi ne 'MULTI');
+    my $stage;
+    my $comp_name;
+    my $current_component;
+    if ($runType eq 'rawExp') {
+        $stage = 'raw';
+        $comp_name = 'class_id';
+    } elsif ($runType eq 'chipRun') {
+        $stage = 'chip';
+        $comp_name = 'class_id';
+    } elsif ($runType eq 'camRun') {
+        $stage = 'camera';
+        $comp_name = 'exposure';
+        $current_component = $comp_name;
+    } elsif ($runType eq 'fakeRun') {
+        $stage = 'fake';
+        $comp_name = 'class_id';
+    } elsif ($runType eq 'warpRun') {
+        $stage = 'warp';
+        $comp_name = 'skycell_id';
+    } elsif ($runType eq 'diffRun') {
+        $stage = 'diff';
+        $comp_name = 'skycell_id';
+    } elsif ($runType eq 'stackRun') {
+        $stage = 'stack';
+        $comp_name = 'skycell_id';
+    } else {
+        &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
+
+    my $new_workdir_value;
+    if ($destdir eq 'none') {
+        $new_workdir_value = "$workdir";
+    } else {
+        $new_workdir_value = "$workdir/$destdir";
+    }
+    my $component_dir;
+    if ($current_component) {
+        $component_dir = $components->{$current_component};
+    }
+    foreach $line (@lines) {
+        my $out_line = $line;
+
+        my ($name, $type, $value) = split " ", $line;
+        # we only edit complete lines
+        if ($name and $type and $value) {
+            my $new_value;
+            if ($name eq $comp_name) {
+                $current_component = $value;
+                $component_dir = $components->{$current_component};
+                &my_die( "$component_dir is null for $value in $filename: $runType\n", 
+                        $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$component_dir;
+            } elsif ($name eq 'workdir') {
+                $new_value = $new_workdir_value;
+            } elsif ($name eq 'tess_id') {
+                # for tess_id strip off any directories just keep the basename.
+                # The site configuration will need to map this to a proper location
+                # XXX: Document this
+                $new_value = basename($value);
+            } elsif ((($name eq 'uri') or ($name eq 'path_base')) and ($value ne 'NULL')) {
+                &my_die( "$component_dir is null and we need it for $name", 
+                        $file_id, $PS_EXIT_PROG_ERROR) if !$component_dir;
+
+                $new_value = caturi($new_workdir_value, $component_dir, basename($value));
+            }
+
+            if ($new_value) {
+                $out_line = "   " . $name . "\t\t" . $type . "\t" . $new_value . "\n";
+            }
+        }
+
+        print OUT $out_line or &my_die( "failed to write to $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
+
+    close OUT
+        or &my_die( "failed to close $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # update the fileset to allow processing of other files
+    my $command = "$receivetool -updatefileset -fileset_id $fileset_id";
+    $command .= " -dbinfo $dbinfo_uri";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to update fileset $fileset_id to\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+
+
+} elsif ($file =~ m|.*\.tgz$|) {        # XXX: perhaps get this off of file type ?
     # Get contents of tarball
     my @files = ();
@@ -151,8 +282,19 @@
     }
 
+    my $component_dir = $components->{$component};
+    &my_die( "Unable to find component_dir for $component $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $component_dir;
+
+    my $target_dir;
+    if ($destdir eq 'none') {
+        $target_dir = "$workdir";
+    } else {
+        $target_dir = "$workdir/$destdir";
+    }
+    $target_dir .= "/$component_dir";
+
     # Move files into filesystem of choice
     foreach my $file ( @files ) {
         my $from = "$tempdir/$file"; # Source for file
-        my $target = "$workdir/$file"; # Target destination for file
+        my $target = "$target_dir/$file"; # Target destination for file
         my $to = $ipprc->file_create( $target ); # Target for move
         system("mv $from $to") == 0 or &my_die( "Unable to move $file into workdir $workdir: $!\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
@@ -162,5 +304,7 @@
 }
 
-unlink $filename or &my_die( "Unable to unlink $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+if (!$save_temps) {
+    unlink $filename or &my_die( "Unable to unlink $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+}
 my $mjd_extract = DateTime->now->mjd;   # MJD of finishing extract
 
@@ -186,4 +330,32 @@
 # Pau.
 
+sub read_dirinfo_file
+{
+    my $filename = shift;
+    my $file_id = shift;
+
+    open INFILE, $filename or &my_die( "Can't open $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my @lines = (<INFILE>);
+
+    my $dirinfo = join "", @lines;
+
+    close INFILE;
+
+    my $metadata = $mdcParser->parse($dirinfo) or
+        &my_die("Unable to parse metadata config doc", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $array = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $dest_hash = $array->[0];
+
+    my $destdir = $dest_hash->{destdir};
+    &my_die("destdir not found in $filename", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$destdir;
+
+    my $components = $array->[1];
+
+    return ($destdir, $components, \@lines);
+}
 
 sub my_die
Index: trunk/ippScripts/scripts/receive_fileset.pl
===================================================================
--- trunk/ippScripts/scripts/receive_fileset.pl	(revision 24104)
+++ trunk/ippScripts/scripts/receive_fileset.pl	(revision 24125)
@@ -92,5 +92,5 @@
 my $new_state;
 if ($numFiles > 0) {
-    $new_state = "new";
+    $new_state = "listed";
     my $command = "receivetool -addfile"; # Command to execute
     $command .= " -fileset_id $fileset_id";