Index: /branches/pap_mops/ippScripts/scripts/publish_file.pl
===================================================================
--- /branches/pap_mops/ippScripts/scripts/publish_file.pl	(revision 25224)
+++ /branches/pap_mops/ippScripts/scripts/publish_file.pl	(revision 25225)
@@ -39,5 +39,5 @@
 
 # Parse the command-line arguments
-my ( $pub_id, $camera, $stage, $stage_id, $format, $product, $workdir );
+my ( $pub_id, $camera, $stage, $stage_id, $fileset, $format, $product, $workdir );
 my ( $dbname, $verbose, $no_update, $save_temps, $redirect );
 
@@ -48,4 +48,5 @@
     'stage_id=s'        => \$stage_id,    # Stage identifier
     'product=s'         => \$product,     # Datastore product name
+    'fileset=s'         => \$fileset,     # Fileset name
     'workdir=s'         => \$workdir,     # Working directory
     'dbname=s'          => \$dbname,    # Database name
@@ -77,19 +78,8 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-
-# Retrieve file name of interest
-my %data;                       # Data to publish
-{
-    my $command;                # Command to run
-
-    if ($stage eq 'diff') {
-        $command =  "difftool -diffskyfile -diff_id $stage_id";
-    } elsif ($stage eq 'camera') {
-        $command =  "camtool -processedexp -cam_id $stage_id";
-    } else {
-        &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR );
-    }
-    $command .= " -dbname $dbname" if defined $dbname;
-
+my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps );
+
+if ($stage eq 'camera') {
+    my $command =  "camtool -processedexp -cam_id $stage_id";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -102,4 +92,29 @@
         &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
 
+    &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$components > 1;
+
+    my $comp = $$components[0]; # Component of interest
+    my $path_base = $comp->{path_base}; # Base name for file
+    my $file = $ipprc->filename( "PSASTRO.OUTPUT", $path_base );
+    my $cam_id = $comp->{cam_id};
+    my $name = "cam_$cam_id";
+    print $dsFile "$file|||$product|$name|\n";
+} elsif ($stage eq 'diff') {
+    my $command =  "difftool -diffskyfile -diff_id $stage_id";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
+
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
+
+    my $components = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR);
+
+    my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
+    my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $product eq 'IPP-MOPS';
+
+    my %positive;               # Data for positive diff detections
+    my %negative;               # Data for negative diff detections
     foreach my $comp ( @$components ) {
         my $path_base = $comp->{path_base}; # Base name for file
@@ -108,111 +123,73 @@
         (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time};
         my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10);
-
-        if ($stage eq 'diff') {
-            my $skycell_id = $comp->{skycell_id};
-
-            my $data = { filename => $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ),
-                         zp => $zp,
-                         zp_err => $comp->{zpt_stdev},
-                         astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
-                         exp_name => $comp->{exp_name_1},
-                         exp_id => $comp->{exp_id_1},
-                         chip_id => $comp->{chip_id_1},
-                         cam_id => $comp->{cam_id_1},
-                         fake_id => $comp->{fake_id_1},
-                         warp_id => $comp->{warp1},
-                         direction => 1,
-                     };
-
-            $data{"$skycell_id.pos"} = $data;
-
-            # Negative direction
-            if (defined $comp->{bothways} and $comp->{bothways}) {
-                my $data = { filename => $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ),
-                             zp => $zp,
-                             zp_err => $comp->{zpt_stdev},
-                             astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
-                             exp_name => $comp->{exp_name_2},
-                             exp_id => $comp->{exp_id_2},
-                             chip_id => $comp->{chip_id_2},
-                             cam_id => $comp->{cam_id_2},
-                             fake_id => $comp->{fake_id_2},
-                             warp_id => $comp->{warp2},
-                             direction => 0,
-                         };
-
-                $data{"$skycell_id.neg"} = $data;
-            }
-        } elsif ($stage eq 'camera') {
-            my $data = { filename => $ipprc->filename( "PSASTRO.OUTPUT", $path_base ),
-                         zp => $zp,
+        my $astrom = sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2);
+
+        my $skycell_id = $comp->{skycell_id};
+        my $filename = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base );
+
+        my $data = { zp => $zp,
+                     zp_err => $comp->{zpt_stdev},
+                     astrom => sqrt($comp->{sigma_ra_1}**2 + $comp->{sigma_dec_1}**2),
+                     exp_name => $comp->{exp_name_1},
+                     exp_id => $comp->{exp_id_1},
+                     chip_id => $comp->{chip_id_1},
+                     cam_id => $comp->{cam_id_1},
+                     fake_id => $comp->{fake_id_1},
+                     warp_id => $comp->{warp1},
+                     direction => 1,
+        };
+
+        diff_check(\%positive, $data);
+
+        if ($product eq 'IPP-MOPS') {
+            print $mopsPositiveFile "$filename\n";
+        } else {
+            print $dsFile "$filename|||$product|${skycell_id}.pos|\n";
+        }
+
+        # Negative direction
+        if (defined $comp->{bothways} and $comp->{bothways}) {
+            my $filename = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base );
+            my $data = { zp => $zp,
                          zp_err => $comp->{zpt_stdev},
                          astrom => sqrt($comp->{sigma_ra_2}**2 + $comp->{sigma_dec_2}**2),
-                         exp_name => $comp->{exp_name},
-                         exp_id => $comp->{exp_id},
-                         chip_id => $comp->{chip_id},
-                         cam_id => $comp->{cam_id},
-                     };
-
-            my $cam_id = $comp->{cam_id};
-            $data{"cam.$cam_id"} = $data;
+                         exp_name => $comp->{exp_name_2},
+                         exp_id => $comp->{exp_id_2},
+                         chip_id => $comp->{chip_id_2},
+                         cam_id => $comp->{cam_id_2},
+                         fake_id => $comp->{fake_id_2},
+                         warp_id => $comp->{warp2},
+                         direction => 0,
+            };
+
+            diff_check(\%negative, $data);
+
+            if ($product eq 'IPP-MOPS') {
+                print $mopsNegativeFile "$filename\n";
+            } else {
+                print $dsFile "$filename|||$product|${skycell_id}.neg|\n";
+            }
         }
     }
-}
-
-# Prepare for data store input
-my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps );
-
-# Process each file
-my ($zp, $zp_err, $astrom, $exp_name, $exp_id, $chip_id, $cam_id, $fake_id, $warp_id, $direction);
-foreach my $comp ( keys %data ) {
-    my $data = $data{$comp};
-    my $file = $ipprc->file_resolve( $data->{filename} ) or
-        &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR);
-    my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file );
-
-    $zp = $data->{zp} unless defined $zp;
-    $zp_err = $data->{zp_err} unless defined $zp_err;
-    $astrom = $data->{astrom} unless defined $astrom;
-    $exp_name = $data->{exp_name} unless defined $exp_name;
-    $exp_id = $data->{exp_id} unless defined $exp_id;
-    $chip_id = $data->{chip_id} unless defined $chip_id;
-    $cam_id = $data->{cam_id} unless defined $cam_id;
-    $fake_id = $data->{fake_id} unless defined $fake_id;
-    $warp_id = $data->{warp_id} unless defined $warp_id;
-    $direction = $data->{direction} unless defined $direction;
-
-    &my_die("zp value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $zp and $zp != $data->{zp};
-    &my_die("zp_err value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $zp_err and $zp_err != $data->{zp_err};
-    &my_die("astrom value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $astrom and $astrom != $data->{astrom};
-    &my_die("exp_name value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $exp_name and $exp_name ne $data->{exp_name};
-    &my_die("exp_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $exp_id and $zp != $data->{exp_id};
-    &my_die("chip_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $chip_id and $chip_id != $data->{chip_id};
-    &my_die("cam_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $cam_id and $cam_id != $data->{cam_id};
-    &my_die("fake_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $fake_id and $fake_id != $data->{fake_id};
-    &my_die("warp_id value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $warp_id and $warp_id != $data->{warp_id};
-    &my_die("direction value for $comp doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $direction and $direction != $data->{direction};
-
-    if ($product eq "IPP-MOPS") {
-        my $outuri = "$outroot.$comp.fits";
-        my $out = $ipprc->file_resolve( $outuri, 'create' ) or
-            &my_die( "Unable to resolve output file $outuri", $pub_id, $PS_EXIT_SYS_ERROR);
-
-        my $command = "$ppMops $file $zp $exp_id $exp_name $direction $out";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
-        &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out );
-        $file = $out;
+
+    close $mopsPositiveFile;
+    close $mopsNegativeFile;
+
+    if ($product eq 'IPP-MOPS') {
+        if (scalar keys %positive > 0) {
+            my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName);
+            print $dsFile "$output|||$product|positive|\n";
+        }
+        if (scalar keys %negative > 0) {
+            my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName);
+            print $dsFile "$output|||$product|negative|\n";
+        }
     }
-
-    # format: filename|filesize|md5sum|filetype|
-    # note: since we omit filesize and md5sum, dsreg will calculate them
-    print $listFile "$file|||$product|$comp|\n";
-}
-
+}
+
+close $dsFile;
 
 unless ($no_update) {
-    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName";
+    my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -233,4 +210,67 @@
 
 ### Pau.
+
+# Check inputs for a diff
+sub diff_check
+{
+    my $data = shift;           # Data hash
+    my $comp = shift;           # Component data
+    my $name = shift;           # Name of component
+
+    $data->{zp} = $comp->{zp} unless defined $data->{zp};
+    $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err};
+    $data->{astrom} = $comp->{astrom} unless defined $data->{astrom};
+    $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name};
+    $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id};
+    $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id};
+    $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id};
+    $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id};
+    $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id};
+    $data->{direction} = $comp->{direction} unless defined $data->{direction};
+
+    &my_die("zp value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp};
+    &my_die("zp_err value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp_err} and $comp->{zp_err} != $data->{zp_err};
+    &my_die("astrom value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{astrom} and $comp->{astrom} != $data->{astrom};
+    &my_die("exp_name value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_name} and $comp->{exp_name} ne $data->{exp_name};
+    &my_die("exp_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{exp_id} and $comp->{zp} != $data->{exp_id};
+    &my_die("chip_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{chip_id} and $comp->{chip_id} != $data->{chip_id};
+    &my_die("cam_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{cam_id} and $comp->{cam_id} != $data->{cam_id};
+    &my_die("fake_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{fake_id} and $comp->{fake_id} != $data->{fake_id};
+    &my_die("warp_id value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id};
+    &my_die("direction value for $namedoesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction};
+
+    return 1;
+}
+
+# Combine multiple files for MOPS
+sub mops_combine
+{
+    my $data = shift;           # Data
+    my $output = shift;         # Output name
+    my $input = shift;          # Input name
+
+    $output = $ipprc->file_resolve( $output, 'create' ) or
+        &my_die( "Unable to resolve output file $output", $pub_id, $PS_EXIT_SYS_ERROR);
+
+    my $command = "$ppMops $input $output";
+    $command .= " -exp_name " . $data->{exp_name} if defined $data->{exp_name};
+    $command .= " -exp_id " . $data->{exp_id} if defined $data->{exp_id};
+    $command .= " -chip_id " . $data->{chip_id} if defined $data->{chip_id};
+    $command .= " -cam_id " . $data->{cam_id} if defined $data->{cam_id};
+    $command .= " -fake_id " . $data->{fake_id} if defined $data->{fake_id};
+    $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id};
+    $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id};
+    $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0;
+    $command .= " -zp " . $data->{zp} if defined $data->{zp};
+    $command .= " -zp_error " . $data->{zp_err} if defined $data->{zp_err};
+    $command .= " -astrom_rms " . $data->{astrom} if defined $data->{astrom};
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to translate", $pub_id, $PS_EXIT_SYS_ERROR) unless $success;
+    &my_die( "Unable to find translated file $output", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $output );
+
+    return $output;
+}