Changeset 32851

Timestamp:

Dec 2, 2011, 11:44:07 AM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20111122

Files:

• ippMonitor (modified) (1 prop)
• ippMonitor/Makefile.in (modified) (1 diff)
• ippMonitor/czartool/czarplot.pl (modified) (8 diffs)
• ippMonitor/czartool/czarpoll.pl (modified) (2 diffs)
• ippMonitor/czartool/czartool/CzarDb.pm (modified) (7 diffs)
• ippMonitor/czartool/czartool/Pantasks.pm (modified) (1 diff)
• ippMonitor/czartool/czartool/Plotter.pm (modified) (14 diffs)
• ippMonitor/raw/czartool_ipptopsps.php (copied) (copied from trunk/ippMonitor/raw/czartool_ipptopsps.php )
• ippMonitor/raw/czartool_labels.php (modified) (4 diffs)
• ippMonitor/raw/ipp.imfiles.dat (modified) (1 diff)
• ippScripts/scripts/addstar_run.pl (modified) (4 diffs)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/ipp_filename.pl (modified) (1 diff)
• ippScripts/scripts/magic_destreak.pl (modified) (1 diff)
• ippScripts/scripts/register_exp.pl (modified) (4 diffs)
• ippScripts/scripts/videophot_process.pl (modified) (3 diffs)
• ippScripts/scripts/whichimage (modified) (2 diffs)
• ippTasks/addstar.pro (modified) (7 diffs)
• ippTasks/survey.pro (modified) (3 diffs)
• ippTools/share/Makefile.am (modified) (5 diffs)
• ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (copied) (copied from trunk/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql )
• ippTools/share/addtool_find_processedexp_staticsky_multi.sql (copied) (copied from trunk/ippTools/share/addtool_find_processedexp_staticsky_multi.sql )
• ippTools/share/addtool_find_sky_id_multi.sql (copied) (copied from trunk/ippTools/share/addtool_find_sky_id_multi.sql )
• ippTools/share/addtool_find_sky_id_multi_dvo.sql (copied) (copied from trunk/ippTools/share/addtool_find_sky_id_multi_dvo.sql )
• ippTools/share/addtool_queue_cam_id.sql (modified) (1 diff)
• ippTools/share/addtool_queue_sky_id.sql (modified) (1 diff)
• ippTools/share/addtool_queue_sky_id_multi.sql (copied) (copied from trunk/ippTools/share/addtool_queue_sky_id_multi.sql )
• ippTools/share/addtool_queue_stack_id.sql (modified) (1 diff)
• ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (copied) (copied from trunk/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql )
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/disttool_listinterests.sql (modified) (1 diff)
• ippTools/share/magicdstool_getskycells.sql (modified) (2 diffs)
• ippTools/share/pstamptool_listfile.sql (copied) (copied from trunk/ippTools/share/pstamptool_listfile.sql )
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff, 1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/addtool.c (modified) (13 diffs)
• ippTools/src/disttool.c (modified) (1 diff)
• ippTools/src/flatcorr.c (modified) (1 diff)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/pstamptool.c (modified) (3 diffs)
• ippTools/src/pstamptool.h (modified) (1 diff)
• ippTools/src/pstamptoolConfig.c (modified) (2 diffs)
• ippTools/src/pxadd.c (modified) (3 diffs)
• ippTools/src/pxadd.h (modified) (1 diff)
• ippTools/src/vptool.c (modified) (3 diffs)
• ippTools/src/vptoolConfig.c (modified) (2 diffs)

branches/eam_branches/ipp-20111122/ippMonitor/Makefile.in

@@ -60 +60 @@
 $(DESTWWW)/czartool_getplot.php \
 $(DESTWWW)/czartool_exposures.php \
+$(DESTWWW)/czartool_ipptopsps.php \
 $(DESTWWW)/histogram.php \
 $(DESTWWW)/show_and_delete_image.php \

branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarplot.pl

@@ -27 +27 @@
 my $nebulous = undef;
 my $savingToFile = undef;
-my $analysis = undef;
 my $exposureId = undef;
 my $log = undef;
+my $psps = undef;
 
 GetOptions (
@@ -45 +45 @@
         "nebulous|n" => \$nebulous,
         "rate|r" => \$rate,
-        "analysis|a" => \$analysis,
         "timeseries|t" => \$timeSeries,
         "verbose|v" => \$verbose,
         "log|g" => \$log,
+        "psps|u" => \$psps,
         );
 
@@ -58 +58 @@
     print "* UNKNKOWN: option                          @ARGV\n";
 }
-if ($analysis && !$stage) {
-    $quit = 1;
-    print "* REQUIRED: choose a stage for analsis      -s <chip|cam|warp|etc>      (default=none)\n";}
 if (!$histogram) {
     print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
@@ -71 +68 @@
 if (!$rate) {
     print "* OPTIONAL: plot histogram of rate          -r                          (default=off)\n";} 
-if (!$analysis) {
-    $analysis = 0;
-    print "* OPTIONAL: include analysis                -a                          (default=$analysis)\n";} 
 if (!$log) {
     $log = 0;
@@ -97 +91 @@
     print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";
 } 
+if (!$psps) {
+    print "* OPTIONAL: plot ippToPsps stages           -u                          (default=off)\n";} 
 print "*\n*******************************************************************************\n";
 
@@ -104 +100 @@
 
 # default values
-if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;}
+if (!$rate && !$magicMask && !$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1;}
 if (!$verbose) {$verbose = 0;}
 if (!$save_temps) {$save_temps = 0;}
@@ -123 +119 @@
     $plotter = czartool::Plotter->new_display($config, $save_temps);
 }
+
+if ($psps) {$plotter->setIppToPspsMode();}
 
 # if a single day has been chosen 
@@ -177 +175 @@
 }
 
-if($analysis) {
-
-    my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end);
-    if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();}
-}

branches/eam_branches/ipp-20111122/ippMonitor/czartool/czarpoll.pl

@@ -37 +37 @@
 
 
-my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub");
-
+my @stages = (
+        "burntool", 
+        "chip", 
+        "cam", 
+        "fake", 
+        "warp", 
+        "stack", 
+        "diff", 
+        "magic", 
+        "magicDS", 
+        "dist", 
+        "pub");
+
+my @ippToPspsStages = (
+        "processed",
+        "loaded_to_datastore",
+        "loaded_to_ODM",
+        "merge_worthy",
+        "deleted_datastore",
+        "deleted_dxlayer",
+        "merged",
+        "deleted_local");
+
+my @allStages = (@stages, @ippToPspsStages);
 
 timePoll($period);
@@ -244 +266 @@
     print "* Generating plots\n";
 
+    # get ippToPsps labels
+    my $ippToPspsLabels = undef;
+    $czarDb->getCurrentIppToPspsLabels($begin, $end, \$ippToPspsLabels);
+    $plotter->setIppToPspsMode();
+    
+    # create ippToPsps labels
+    foreach $stage (@ippToPspsStages) {
+        foreach $row ( @{$ippToPspsLabels} ) {
+            my ($label) = @{$row};
+            
+            chomp($label);
+            $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1);
+        }
+    }
+
+    # create plots for each label for all stages for ippToPsps
+    foreach $row ( @{$ippToPspsLabels} ) {
+        my ($label) = @{$row};
+
+        $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1);
+    }
+
+
     # create plots for each label for each stage
+    $plotter->setStandardMode();
     foreach $stage (@stages) {
         foreach $row ( @{$rows} ) {

branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/CzarDb.pm

@@ -8 +8 @@
 use File::Temp;
 
-my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); # TODO put elsewhere
+my @stages = (
+        "burntool", 
+        "chip", 
+        "cam", 
+        "fake", 
+        "warp", 
+        "stack", 
+        "diff", 
+        "magic", 
+        "magicDS", 
+        "dist", 
+        "pub"); # TODO put elsewhere
+
+my @ippToPspsStages = (
+        "processed",
+        "loaded_to_datastore",
+        "loaded_to_ODM",
+        "merge_worthy",
+        "deleted_datastore",
+        "deleted_dxlayer",
+        "merged",
+        "deleted_local");
+
+my @allStages = (@stages, @ippToPspsStages);
 
 use base 'czartool::MySQLDb';
@@ -33 +56 @@
 ###########################################################################
 #
-# Gets current_labels table
+# Gets current ippToPsps labels
+#
+###########################################################################
+sub getCurrentIppToPspsLabels {
+    my ($self, $begin, $end, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT DISTINCT label
+        FROM processed
+        WHERE timestamp >= '$begin' 
+        AND timestamp <= '$end' 
+SQL
+
+    if (!$query->execute) {return 0;}
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Gets current labels from the current_labels table
 #
 ###########################################################################
@@ -672 +718 @@
             # times
             $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp);
+            if ($timeSep == 0) {$timeSep = 1;}; # HACK to prevent divide-by-zero
+
             $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp);
             $formattedMiddleTimestamp = $self->getFormattedDate($middleTime);
@@ -681 +729 @@
             # added, and store value to subtract from linear value below. If > 1000
             # processed per hour, then it must be wrong
-            if (($thisProcessed - $lastProcessed)/(($timeSep/3600)+1) > 1000) {
+            if (($thisProcessed - $lastProcessed)/($timeSep/3600) > 1000) {
                 $positiveJump = $thisProcessed - $lastProcessed;
             }
@@ -701 +749 @@
 
             # calculate first derivative in units of images per hour
-            $derivPending = $diffPending/($timeSep/3600 +1); 
-            $derivFaults = $diffFaults/($timeSep/3600 +1); 
-            $derivProcessed = $diffProcessed/($timeSep/3600 +1); 
+            $derivPending = $diffPending/($timeSep/3600); 
+            $derivFaults = $diffFaults/($timeSep/3600); 
+            $derivProcessed = $diffProcessed/($timeSep/3600); 
 
         }
@@ -1107 +1155 @@
         my $stage = undef;
         $totalDeleted = 0;
-        foreach $stage (@stages) {
+        foreach $stage (@allStages) {
 
             if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;}
@@ -1213 +1261 @@
 
     my $stage = undef;
-    foreach $stage (@stages) {
+    foreach $stage (@allStages) {
 
         $self->optimizeTable($stage);

branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Pantasks.pm

@@ -10 +10 @@
         "addstarlap", 
         "cleanup", 
+        "deepstack",
         "detrend", 
         "distribution", 

branches/eam_branches/ipp-20111122/ippMonitor/czartool/czartool/Plotter.pm

@@ -10 +10 @@
 use czartool::Config;
 
-my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub");
+my @allStages = (
+        "burntool", 
+        "chip", 
+        "cam", 
+        "fake", 
+        "warp", 
+        "stack", 
+        "diff", 
+        "magic", 
+        "magicDS", 
+        "dist", 
+        "pub");
+
+my @allIppToPspsStages = (
+        "processed", 
+        "loaded_to_datastore", 
+        "loaded_to_ODM", 
+        "merge_worthy", 
+        "deleted_datastore", 
+        "deleted_dxlayer", 
+        "merged", 
+        "deleted_local");
+
 
 ###########################################################################
@@ -75 +97 @@
 ###########################################################################
 #
+# Sets plotter to use standard IPP mode, i.e. chip, cam etc 
+#
+###########################################################################
+sub setStandardMode {
+    my ($self) = @_;
+
+    $self->{allStages} = \@allStages;
+}
+
+###########################################################################
+#
+# Sets plotter to use ippToPsps mode, i.e. processed, loaded_to_datastore etc 
+#
+###########################################################################
+sub setIppToPspsMode {
+    my ($self) = @_;
+
+    $self->{allStages} = \@allIppToPspsStages;
+}
+
+###########################################################################
+#
 # Some initialisation code common to all constructors
 #
@@ -83 +127 @@
     $self->{_gpc1Db} = $self->{_config}->getGpc1Instance();
     $self->{_czarDb} = $self->{_config}->getCzarDbInstance();
+    $self->setStandardMode();
 }
 
@@ -140 +185 @@
     my $stages = undef;                 
     if (!$selectedStage) {
-        $stages = \@allStages;
+        $stages = \@{$self->{allStages}};
         $selectedStage = 'all_stages';
     }
@@ -203 +248 @@
         "set style histogram rowstacked;" .
         "set style fill solid border -1;" .
-        "set ylabel \"Exposures processed per $interval\";" .
+        "set ylabel \"Items processed per $interval\";" .
         "set boxwidth 0.75;" .
         "plot ";
 
     my $first = 1;
-    foreach my $stage (@allStages) {
+    foreach my $stage (@{$self->{allStages}}) {
 
 
@@ -226 +271 @@
 ###########################################################################
 #
-# Plots a time series for all stages for this label
+# Plots a time series for all or one stage for this label
 #
 ###########################################################################
@@ -238 +283 @@
     my $stages = undef;                 
 
-    if (!$selectedStage) {$stages = \@allStages;}
+    if (!$selectedStage) {$stages = \@{$self->{allStages}};}
     else {$stages = ["$selectedStage"]};        
 
@@ -297 +342 @@
                 $timeDiff, 
                 "",
-                "Exposures");
+                "Items");
     }
 
@@ -314 +359 @@
                 $timeDiff, 
                 "",
-                "Log(Exposures)");
+                "Log(Items)");
     }
 
@@ -331 +376 @@
                 $timeDiff, 
                 "",
-                "Exposures per hour");
+                "Items per hour");
 }    
 }                                                
@@ -351 +396 @@
     my $stage = undef;
     my $pendingMinusFaults = undef;
-    foreach $stage (@allStages) {
+    foreach $stage (@{$self->{allStages}}) {
 
         if (!$self->{_czarDb}->countProcessedPendingAndFaults(
@@ -381 +426 @@
         "set style histogram rowstacked;" .
         "set style fill solid border -1;" .
-        "set ylabel \"Exposures\";" .
+        "set ylabel \"Items\";" .
         "set boxwidth 0.75;" .
         "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . 
@@ -520 +565 @@
     my $firstIn = 1;
     # loop through stages array so that they are ordered properly (not maintained in hash)
-    foreach my $stage (@allStages) {
+    #foreach my $stage (@allStages) {
+    foreach my $stage (@{$self->{allStages}}) {
 
         if(!$gnuplotFiles->{$stage}) {next;}
@@ -620 +666 @@
         "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" .
         "set style fill solid 1.00 border -1;" .
+        "set key center top;" .
         "set style histogram rowstacked;" .
         "set style data histograms;" .

branches/eam_branches/ipp-20111122/ippMonitor/raw/czartool_labels.php

@@ -27 +27 @@
 menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']);
 
-$states=array("full","new","drop","wait");
-$stages=array("burntool", "chip","cam","fake","warp","stack","diff","magic","magicDS","dist", "pub");
+$stages=array(
+        "burntool", 
+        "chip",
+        "cam",
+        "fake",
+        "warp",
+        "stack",
+        "diff",
+        "magic",
+        "magicDS",
+        "dist", 
+        "pub");
+
 $servers=array(
         "addstar",
@@ -150 +161 @@
 echo "<br>";
 echo $table;
-#"<table width=\"$width\" align=\"middle\" border=\"1\" cellspacing=\"0\" cellpadding=\"4\" style=\"font-size:80%\">\n";
+
 createTableTitle("Status and links", 3);
 echo "<tr>\n";
@@ -180 +191 @@
          $pubLabels, 
          $stages, 
-         $states, 
          "new", 
          $selectedLabel, 
@@ -349 +359 @@
         $pubLabels, 
         $stages, 
-        $states, 
         $selectedState, 
         $selectedLabel, $selectedStage, $plotType) {

branches/eam_branches/ipp-20111122/ippMonitor/raw/ipp.imfiles.dat

@@ -21 +21 @@
 menutop   | menutop      | link    | simple plot - chip (to remove?) | simplePlot.php
 menutop   | menutop      | link    | simple plot - cam (to remove?) | simplePlotcam.php
-menutop   | menutop      | link    | czartool                     | czartool_labels.php
+menutop   | menutop      | link    | czartool - standard          | czartool_labels.php
+menutop   | menutop      | link    | czartool - ipptopsps         | czartool_ipptopsps.php
 menutop   | menutop      | link    | mask stats                   | maskStats.php
 

branches/eam_branches/ipp-20111122/ippScripts/scripts/addstar_run.pl

@@ -37 +37 @@
 }
 my $minidvodb_path;
-my ( $add_id, $camera, $stage, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
+my ( $add_id, $camera, $stage, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update,
      $no_op, $redirect, $save_temps);
 GetOptions(
@@ -43 +43 @@
     'camera|c=s'        => \$camera, # Camera
     'stage|s=s'        => \$stage, # Camera
-    
+    'multi_num|w=s'    => \$multi_num, # the number for a staticskymulti (for finding cmf)
     'dbname|d=s'        => \$dbname, # Database name
     'outroot|w=s'       => \$outroot, # output file base name
@@ -114 +114 @@
     
 }
+
+if ($stage =~/staticsky_multi/) {
+    $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above
+    &my_die( "can't find the filter_num for staticsky_multi, giving up.", $add_id, $PS_EXIT_SYS_ERROR) unless (defined $multi_num);
+    my $nice_num = sprintf ("%03d", $multi_num);
+    $fpaObjects =~ s/cmf$/$nice_num.cmf/;  #this make it look for .001.cmf, etc
+ 
+}
+
 my $traceDest  = $ipprc->filename("TRACE.EXP",          $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR);
 
@@ -190 +199 @@
         $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference
             $command .= " -image" if $image_only;
-            if ($stage = ~/staticsky/) {
+            if ($stage =~ /staticsky/) {
                 $command .= " -accept-astrom ";
-            }
+            }  #careful here - this matches staticsky and staticsky_multi
         my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script
 

branches/eam_branches/ipp-20111122/ippScripts/scripts/ipp_filename.pl

@@ -10 +10 @@
 #print "$ENV{'PATH'}\n";
 #print "$ENV{'PERL5LIB'}\n";
+
+# CZW: This is a horrible hack, but I don't want to have to debug all of ippMonitor to figure out why it's not working.
+unless (defined($ENV{'NEB_SERVER'})) {
+    $ENV{'NEB_SERVER'} = 'http://nebserver.ipp.ifa.hawaii.edu:80/nebulous';
+}
+
 
 use PS::IPP::Config;

branches/eam_branches/ipp-20111122/ippScripts/scripts/magic_destreak.pl

@@ -254 +254 @@
                     if ($skycell->{data_state} eq "full" and $skycell->{quality} == 0) {
                         $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                        my $skycell_id = $skycell->{skycell_id};
                         $diff_skycells{$skycell_id} = $skycell_uri;
                     } else {

branches/eam_branches/ipp-20111122/ippScripts/scripts/register_exp.pl

@@ -121 +121 @@
 my ($data_group,$dist_group,$chip_workdir,$reduction);
 ($label,$data_group,$dist_group,$end_stage,
- $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-dateobs"),
+ $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-exp_name"),
+                                                        &value_for_flag($cmdflags,"-dateobs"),
                                                         &value_for_flag($cmdflags,"-exp_type"),
                                                         &value_for_flag($cmdflags,"-obs_mode"),
@@ -170 +171 @@
 ### Pau.
 sub advance_decisions {
+    my $exp_name = shift;
     my $dateobs = shift;
     my $exptype = shift;
@@ -178 +180 @@
 
     # The strings come pre-quoted.
+    $exp_name =~ s/\'//g;
     $dateobs =~ s/\'//g;
     $exptype =~ s/\'//g;
@@ -188 +191 @@
     my $target;
 
+    if ($exp_name =~ /^c/) {
+        print "Skipping because this is a camera commanded exposure: $exp_name\n";
+        return(undef,undef,undef,"reg",undef,undef,undef);
+    }
+    
     if ($exptype ne 'OBJECT') {
         print "Skipping because exptype doesn't claim to be OBJECT: $exptype\n";

branches/eam_branches/ipp-20111122/ippScripts/scripts/videophot_process.pl

@@ -32 +32 @@
 my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1);
 my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
+my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -116 +117 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
+        check_input_file($uri, $vp_id, $class_id, 'nocell');
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         &my_die("Unable to perform $command: $error_code", $vp_id, $error_code);
@@ -244 +246 @@
 exit 0;
 
+sub check_input_file {
+    my $uri = shift;
+    my $vp_id = shift;
+    my $class_id = shift;
+    my $cell_id = shift;
+    my $resolved = $ipprc->file_resolve($uri);
+
+    my $tryrepair = 0;
+    if (!-e $resolved) {
+        printf STDERR "instance $resolved for $uri does not exist\n";
+        $tryrepair = 1;
+    } elsif (-s $resolved == 0) {
+        printf STDERR "instance $resolved for $uri is empty\n";
+        $tryrepair = 1;
+    }
+    if ($tryrepair) {
+        my $scheme = file_scheme($uri);
+        if ($scheme and ($scheme = 'neb')) {
+            my $command = "$nebrepair $uri";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                my $err_message = "Unable to perform nebrepair";
+                &my_die("$err_message: $error_code", $vp_id, $error_code);
+            }
+       }
+    }
+}
+
 sub my_die
 {

branches/eam_branches/ipp-20111122/ippScripts/scripts/whichimage

@@ -60 +60 @@
 
     my $sign = 1;
-    if ($h < 0) {
+    if ($h =~ /-/) {
         $sign = -1;
         $h = - $h;
@@ -76 +76 @@
 
     my $sign = 1;
-    if ($d < 0) {
+    if ($d =~ /-/) {
         $sign = -1;
         $d = - $d;

branches/eam_branches/ipp-20111122/ippTasks/addstar.pro

@@ -26 +26 @@
     active true
   end
+  task addstar.exp.load.staticsky_multi
+    active true
+  end  
   task addstar.exp.run
     active true
@@ -39 +42 @@
   end
   task addstar.exp.load.staticsky
+    active false
+  end
+  task addstar.exp.load.staticsky_multi
     active false
   end
@@ -175 +181 @@
   end
 end
+
 task           addstar.exp.load.staticsky
   host         local
@@ -229 +236 @@
   end
 end
+
+task           addstar.exp.load.staticsky_multi
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/addstar.exp.log
+
+  task.exec
+   # if ($LABEL:n == 0) break
+    $run = addtool -pendingexp -stage staticsky_multi
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$addstar_DB
+      $run = $run -dbname $DB:$addstar_DB
+      $addstar_DB ++
+      if ($addstar_DB >= $DB:n) set addstar_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook addPendingExp
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup addPendingExp
+  end
+
+  # default exit status
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+
 # run the addstar script on pending exposures
 task           addstar.exp.run
@@ -252 +316 @@
     book getword addPendingExp $pageName exp_tag -var EXP_TAG
     book getword addPendingExp $pageName add_id -var ADD_ID
+    book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1
     book getword addPendingExp $pageName stageroot -var STAGEROOT
     book getword addPendingExp $pageName stage -var STAGE  
@@ -285 +350 @@
         sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
     end
+    if ("$STAGE" == "staticsky_multi")
+        sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
+    end
     if ("$STAGE" == "stack")
         sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
@@ -296 +364 @@
     if ("$REDUCTION" != "NULL")
       $run = $run --reduction $REDUCTION
+    end
+    if ("$STAGE" == "staticsky_multi")
+      $run = $run --multi_num $STAGE_EXTRA1
     end
     if ("$IMAGE_ONLY" == "T")

branches/eam_branches/ipp-20111122/ippTasks/survey.pro

@@ -33 +33 @@
  book create SURVEY_WARP_BG
  book create SURVEY_PUBLISH
- book create SURVEY_STATICSKYSINGLE   
+ book create SURVEY_STATICSKYSINGLE 
  $haveSurveyBooks = TRUE
 end
@@ -48 +48 @@
 $SURVEY_PUBLISH_DB = 0
 $SURVEY_STATICSKYSINGLE_DB = 0
-
 $SURVEY_EXEC = 120
 $SURVEY_POLL = 10
@@ -839 +838 @@
         $run = $run -uncensored
     end
+    if ("$stage" == "staticsky_multi") 
+        #only queue uncensored staticsky (multi filter)
+        $run = $run -uncensored
+    end
+
     if ("$stage" == "stack") 
         #only queue uncensored stacks

branches/eam_branches/ipp-20111122/ippTools/share/Makefile.am

@@ -10 +10 @@
         addtool_find_sky_id_dvo.sql \
         addtool_find_sky_id.sql \
+        addtool_find_sky_id_multi_dvo.sql \
+        addtool_find_sky_id_multi.sql \
         addtool_find_stack_id_dvo.sql \
         addtool_find_stack_id.sql \
@@ -16 +18 @@
         addtool_find_pendingexp_stack.sql \
         addtool_find_pendingexp_staticsky.sql \
+        addtool_find_pendingexp_staticsky_multi.sql \
         addtool_find_pendingmergeprocess.sql \
         addtool_find_processedexp_cam.sql \
         addtool_find_processedexp_stack.sql \
         addtool_find_processedexp_staticsky.sql \
+        addtool_find_processedexp_staticsky_multi.sql \
         addtool_find_minidvodbprocessed.sql \
         addtool_find_minidvodbrun.sql \
@@ -27 +31 @@
         addtool_queue_stack_id.sql \
         addtool_queue_sky_id.sql \
+        addtool_queue_sky_id_multi.sql \
         addtool_queue_minidvodbrun.sql \
         addtool_revertminidvodbprocessed.sql \
@@ -32 +37 @@
         addtool_revertprocessedexp_stack.sql \
         addtool_revertprocessedexp_staticsky.sql \
+        addtool_revertprocessedexp_staticsky_multi.sql \
         bgtool_advancechip.sql \
         bgtool_advancewarp.sql \
@@ -308 +314 @@
         pstamptool_getdependent.sql \
         pstamptool_listjob.sql \
+        pstamptool_listfile.sql \
         pstamptool_pendingcleanup.sql \
         pstamptool_pendingdependent.sql \

branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_cam_id.sql

@@ -4 +4 @@
         'cam',          -- stage
         cam_id,         -- stage_id
+        %d,             -- stage_extra1
         '%s',           -- state
         '%s',           -- workdir

branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_sky_id.sql

@@ -4 +4 @@
         'staticsky',            -- stage
         sky_id,         -- stage_id
+        %d,             -- stage_extra1
         '%s',           -- state
         '%s',           -- workdir

branches/eam_branches/ipp-20111122/ippTools/share/addtool_queue_stack_id.sql

@@ -4 +4 @@
         'stack',                -- stage
         stack_id,         -- stage_id
+        %d,               --stage_extra1
         '%s',           -- state
         '%s',           -- workdir

branches/eam_branches/ipp-20111122/ippTools/share/disttool_listinterests.sql

@@ -9 +9 @@
     stage,
     clean,
+    target_id,
     distTarget.state as target_state
 FROM rcInterest 

branches/eam_branches/ipp-20111122/ippTools/share/magicdstool_getskycells.sql

@@ -5 +5 @@
     diffSkyfile.skycell_id,
     diffSkyfile.path_base,
-    diffSkyfile.data_state
+    diffSkyfile.data_state,
+    diffSkyfile.quality
 FROM magicDSRun
 JOIN magicRun USING(magic_id)
@@ -37 +38 @@
     diffSkyfile.skycell_id,
     diffSkyfile.path_base,
-    diffSkyfile.data_state
+    diffSkyfile.data_state,
+    diffSkyfile.quality
 FROM magicDSRun
 JOIN magicRun USING(magic_id)

branches/eam_branches/ipp-20111122/ippTools/share/pxadmin_create_tables.sql

@@ -1479 +1479 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+CREATE TABLE pstampFile (
+    file_id BIGINT AUTO_INCREMENT,
+    job_id BIGINT NOT NULL,
+    path VARCHAR(255),
+    PRIMARY KEY(file_id),
+    KEY(job_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE pstampWebRequest (
         num BIGINT AUTO_INCREMENT,

branches/eam_branches/ipp-20111122/ippTools/src/addtool.c

@@ -143 +143 @@
     pxAddLabelSearchArgs (config, where, "-filter",     "stackRun.filter", "=="); // define using camRun label
     }
-    
+        if (strcmp(stage, "staticsky_multi")== 0) {
+
+    pxAddLabelSearchArgs (config, where, "-label",     "staticskyRun.label", "=="); // define using camRun label
+    pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label
+    PXOPT_COPY_STR(config->args, where,  "-reduction", "staticskyyRun.reduction", "==");
+    //no filter here
+    }
     if (!psListLength(where->list)) {
         psFree(where);
@@ -237 +243 @@
     }
 
-
+    if (strcmp(stage,"staticsky_multi") == 0) {
+      if (dvodb ) {
+        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage);
+        // find the cam_id of all the exposures that we want to queue up.
+        bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql");
+        // user supplied dvodb
+        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
+    } else {
+      psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage);
+        // find the cam_id of all the exposures that we want to queue up.
+        bare_query = pxDataGet("addtool_find_sky_id_multi.sql");
+        // inherit dvodb from camRun, avoid matching NULL
+        psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)");
+    }
+    }
 
     if (!bare_query) {
@@ -292 +312 @@
         psStringAppend(&query, " GROUP BY stack_id");
       }
-
+    if (strcmp(stage,"staticsky_multi") == 0) {
+      psStringAppend(&query, " GROUP BY sky_id");  //some reason it needs this
+      }
 
 
@@ -405 +427 @@
     }
     }
+    if (strcmp(stage,"staticsky_multi") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        bool status = false;
+        psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs");
+        if (!status) {
+          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs");
+          return false;
+        }
+        if (num_inputs < 0) {
+          psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs");
+          return false;
+        }
+
+        staticskyRunRow *row = staticskyRunObjectFromMetadata(md);
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
+            psFree(output);
+            return false;
+        }
+
+        if (!dvodb) {  //there's no staticsky.dvodb
+            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id);
+            psFree(output);
+            return false;
+        }
+        if (!workdir && !row->workdir) {
+            psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id);
+            psFree(output);
+            return false;
+        }
+
+        psFree(row);
+    }
+    }
+
+
+
 
     // start a transaction so we don't end up with an exp without any associted
@@ -438 +499 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -479 +541 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -520 +583 @@
                                stage,
                                stage_id,
+                               0,
                                workdir     ? workdir   : row->workdir,
                                reduction   ? reduction : row->reduction,
@@ -544 +608 @@
       }
 
+    if (strcmp(stage,"staticsky_multi") == 0) {
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        psS64 stage_id =0; 
+        bool status = false;
+        psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs");
+        if (!status) {
+          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs");
+          return false;
+        }
+        staticskyRunRow *row = staticskyRunObjectFromMetadata(md);
+        stage_id = row->sky_id;
+        
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
+            psFree(output);
+            return false;
+        }
+
+        // queue the exp
+        for (int filter_id = 0; filter_id < num_inputs; filter_id++) {  
+        if (!pxaddQueueByCamID(config,
+                               stage,
+                               stage_id,
+                               filter_id,
+                               workdir     ? workdir   : row->workdir,
+                               reduction   ? reduction : row->reduction,
+                               label       ? label     : row->label,
+                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
+                               dvodb       ? dvodb     : NULL,
+                               note        ? note      : NULL,
+                               image_only,
+                               minidvodb,
+                               minidvodb_group,
+                               minidvodb_name
+        )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error sfg");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
+            psFree(row);
+            psFree(output);
+            return false;
+        }
+        }
+        psFree(row);
+    }
+      }
+
+
+
+
     psFree(output);
 
@@ -585 +702 @@
     }
     if (strcmp(stage, "staticsky")==0) {
+    query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
+    }
+    if (strcmp(stage, "staticsky_multi")==0) {
     query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
     }
@@ -626 +746 @@
     query = pxDataGet("addtool_find_pendingexp_staticsky.sql");
     }
-
+    if (strcmp(stage, "staticsky_multi")==0) { 
+    query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql");
+    }
+    
 
     if (!query) {
@@ -648 +771 @@
       psStringAppend(&query, " GROUP BY %s", "sky_id");
     }
-
+    if (strcmp(stage, "staticsky_multi") == 0) {
+      //this group by is needed to join against all the warps (to get camera)
+      psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1");
+    }
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -838 +964 @@
     query = pxDataGet("addtool_find_processedexp_staticsky.sql");
     }
+    if (strcmp (stage,"staticsky_multi") == 0) {
+    query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql");
+    }
+
    
     if (!query) {
@@ -948 +1078 @@
          query = pxDataGet("addtool_revertprocessedexp_staticsky.sql");
       }
+   if (strcmp(stage, "staticsky_multi") == 0) {
+         query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql");
+      }
+
+
+     
       if (!query) {
             // rollback

branches/eam_branches/ipp-20111122/ippTools/src/disttool.c

@@ -419 +419 @@
     }
 
-    if (!strcmp(stage, "sky") && single) {
-        psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1");
+    if (!strcmp(stage, "sky")) {
+        if (single) {
+            psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1");
+        } else {
+            psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) > 1");
+        }
     }
 

branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c

@@ -666 +666 @@
                 "cam",
                 row->cam_id,
+                0,
                 row->workdir,
                 row->reduction,

branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c

@@ -56 +56 @@
 static bool revertdependentMode(pxConfig *config);
 static bool getwebrequestnumMode(pxConfig *config);
+static bool addfileMode(pxConfig *config);
+static bool listfileMode(pxConfig *config);
+static bool deletefileMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -103 +106 @@
         MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode);
         MODECASE(PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumMode);
+        MODECASE(PSTAMPTOOL_MODE_ADDFILE, addfileMode);
+        MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode);
+        MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -1476 +1482 @@
     return true;
 }
+
+static bool addfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false);
+    PXOPT_LOOKUP_STR(path,   config->args, "-path",   true, false);
+
+    if (!pstampFileInsert(config->dbh,
+            0, // file_id
+            job_id,
+            path
+            )) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    if (affected != 1) {
+        psError(PS_ERR_UNKNOWN, false,
+            "should have affected one row but %" PRIu64 " rows were modified",
+            affected);
+        return false;
+    }
+
+    psS64 file_id = psDBLastInsertID(config->dbh);
+    printf("%" PRId64 "\n", file_id);
+
+    return true;
+}
+
+static bool listfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-file_id", "file_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    if (!psListLength(where->list)) {
+        fprintf(stderr, "search arguments are required\n");
+        exit (1);
+    }
+
+    psString query = pxDataGet("pstamptool_listfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("pstamptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "pstampFile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool deletefileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false);
+
+    psString query = NULL; 
+    psStringAppend(&query, "DELETE FROM pstampFile WHERE job_id = %" PRId64, job_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("pstamptool", PS_LOG_INFO, "Deleted %" PRIu64 " rows from pstampFile", affected);
+
+    return true;
+}

branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h

@@ -50 +50 @@
     PSTAMPTOOL_MODE_REVERTDEPENDENT,
     PSTAMPTOOL_MODE_GETWEBREQUESTNUM,
+    PSTAMPTOOL_MODE_ADDFILE,
+    PSTAMPTOOL_MODE_LISTFILE,
+    PSTAMPTOOL_MODE_DELETEFILE,
 } pstamptoolMode;
 

branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c

@@ -274 +274 @@
     psMetadata *modes = psMetadataAlloc();
 
+    // -addfile
+    psMetadata *addfileArgs = psMetadataAlloc();
+    psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-job_id",       0, "define job ID for file (required)", 0);
+    psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-path",         0, "define path for file (required)", NULL);
+
+    // -listfile
+    psMetadata *listfileArgs = psMetadataAlloc();
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-file_id",     0, "select by file ID", 0);
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-job_id",      0, "select by job ID", 0);
+    psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-req_id",      0, "select by request ID", 0);
+    psMetadataAddU64(listfileArgs, PS_LIST_TAIL, "-limit",       0, "limit result set to N items", 0);
+    psMetadataAddBool(listfileArgs, PS_LIST_TAIL, "-simple",     0, "use the simple output format", false);
+
+    // -deletefile
+    psMetadata *deletefileArgs = psMetadataAlloc();
+    psMetadataAddS64(deletefileArgs, PS_LIST_TAIL, "-job_id",      0, "select by job ID (required)", 0);
+
     PXOPT_ADD_MODE("-addreq",          "", PSTAMPTOOL_MODE_ADDREQ,       addreqArgs);
     PXOPT_ADD_MODE("-pendingreq",      "", PSTAMPTOOL_MODE_PENDINGREQ,   pendingreqArgs);
@@ -302 +319 @@
     PXOPT_ADD_MODE("-project",         "", PSTAMPTOOL_MODE_PROJECT,    projectArgs);
     PXOPT_ADD_MODE("-getwebrequestnum","", PSTAMPTOOL_MODE_GETWEBREQUESTNUM,   getwebrequestnumArgs);
+    PXOPT_ADD_MODE("-addfile",         "", PSTAMPTOOL_MODE_ADDFILE,   addfileArgs);
+    PXOPT_ADD_MODE("-listfile",        "", PSTAMPTOOL_MODE_LISTFILE,   listfileArgs);
+    PXOPT_ADD_MODE("-deletefile",      "", PSTAMPTOOL_MODE_DELETEFILE,   deletefileArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c

@@ -131 +131 @@
                        char *stage,
                        psS64 stage_id,
+                       psS32 stage_extra1,
                        char *workdir,
                        char *reduction,
@@ -159 +160 @@
         psMemSetPersistent(query, true);
       }
+      if (strcmp(stage,"staticsky_multi") == 0) {
+        query = pxDataGet("addtool_queue_sky_id_multi.sql");
+        psMemSetPersistent(query, true);
+      }
         if (!query) {
             psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -170 +175 @@
     // on 32/64
     if (!p_psDBRunQueryF(config->dbh, query,
+                         stage_extra1 ? stage_extra1 : 0,
                          "new", // state
                          workdir  ? workdir   : "NULL",

branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h

@@ -33 +33 @@
                        char *stage,
                        psS64 stage_id,
+                       psS32 stage_extra1,
                        char *workdir,
                        char *reduction,

branches/eam_branches/ipp-20111122/ippTools/src/vptool.c

@@ -178 +178 @@
     PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
-    psMetadata *where = psMetadataAlloc();
-    pxchipGetSearchArgs (config, where); // rawExp only
-    pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label
-
-    if (!psListLength(where->list)) {
-        psFree(where);
-        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
-        return false;
-    }
 
     PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false);
@@ -195 +186 @@
 
     // default
-    PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false);
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+    pxchipGetSearchArgs (config, where); // rawExp only
+    pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "="); // define using newExp label
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     // find the exp_id of all the exposures that we want to queue up.
@@ -213 +214 @@
     psFree(where);
 
-    if (unique) {
-      psStringAppend(&query, "AND vp_id IS NULL");
+    if (!rerun) {
+      psStringAppend(&query, "\n  AND vp_id IS NULL\n  AND (vpRun.label = '%s' OR vpRun.label IS NULL)", label);
     }
     

branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c

@@ -50 +50 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by newExp label (LIKE comparison)", NULL);
 
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",  0,            "define label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",  0,            "define label (required)", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group",  0,      "define data group", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir",  0,            "define workdir (required)", NULL);
@@ -56 +56 @@
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-set_dest_id",  0,      "define destination", 0);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend",  0,            "do not actually modify the database", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique",   0,           "only queue exposures that have no previous vpRun", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun",   0,           "queue exposures even if a previous vpRun exists", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     // -pendingrun