Changeset 37242

Timestamp:

Aug 12, 2014, 3:55:38 PM (12 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20140717

Files:

• ippMonitor/czartool/czartool/Plotter.pm (modified) (2 diffs, 1 prop)
• ippScripts/Build.PL (modified) (1 diff)
• ippScripts/MANIFEST (modified) (1 diff)
• ippScripts/scripts/dist_defineruns.pl (modified) (1 diff)
• ippScripts/scripts/lap_science.pl (modified) (6 diffs)
• ippScripts/scripts/permcheck.pl (modified) (3 diffs)
• ippScripts/scripts/queuestaticsky.pl (modified) (1 prop)
• ippScripts/scripts/sc_prepare_camera.pl (modified) (6 diffs)
• ippScripts/scripts/sc_prepare_chip.pl (modified) (4 diffs)
• ippScripts/scripts/sc_prepare_stack.pl (modified) (4 diffs)
• ippScripts/scripts/sc_prepare_warp.pl (modified) (5 diffs)
• ippScripts/scripts/sc_remote_exec.pl (modified) (16 diffs, 1 prop)
• ippScripts/scripts/sc_transfer_tool.pl (modified) (5 diffs)
• ippScripts/scripts/sc_validate_files.pl (deleted)
• ippScripts/scripts/sc_validate_processing.pl (deleted)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/releasetool_deleterelexp.sql (copied) (copied from trunk/ippTools/share/releasetool_deleterelexp.sql )
• ippTools/share/releasetool_deleterelstack.sql (copied) (copied from trunk/ippTools/share/releasetool_deleterelstack.sql )
• ippTools/share/releasetool_setrelstackcalibratedfromskycal.sql (copied) (copied from trunk/ippTools/share/releasetool_setrelstackcalibratedfromskycal.sql )
• ippTools/src/disttool.c (modified) (1 diff)
• ippTools/src/disttoolConfig.c (modified) (1 diff)
• ippTools/src/releasetool.c (modified) (5 diffs)
• ippTools/src/releasetool.h (modified) (1 diff)
• ippTools/src/releasetoolConfig.c (modified) (4 diffs)
• ippTools/src/stacktool.c (modified) (1 diff)
• ippTools/src/stacktoolConfig.c (modified) (1 diff)
• ippTools/src/staticskytool.c (modified) (3 diffs)
• ippTools/src/staticskytoolConfig.c (modified) (2 diffs)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1/psastro.config (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)
• psModules (modified) (1 prop)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (1 prop)
• pstamp/scripts/pstamp_finish.pl (modified) (1 prop)
• pstamp/scripts/pstampparse.pl (modified) (1 prop)

branches/eam_branches/ipp-20140717/ippMonitor/czartool/czartool/Plotter.pm

@@ -671 +671 @@
         "set ylabel \"Space (TB)\";" .
         "set xtic rotate by -90 scale 0;" .
+        "set yrange [:100];" .
         "plot '".$tmpFile->filename."' " .
         "using 2:xtic(1) t \"Used\" lt 7," .
@@ -715 +716 @@
         "set style data histograms;" .
         "set ylabel \"Space (TB)\";" .
+        "set yrange [:100];" .
         "set xtic rotate by -90 scale 0;" .
         "plot '".$tmpFile->filename."' " .

branches/eam_branches/ipp-20140717/ippScripts/Build.PL

@@ -143 +143 @@
         scripts/sc_remote_exec.pl
         scripts/sc_transfer_tool.pl
-        scripts/sc_validate_files.pl
-        scripts/sc_validate_processing.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',

branches/eam_branches/ipp-20140717/ippScripts/MANIFEST

@@ -56 +56 @@
 scripts/sc_remote_exec.pl
 scripts/sc_transfer_tool.pl
-scripts/sc_validate_files.pl
-scripts/sc_validate_processing.pl
 t/00_distribution.t

branches/eam_branches/ipp-20140717/ippScripts/scripts/dist_defineruns.pl

@@ -90 +90 @@
     push @stages, $stage;
 } else {
-    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle ff);
+    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle skycal ff);
 }
 

branches/eam_branches/ipp-20140717/ippScripts/scripts/lap_science.pl

@@ -34 +34 @@
 my $qstack_threshold     = 0.05; # Only make a quickstack if more than 5% of the exposures require it.
 my $minimum_stack_inputs = 2;    # We can avoid magicking stack inputs if we have more than this number.
-my $do_updates_pv2       = 1;    # Do updates using the calls needed for PV2.  
+my $maximum_stack_inputs = 25;   # This is the place-holder guess on how big a stack we should process locally
+my $do_updates_pv2       = 0;    # Do updates using the calls needed for PV2.  
 
 GetOptions(
@@ -189 +190 @@
     $chip_cmd .= " -set_label $exposure->{label} -set_data_group $data_group ";
     $chip_cmd .= " -set_workdir $workdir -set_dist_group $exposure->{dist_group} ";
-#    $chip_cmd .= " -set_reduction LAP_SCIENCE ";
-    $chip_cmd .= " -set_reduction LAP_POLE ";
+    $chip_cmd .= " -set_reduction LAP_SCIENCE ";
+#    $chip_cmd .= " -set_reduction LAP_POLE ";
 
     my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -1037 +1038 @@
     my $data_group = "${label}.${proj_cell}.final.${lap_id}";
 
+
+    ##
+    ## Queue the "shallow" stacks that will run at los alamos.
     my $command = "$stacktool ";
     $command .= " -pretend " if defined $debug;
@@ -1043 +1047 @@
 #    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
     $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group ";
-    $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
+    $command .= " -min_num ${minimum_stack_inputs} -max_num ${maximum_stack_inputs} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
     $command .= " $warps ";
 
@@ -1054 +1058 @@
     my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, "");    
-    
+
+    ##
+    ## Queue the "deep" stacks that will run locally.
+    my $deep_minimum = $maximum_stack_inputs + 1;
+    $command = "$stacktool ";
+    $command .= " -pretend " if defined $debug;
+    $command .= " -dbname $dbname " if defined $dbname;
+    $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter ";
+#    $command .= " -select_label $label "; # Removed to allow exposure sharing to work
+    $command .= " -set_label ${label}.local -set_workdir $workdir -set_data_group $data_group ";
+    $command .= " -min_num ${deep_minimum} -set_reduction THREEPI_STACK_1DG -set_dist_group $exposure->{dist_group} ";
+    $command .= " $warps ";
+
+    ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id);
+    }
+    $stacks_made .= $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, "");    
+
+
+    ## Determine the sass_id so we can record it in the database.
     $command = "$stacktool ";
     $command .= " -dbname $dbname " if defined $dbname;
@@ -1416 +1443 @@
 
     my @clean_modes = (
-        'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
+#       'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
 #       'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
         'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',

branches/eam_branches/ipp-20140717/ippScripts/scripts/permcheck.pl

@@ -51 +51 @@
 my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
                     'ippb02' => 1, 'ippb03' => 1,
-                    'ippb04' => 1, 'ippb05' => 1
+                    'ippb04' => 1, 'ippb05' => 1,
+                    'ippb06' => 1
     );
-my %backup_destinations = ('ippb04' => 1,
-                           'ippb05' => 1);
+my %backup_destinations = (#'ippb04' => 1,
+                           #'ippb05' => 1,
+                           'ippb06' => 1
+);
 my $backup_Nvols = 3;
 
@@ -111 +114 @@
 }
 
-my %components = ('camera' => ['PSASTRO.OUTPUT'],
+my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
                   'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
                                'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
@@ -122 +125 @@
     my $hostname = $entry->{hostname};
     foreach my $product (@{ $components{$stage} }) {
-        my $key = $ipprc->filename($product,$path_base);
-
+        my @keys = ();
+        if (($stage eq 'camera')&&($product eq 'PSASTRO.OUTPUT.MASK')) {
+            my @otas = ('XY01','XY02','XY03','XY04','XY05','XY06',
+                        'XY10','XY11','XY12','XY13','XY14','XY15','XY16','XY17',
+                        'XY20','XY21','XY22','XY23','XY24','XY25','XY26','XY27',
+                        'XY30','XY31','XY32','XY33','XY34','XY35','XY36','XY37',
+                        'XY40','XY41','XY42','XY43','XY44','XY45','XY46','XY47',
+                        'XY50','XY51','XY52','XY53','XY54','XY55','XY56','XY57',
+                        'XY60','XY61','XY62','XY63','XY64','XY65','XY66','XY67',
+                        'XY71','XY72','XY73','XY74','XY75','XY76');
+            foreach my $ota (@otas) {
+                push @keys, $ipprc->filename($product,$path_base,$ota);
+            }       
+        }
+        else {
+            push @keys, $ipprc->filename($product,$path_base);
+        }
 # Do validation
-
-        # neb-stat level handling
-        my $stat = $neb->stat($key);
-        die "nebulous key: $key not found" unless $stat;   
-        my $instances = $neb->find_instances($key, 'any');
-        die "no instances found" unless $instances;   
         
-        my $user_copies;
-        eval {
-            $user_copies = $neb->getxattr($key, "user.copies");
-        };
-        unless(defined($user_copies)) {
-            $user_copies = 1;
-        }
-
-        my $md5sum;
-        my @validation;
-        my %md5sum_uniq;
-        my $existing_copies = 0;
-        
-        my @existance;
-        my @md5sums;
-        my @diskfiles = map {URI->new($_)->file if $_} @$instances;
-        my @diskvols;
-        my @diskhosts;
-        my @quality;
-        my $Ngood = 0;
-        my $quality_mask = 0;
-        
-        for (my $i = 0; $i <= $#diskfiles; $i++) {
-            if (-e $diskfiles[$i]) {
-                $existance[$i] = 1;
-                $existing_copies++;
-                $md5sums[$i] = local_md5sum($diskfiles[$i]);
-                $md5sum_uniq{$md5sums[$i]} = 1;
-            }
-            else {
-                $existance[$i] = 0;
-                $md5sums[$i] = 'NON-EXISTANT';
-                $md5sum_uniq{$md5sums[$i]} = 1;
-            }
-            ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
-            $validation[$i] = sprintf("% 3d %32s %s %s %d",
-                                      $existance[$i],
-                                      $md5sums[$i],
-                                      $diskfiles[$i],
-                                      $diskhosts[$i],$diskvols[$i]
-            );
-            
-            # Pre-parse decisions
-            if ($existance[$i] == 0) {
-                $quality[$i] = 0;
-            }
-            elsif (is_backup_volume($diskhosts[$i])) {
-                $quality[$i] = 1;
-                $quality_mask = $quality_mask | 1;
-                $Ngood++;
-            }
-            else {
-                $quality[$i] = 2;
-                $Ngood++;
-            }
-        }
-        if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
-            die "There are multiple md5sum values for $key";
-        }
-        $md5sum = (keys(%md5sum_uniq))[0];
-
-        print "\n$key $data_state $md5sum $hostname\n";
-        if (1) {
-            
-            print
-                "object id:             ", @$stat[0], "\n",
-                "key:                   ", @$stat[1], "\n";
-            print 
-                "epoch:                 ", @$stat[4], "\n";
-            print 
-                "md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
-            print 
-                "requested instances:   ", $user_copies, "\n",
-                "available instances:   ", @$stat[6], "\n",
-                "existing instances:    ", $existing_copies, "\n",
-                "total instances:       ", @$stat[7], "\n",
-                "instance location:\n", " " x 4;
-            print
-                join("\n" . " " x 4, @validation), "\n";
-        }
-
-        # Decide what to do
-        if ($Ngood == 0) {
-            # DO something to attempt to fix this.
-            my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
-            
-            open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
-            my $good_file = '';
-            while (<DD>) {
-                $_ =~ s/^\s+//;
-                my ($z,undef,$ff) = split /\s+/;
-                if (($ff)&&(-e $ff)) {
-                    my $md_response = `md5sum $ff`;
-                    if ($md_response =~ /$md5sum/) {
-                        $good_file = (split /\s+/,$md_response)[1];
-                    }
-                }
-            }
-            close(DD);
-            if ($good_file eq '') {
-                die "No valid instance of key: $key";
-            }
-            else {
-                $quality[0] = 1;
-                print "cp $good_file $diskfiles[0]\n";
-                system("cp $good_file $diskfiles[0]");
-            }
-            # Begin my best validation thought
-            {
-                my $tmpmd5 = local_md5sum($diskfiles[0]);
-                if ($tmpmd5 ne $md5sum) { 
-                    die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                }
-            }
-            # End my best validation thought.
-            
-            $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
-        }
-    
-        if ($quality[0] == 0) { # The first instance is bad.
-            # But since we're here, and not up there, there must be at least one good copy.  Find it.
-            for (my $i = 0; $i <= $#md5sums; $i++) {
-                if ($md5sums[$i] eq $md5sum) { # Found it.
-                    print "cp $diskfiles[$i] $diskfiles[0]\n";
-                    system("cp $diskfiles[$i] $diskfiles[0]");
-                    # Begin my best validation thought
-                    {
-                        my $tmpmd5 = local_md5sum($diskfiles[0]);
+        foreach my $key (@keys) {
+            # neb-stat level handling
+            my $stat = $neb->stat($key);
+            die "nebulous key: $key not found" unless $stat;   
+            my $instances = $neb->find_instances($key, 'any');
+            die "no instances found" unless $instances;   
+            
+            my $user_copies;
+            eval {
+                $user_copies = $neb->getxattr($key, "user.copies");
+            };
+            unless(defined($user_copies)) {
+                $user_copies = 1;
+            }
+            
+            my $md5sum;
+            my @validation;
+            my %md5sum_uniq;
+            my $existing_copies = 0;
+            
+            my @existance;
+            my @md5sums;
+            my @diskfiles = map {URI->new($_)->file if $_} @$instances;
+            my @diskvols;
+            my @diskhosts;
+            my @quality;
+            my $Ngood = 0;
+            my $quality_mask = 0;
+            
+            for (my $i = 0; $i <= $#diskfiles; $i++) {
+                if (-e $diskfiles[$i]) {
+                    $existance[$i] = 1;
+                    $existing_copies++;
+                    $md5sums[$i] = local_md5sum($diskfiles[$i]);
+                    $md5sum_uniq{$md5sums[$i]} = 1;
+                }
+                else {
+                    $existance[$i] = 0;
+                    $md5sums[$i] = 'NON-EXISTANT';
+                    $md5sum_uniq{$md5sums[$i]} = 1;
+                }
+                ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]);
+                $validation[$i] = sprintf("% 3d %32s %s %s %d",
+                                          $existance[$i],
+                                          $md5sums[$i],
+                                          $diskfiles[$i],
+                                          $diskhosts[$i],$diskvols[$i]
+                    );
+                
+                # Pre-parse decisions
+                if ($existance[$i] == 0) {
+                    $quality[$i] = 0;
+                }
+                elsif (is_backup_volume($diskhosts[$i])) {
+                    $quality[$i] = 1;
+                    $quality_mask = $quality_mask | 1;
+                    $Ngood++;
+                }
+                else {
+                    $quality[$i] = 2;
+                    $Ngood++;
+                }
+            }
+            if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth.
+                die "There are multiple md5sum values for $key";
+            }
+            $md5sum = (keys(%md5sum_uniq))[0];
+            
+            print "\n$key $data_state $md5sum $hostname\n";
+            if (1) {
+                
+                print
+                    "object id:             ", @$stat[0], "\n",
+                    "key:                   ", @$stat[1], "\n";
+                print 
+                    "epoch:                 ", @$stat[4], "\n";
+                print 
+                    "md5sum count:          ", scalar(keys %md5sum_uniq), "\n";
+                print 
+                    "requested instances:   ", $user_copies, "\n",
+                    "available instances:   ", @$stat[6], "\n",
+                    "existing instances:    ", $existing_copies, "\n",
+                    "total instances:       ", @$stat[7], "\n",
+                    "instance location:\n", " " x 4;
+                print
+                    join("\n" . " " x 4, @validation), "\n";
+            }
+            
+            # Decide what to do
+            if ($Ngood == 0) {
+                # DO something to attempt to fix this.
+                my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/;
+                
+                open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |");
+                my $good_file = '';
+                while (<DD>) {
+                    $_ =~ s/^\s+//;
+                    my ($z,undef,$ff) = split /\s+/;
+                    if (($ff)&&(-e $ff)) {
+                        my $md_response = `md5sum $ff`;
+                        if ($md_response =~ /$md5sum/) {
+                            $good_file = (split /\s+/,$md_response)[1];
+                        }
+                    }
+                }
+                close(DD);
+                if ($good_file eq '') {
+                    die "No valid instance of key: $key";
+                }
+                else {
+                    $quality[0] = 1;
+                    print "cp $good_file $diskfiles[0]\n";
+                    system("cp $good_file $diskfiles[0]");
+                }
+                # Begin my best validation thought
+                {
+                    my $tmpmd5 = local_md5sum($diskfiles[0]);
+                    if ($tmpmd5 ne $md5sum) { 
+                        die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+                    }
+                }
+                # End my best validation thought.
+                
+                $Ngood = 1;  # We now hand off this single valid instance object to be handled by the Ngood=1 case.
+            }
+            
+            if ($quality[0] == 0) { # The first instance is bad.
+                # But since we're here, and not up there, there must be at least one good copy.  Find it.
+                for (my $i = 0; $i <= $#md5sums; $i++) {
+                    if ($md5sums[$i] eq $md5sum) { # Found it.
+                        print "cp $diskfiles[$i] $diskfiles[0]\n";
+                        system("cp $diskfiles[$i] $diskfiles[0]");
+                        # Begin my best validation thought
+                        {
+                            my $tmpmd5 = local_md5sum($diskfiles[0]);
+                            if ($tmpmd5 ne $md5sum) { 
+                                die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+                            }
+                        }
+                        # End my best validation thought.
+                        last; # We're done here now.
+                    }
+                }       
+            }
+            
+            if ($Ngood == 1) { # We have only one version
+                if ($quality_mask & 1) { # ANd it's on a backup volume
+                    print "neb-replicate $key\n";
+                    if ($do_ops) {
+                        $neb->replicate($key) or die "failed to replicate the single valid copy";
+                        if ($@) { die $@; }
+                    }
+                }
+                else { # And it's not, so put one there
+                    my $rep_vol = get_random_backup_volume();
+                    print "neb-replicate --volume $rep_vol  $key\n";
+                    if ($do_ops) {
+                        $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
+                        if ($@) { die $@; }
+                        
+                        # Begin my best validation thought
+                        system("sync") == 0 or die "Couldn't sync?";
+                        my $uris = $neb->find_instances($key,$rep_vol);
+                        @$uris = map {URI->new($_)->file if $_} @$uris;
+                        my $tmpmd5 = local_md5sum(${ $uris }[0]);
                         if ($tmpmd5 ne $md5sum) { 
                             die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
                         }
-                    }
-                    # End my best validation thought.
-                    last; # We're done here now.
-                }
-            }   
-        }
-        
-        if ($Ngood == 1) { # We have only one version
-            if ($quality_mask & 1) { # ANd it's on a backup volume
-                print "neb-replicate $key\n";
-                if ($do_ops) {
-                    $neb->replicate($key) or die "failed to replicate the single valid copy";
-                    if ($@) { die $@; }
-                }
-            }
-            else { # And it's not, so put one there
-                my $rep_vol = get_random_backup_volume();
-                print "neb-replicate --volume $rep_vol  $key\n";
-                if ($do_ops) {
-                    $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node";
-                    if ($@) { die $@; }
-                    
-                    # Begin my best validation thought
-                    system("sync") == 0 or die "Couldn't sync?";
-                    my $uris = $neb->find_instances($key,$rep_vol);
-                    @$uris = map {URI->new($_)->file if $_} @$uris;
-                    my $tmpmd5 = local_md5sum(${ $uris }[0]);
-                    if ($tmpmd5 ne $md5sum) { 
-                        die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                    }
-                    # End my best validation thought.
-                }           
-            }
-        }
-        else { # N >= 2
-            if (!($quality_mask & 1)) { # no backup copy
-                my $rep_vol = get_random_backup_volume();
-                print "neb-replicate --volume $rep_vol  $key\n";
-                if ($do_ops) {
-                    $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
-                    if ($@) { die $@; }
-                    
-                    # Begin my best validation thought
-                    system("sync") == 0 or die "Couldn't sync?";
-                    my $uris = $neb->find_instances($key,$rep_vol);
-                    @$uris = map {URI->new($_)->file if $_} @$uris;
-                    my $tmpmd5 = local_md5sum(${ $uris }[0]);
-                    if ($tmpmd5 ne $md5sum) { 
-                        die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
-                    }
-                    # End my best validation thought.
-                }           
-            }
-            if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
+                        # End my best validation thought.
+                    }       
+                }
+            }
+            else { # N >= 2
+                if (!($quality_mask & 1)) { # no backup copy
+                    my $rep_vol = get_random_backup_volume();
+                    print "neb-replicate --volume $rep_vol  $key\n";
+                    if ($do_ops) {
+                        $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node";
+                        if ($@) { die $@; }
+                        
+                        # Begin my best validation thought
+                        system("sync") == 0 or die "Couldn't sync?";
+                        my $uris = $neb->find_instances($key,$rep_vol);
+                        @$uris = map {URI->new($_)->file if $_} @$uris;
+                        my $tmpmd5 = local_md5sum(${ $uris }[0]);
+                        if ($tmpmd5 ne $md5sum) { 
+                            die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
+                        }
+                        # End my best validation thought.
+                    }       
+                }
+                if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid.
+                    for (my $i = 0; $i <= $#diskfiles; $i++) {
+                        if ($quality[$i] == 2) {
+                            $quality[$i] = 1;
+                            last;
+                        }
+                    }
+                }
                 for (my $i = 0; $i <= $#diskfiles; $i++) {
-                    if ($quality[$i] == 2) {
-                        $quality[$i] = 1;
-                        last;
-                    }
-                }
-            }
-            for (my $i = 0; $i <= $#diskfiles; $i++) {
 #           print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
-                if ($quality[$i] != 1) {
-                    if ($existance[$i] == 0) { # This disk file doesn't exist.
-                        system("touch $diskfiles[$i]");
-                    }
-                    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
-                    print "neb-cull --volume $cull_vol $key\n";
-                    if ($do_ops) {
-                        # The tilde here is to force hard volumes.  Don't touch it.
-                        $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
-                        if ($@) { die "$@"; }
-                    }
-                }
-            }
-        }
+                    if ($quality[$i] != 1) {
+                        if ($existance[$i] == 0) { # This disk file doesn't exist.
+                            system("touch $diskfiles[$i]");
+                        }
+                        my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
+                        print "neb-cull --volume $cull_vol $key\n";
+                        if ($do_ops) {
+                            # The tilde here is to force hard volumes.  Don't touch it.
+                            $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
+                            if ($@) { die "$@"; }
+                        }
+                    }
+                }
+            }
+        } # end keys for this product
     } # end product for this entry
 } # end entry for this id

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_prepare_camera.pl

@@ -18 +18 @@
 
 # Hard coded values
-my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+$remote_root = '/scratch3/watersc1/';
 my $threads       = 2;                      # How many threads are we going to use?
 my $job_cost      = 1700 / 60 / 60;         # Estimate of how long a job runs, in hours.
@@ -27 +29 @@
 my $max_time      = 8;                      # longest allocation to ask for
 my $remote_hostname= "LANL/Mustang";         # Name of the remote node.
-my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP","TRACE.EXP", "PSASTRO.STATS");
+my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP", "PSASTRO.STATS");
 # Look for programs we need
 my $missing_tools;
@@ -186 +188 @@
     $psastro_command   .= " -recipe PSASTRO $recipe_psastro ";
     $psastro_command   .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
-    $psastro_command   .= " -dumpconfig ${remote_outroot}.mdc -stats ${remote_outroot}.stats ";
+    $psastro_command   .= " -dumpconfig ${remote_outroot}.psastro.mdc -stats ${remote_outroot}.stats ";
     $psastro_command   .= " -recipe PPSTATS CAMSTATS ";
     
@@ -202 +204 @@
         my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
         my $remote_outroot_dir = dirname($ipp_disk);
-        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} ";
+        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
     }
     foreach my $chipInfo (@{ $inpData2 }) {
@@ -208 +210 @@
         my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
         my $remote_outroot_dir = dirname($ipp_disk);
-        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} ";
+        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
     }
     print CONFIG "\n";
@@ -239 +241 @@
     $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
 }
+$time_req += 1; # Safety addition.
 
 open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR);

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_prepare_chip.pl

@@ -18 +18 @@
 
 # Hard coded values
-my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+$remote_root = '/scratch3/watersc1/';
 my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
 my $threads       = 2;                      # How many threads are we going to use?
@@ -152 +154 @@
 #print Dumper(%detrends);
 
-my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","TRACE.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
+my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
 
 #
@@ -348 +350 @@
             my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
             my $remote_outroot_dir = dirname($ipp_disk);
-            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} ";
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
 #           print "  $filename $ipp_disk $remote_disk\n";
 #           die;
@@ -405 +407 @@
     $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
 }
+$time_req += 1; # Safety addition.
 
 open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR);

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_prepare_stack.pl

@@ -18 +18 @@
 
 # Hard coded values
-my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+#my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+$remote_root = '/scratch3/watersc1/';
 my $threads       = 2;                      # How many threads are we going to use?
 my $job_cost      = 1500 / 60 / 60;           # Estimate of how long a job runs, in hours.
@@ -29 +31 @@
 my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE",
                              "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT",
-                             "LOG.EXP","TRACE.EXP", "PSASTRO.STATS",
-                             "PPSTACK.CONFIG","PPSTACK.TARGET.PSF");
+                             "LOG.EXP", "PSASTRO.STATS",
+                             "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE");
 #,"PPSTACK.CONV.KERNEL");
 # Look for programs we need
@@ -147 +149 @@
     # This actually returns all the individual stack/skyfiles that comprise this run.  Because consistency.
     my $command = "$stacktool -inputskyfile -stack_id $stack_id ";
-
+    $command .= " -dbname $dbname " if defined($dbname);
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 0);
@@ -208 +210 @@
         my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
         my $remote_outroot_dir = dirname($ipp_disk);
-        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} ";
+        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
     }
 

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_prepare_warp.pl

@@ -18 +18 @@
 
 # Hard coded values
-my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
+# my $remote_root   = '/scratch3/watersc1/';  
+my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+$remote_root = '/scratch3/watersc1/';
 my $threads       = 2;                      # How many threads are we going to use?
-my $job_cost      = 70 / 60 / 60;           # Estimate of how long a job runs, in hours.
+my $job_cost      = 110 / 60 / 60;          # Estimate of how long a job runs, in hours.
 my $proc_per_node = 24;                     # processors per node
 my $min_nodes     = 1;                      # smallest allocation to ask for
@@ -27 +29 @@
 my $max_time      = 8;                      # longest allocation to ask for
 my $remote_hostname= "LANL/Mustang";         # Name of the remote node.
-my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SAVE","LOG.EXP","TRACE.EXP");
+my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
 # Look for programs we need
 my $missing_tools;
@@ -202 +204 @@
         $pswarp_command    .= " -threads 4 "; # -image_id ${image_id} -source_id ${source_id} ";
         $pswarp_command    .= " -recipe PPSTATS WARPSTATS ";
-        $pswarp_command    .= " -dumpconfig ${remote_outroot}.mdc -stats ${remote_outroot}.stats ";
+        $pswarp_command    .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats ";
 
 #       print "$pswarp_command \n";
@@ -218 +220 @@
         # Determine which output files need to be returned
         foreach my $component(@return_component_list) {
-            my $filename = $ipprc->filename($component,$ipp_outroot);
+            my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id);
             my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
             my $remote_outroot_dir = dirname($ipp_disk);
-            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} ";
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
         }
 
@@ -255 +257 @@
     $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
 }
+$time_req += 1; # Safety addition.
 
 open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR);

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_remote_exec.pl

@@ -16 +16 @@
 # Hard coded values
 my $DMZ_HOST = 'wtrw';
-my $SEC_HOST = 'mu-fe4';
+my $SEC_HOST = 'mu-fe';
 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
 my $remote_root  = '/scratch3/watersc1/';
@@ -31 +31 @@
 }
 
-
 # Options
-my ($remote_id,$path_base,$policy,$poll,$job_id,$dbname,$verbose,$no_update,$camera);
-
+my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera);
+$force = 0;
+$verbose = 0;
 GetOptions(
     'remote_id=s'   => \$remote_id,
@@ -41 +41 @@
     'policy=s'      => \$policy,
     'poll'          => \$poll,
+    'force-poll'    => \$force,
     'camera=s'      => \$camera,
     'dbname=s'      => \$dbname,
@@ -53 +54 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
-
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files                                    
 
@@ -61 +61 @@
 
 print "passed authentication challenge.\n";
+
+# Phase 1.5: Grab the information about this run;
+my $rt_cmd = "$remotetool -listrun -remote_id $remote_id ";
+$rt_cmd   .= " -dbname $dbname " if defined($dbname);
+my ( $rt_success, $rt_error_code, $rt_full_buf, $rt_stdout_buf, $rt_stderr_buf ) =
+    run(command => $rt_cmd, verbose => $verbose);
+unless ($rt_success) {
+    $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("UNable to run remotetool to determine remote run status",
+            $remote_id,$rt_error_code);
+}
+my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 
+    &my_die("UNable to run remotetool to determine remote run status",
+            $remote_id,$rt_error_code);
+my $rrData2 = parse_md_list($rrData);
+my $runData = ${ $rrData2 }[0]; # There should be only one
 
 # These are operations we only need to do on the first call, not on subsequent poll operations.
@@ -92 +108 @@
     my (undef,$remote_check) = uri_convert($uri_check);
     my (undef,$remote_transfer)= uri_convert($uri_transfer);
-#    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_validate_files.pl --check_list $remote_check --transfer_list $remote_transfer");
+
     my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
     # We no longer need to parse this output, as it retrieves files itself.
@@ -130 +146 @@
 # Poll the job status?
 my $poll_count = 0;
-my $poll_max   = 5;
+my $poll_max   = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes.
 my $poll_sleep = 60;
 my $poll_response = 0;
-while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)) {
+my $poll_word  = "nothing";
+
+if ($force) {
+    $poll_response = 1;
+    $poll_word     = "Completed";
+}
+
+while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
     if ($verbose) {
         print "Polling code.  Sleeping.  $poll_count of $poll_max $poll_response\n";
@@ -140 +163 @@
         sleep($poll_sleep);
     }
-    $poll_response = poll_job($job_id);
+    ($poll_response,$poll_word) = poll_job($job_id);
 
     if ($poll_response == 1) { # This job has completed
@@ -146 +169 @@
     }
     elsif ($poll_response == -1) { # This job has an error
-        &my_die("The job exited incorrectly.",$remote_id, $PS_EXIT_PROG_ERROR);
+        warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
+        $poll_response = 1; # To fool the check later into proceding
     }
     $poll_count++;
@@ -156 +180 @@
     &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
 }
-# Retrieve validation summary
-my $fault = 0;
+
 
 # Initialize the remote side to sync things
@@ -165 +188 @@
 my $uri_return  = $path_base . ".return";
 my ($ipp_return,$remote_return) = uri_convert($uri_return);
-scp_get($remote_return,$ipp_return);
-# Feed teh return list into the 
-my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return");
-print "$return_push_output\n";
-
-
+# We shouldn't need to get this file.  We made that file.
+#scp_get($remote_return,$ipp_return);
+
+# Construct list of dbinfo files we'll need to execute.
+my @returned_files = ();
+my @dbinfo_files   = ();
+my @dbindex        = ();
+my $iter           = 0;
 open(RETURN,   "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR);
-while(<RETURN>) {
+while (<RETURN>) {
     chomp;
+    push @returned_files, $_;
     if ($_ =~ /dbinfo/) {
-        my $file = $_;
-#       $file =~ s/${remote_root}.tmp//; THis shouldn't be needed, as the transfer tool munges the files on that side.
-        open(DBF,"$file") || warn "Missing file $file\n";
-        my $cmd = <DBF>;
-        chomp($cmd);
-        $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $cmd, verbose => $verbose);
-        unless ($success) {
-            # This shouldn't fail, but we also can't suddenly abort if something does fail.  Maybe retry if this looks to be a continuing issue?
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            warn("The command that shouldn't fail has failed: $cmd $remote_id $error_code");##,$remote_id,$error_code);
+        push @dbinfo_files, $_;
+        push @dbindex, $iter;
+    }
+    $iter++;
+
+}
+close(RETURN);
+print STDERR "Expect $#dbinfo_files dbinfo files\n";
+
+my $first = 0;
+my $last  = $#dbinfo_files;
+my $mid   = 0;
+my $range = 20;  # How wide of a chunk to look through to catch threading drops
+my $offset = 0;  # Offset to start from.
+
+# Feed the return list into the transfer tool
+# If this fails, we should go to an auth wait point.
+if ($poll_word eq 'Completed') {  
+    # We're complete.  All files should exist on disk
+    my $iter = 0;
+    my $transfer_success = 1;
+    do {
+        $transfer_success = 1;
+        $iter++;
+        check_ssh_connection(); 
+        my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
+        my $return_push_output_print = join "\n", @{ $return_push_output };
+        if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; }
+        print "$return_push_output_print\n";
+    } while (($transfer_success != 1) &&  ($iter < 10));  # Try harder;
+}
+else {
+    # Just try and fetch things, because we didn't complete correctly, it seems.
+    my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
+    my $return_push_output_print = join "\n", @{ $return_push_output };
+    print "$return_push_output_print\n";
+}
+
+foreach my $file (@dbinfo_files) {
+    open(DBF,"$file") || warn "Missing file $file\n";
+    my $cmd = <DBF>;
+    close(DBF);
+    chomp($cmd);
+#    print STDERR "$file $cmd\n";
+    $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+
+        # This shouldn't fail, but we also can't suddenly abort if something does fail.  
+        # Now we're going to drop the component that owns this dbinfo, which should force a retry.
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
+        if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
+            drop_component($remote_id,$cmd,$file);
         }
-        close(DBF);
-    }
-}
-close(RETURN);
+    }
+
+}
 
 # We're done, so set the state and exit.
 &my_die("Finished",$remote_id,0,"full");
-
 
 
@@ -209 +276 @@
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         &my_die("Authorization check failed.",$remote_id,0,'auth');
-
-#       my $update_command = "$remotetool -updaterun -remote_id $remote_id ";
-#       $update_command .= " -set_state auth ";
-#       $update_command .= " -dbname $dbname " if defined $dbname;
-#       system($update_command);
-#       exit(0);  # This isn't an error.  It's a state we monitor and move to automatically.
     }
 }
@@ -250 +311 @@
 sub ssh_exec_command {
     my $cmd = shift;
-    $cmd = "$ssh $DMZ_HOST ssh ${SEC_HOST} $cmd";
+    $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $cmd, verbose => $verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        # If we failed, see if we failed due to authorization.
+        check_ssh_connection(); 
         &my_die("Failed to execute command: >>$cmd<<"); 
     }
@@ -266 +330 @@
     
     if ($response =~ /State: Running/) { 
-        return(0);
+        return(0,"Running");
     }
     elsif ($response =~ /State: Completed/) {
-        return(1);
+        return(1,"Completed");
     }
     elsif ($response =~ /State: Idle/) {
-        return(-2);
+        return(-2,"Idle");
     }
     else {
-        return(-1);
+        my $response_word = $response;
+        $response_word =~ s/.*State: //;
+        $response_word =~ s/\n.*//;
+        return(-1,$response_word);
     }
 }
@@ -324 +391 @@
     return($local_uri);
 }
+
+sub drop_component {
+    my $remote_id = shift;
+    my $cmd = shift;
+    my $file = shift;
+
+    if ($cmd eq '') { 
+        # Zero byte file.  This dbinfo was never constructed properly.
+        
+        # Steal the stage_id from the filename.  This isn't something I'm happy about.
+        my $stage = $runData->{stage};
+        my $stage_id = $file;
+        if ($stage eq 'chip') { 
+            $stage_id =~ s/^.*ch\.//;
+        }
+        elsif ($stage eq 'camera') {
+            $stage_id =~ s/^.*cm\.//;
+        }
+        elsif ($stage eq 'warp') {
+            $stage_id =~ s/^.*wrp\.//;
+        }
+        elsif ($stage eq 'stack') {
+            $stage_id =~ s/^.*stk\.//;
+        }
+        $stage_id =~ s/\..*$//;
+        
+        # Drop this from the database
+        my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";
+        $drop_cmd   .= " -dbname $dbname " if defined($dbname);
+        system($drop_cmd);
+    }
+
+}
+
 sub my_die {
     my $msg = shift;
@@ -347 +448 @@
 }
 
-# I don't think I need this anymore, but I don't want to burn it until I"m sure.
-
-# exit(1);
-# # Read this as a metadata object
-# my $ssh_exec_stdout = ssh_exec_command("sc_validate_processing.pl ${path_base}.out");
-# my $retrieveData = $mdcParser->parse(join "", @$ssh_exec_stdout);
-# my $retrieveData2= parse_md_list($retrieveData);
-
-# foreach my $retrieveEntry (@$retrieveData2) {
-#     my $process_fault = $retrieveEntry->{fault};
-#     my $dbcommand = $retrieveEntry->{dbcommand};
-#     my $filelist  = $retrieveEntry->{filelist};
-
-#     scp_get("${filelist}",${filelist});
-#     open(FL,${filelist});
-#     while(<FL>) {
-#       chomp;
-#       my $f = $_;
-#       scp_get("${f}",$f);
-#     }
-#     close(FL);
-
-#     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#       run(command => $dbcommand, verbose => $verbose);
-#     unless ($success) {
-#       $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-#       &my_die("The database command returned did not execute correctly. $dbcommand $process_fault",
-#               $remote_id,$error_code); 
-#     }
-
-#     if ($process_fault != 0) {
-#       $fault = 1;
-#     }
-# }
-
-# if (($policy ne 'ignore')&&($fault != 0)) {
-#     &my_die("This job exited without all components finishing successfully.  Halting.",
-#           $remote_id,$fault,"halt");
-# }
-# else {
-#     print "This job exited without all components finishing successfully. $fault\n" if $fault;
-#     &my_die("This job finished.  Setting remote job to full.",
-#           $remote_id,0,"full");
-# }
-
-# Promote to finished.
-#&my_die(); # Not really, but I don't have the tool interface finished, so these are all just placeholders.
 # Quick review:
 # new -> pending -> run -> full

branches/eam_branches/ipp-20140717/ippScripts/scripts/sc_transfer_tool.pl

@@ -10 +10 @@
 use Sys::Hostname;
 
-my $local_raw = '/scratch3/watersc1/raw/';
-my $local_tmp = '/scratch3/watersc1/tmp/';
+my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
+$remote_root = '/scratch3/watersc1/';
+my $local_raw = "${remote_root}/raw/";
+my $local_tmp = "${remote_root}/tmp/";
 my $threads = 10;
 my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
@@ -68 +70 @@
     }
     elsif (!($fetch)) { # We are pushing
-        print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
+        # The rsync call expects to find files of a given name in the directory specified.
+#       print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
+        print { $filehandles[$i] } "$_" . "\n";
     }
 }
@@ -112 +116 @@
 
     # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
-    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
+#    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
+    my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
 
@@ -118 +123 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-
         print STDERR "ERROR:  $error_code\n";
 #       my $std = join "\n", @{ $stdout_buf };
@@ -125 +129 @@
 #       print "STDERR: $std\n";
 
-        foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
-            if ($line =~ /No such file or directory/) { 
-                my $file = (split /\s+/, $line)[1];
-                $file =~ s/:$//;
-                if ($file !~ /nebulous/) { next; }
-                print STDERR "Touching $file so tar can continue.\n";
-                system("touch $file");
-            }
-        }
+#       foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
+#           if ($line =~ /No such file or directory/) { 
+#               my $file = (split /\s+/, $line)[1];
+#               $file =~ s/:$//;
+#               if ($file !~ /nebulous/) { next; }
+#               print STDERR "Touching $file so tar can continue.\n";
+#               system("touch $file");
+#           }
+#       }
 
         warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");

branches/eam_branches/ipp-20140717/ippTools/share/Makefile.am

@@ -484 +484 @@
         releasetool_definerelstack.sql \
         releasetool_definerelstack_with_skycal.sql \
+        releasetool_deleterelexp.sql \
+        releasetool_deleterelstack.sql \
         releasetool_listrelstack.sql \
+        releasetool_setrelstackcalibratedfromskycal.sql 
         releasetool_summary.sql \
         releasetool_definerelgroup_select_lap.sql \

branches/eam_branches/ipp-20140717/ippTools/src/disttool.c

@@ -605 +605 @@
     PXOPT_COPY_TIME(config->args, where, "-time_stamp_begin", "distRun.time_stamp", ">=");
     PXOPT_COPY_TIME(config->args, where, "-time_stamp_end", "distRun.time_stamp", "<=");
+    PXOPT_COPY_S64(config->args, where, "-dist_id_min", "dist_id", ">=");
+    PXOPT_COPY_S64(config->args, where, "-dist_id_max", "dist_id", "<=");
 
     PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);

branches/eam_branches/ipp-20140717/ippTools/src/disttoolConfig.c

@@ -85 +85 @@
     psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-time_stamp_begin", 0, "limit updates by time_stamp (>=)", NULL); 
     psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-time_stamp_end", 0, "limit updates by time_stamp (<=)", NULL); 
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dist_id_min",  0, "limit updates by dist_id (>=)", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dist_id_max",  0, "limit updates by dist_id (<=)", 0);
     psMetadataAddBool(updaterunArgs, PS_LIST_TAIL,"-clean",     0, "limit updates to clean distRuns", false);
     psMetadataAddBool(updaterunArgs, PS_LIST_TAIL,"-full",      0, "limit updates to not clean distRuns", false);

branches/eam_branches/ipp-20140717/ippTools/src/releasetool.c

@@ -44 +44 @@
 static bool updaterelexpMode(pxConfig *config);
 static bool listrelexpMode(pxConfig *config);
+static bool deleterelexpMode(pxConfig *config);
 static bool definerelstackMode(pxConfig *config);
 static bool updaterelstackMode(pxConfig *config);
+static bool setrelstackcalibratedfromskycalMode(pxConfig *config);
 static bool listrelstackMode(pxConfig *config);
+static bool deleterelstackMode(pxConfig *config);
 static bool summaryMode(pxConfig *config);
 static bool definerelgroupMode(pxConfig *config);
@@ -81 +84 @@
         MODECASE(RELEASETOOL_MODE_UPDATERELEXP,     updaterelexpMode);
         MODECASE(RELEASETOOL_MODE_LISTRELEXP,       listrelexpMode);
+        MODECASE(RELEASETOOL_MODE_DELETERELEXP,     deleterelexpMode);
 
         MODECASE(RELEASETOOL_MODE_DEFINERELSTACK,   definerelstackMode);
         MODECASE(RELEASETOOL_MODE_UPDATERELSTACK,   updaterelstackMode);
+        MODECASE(RELEASETOOL_MODE_SETRELSTACKCALIBRATEDFROMSKYCAL, setrelstackcalibratedfromskycalMode);
         MODECASE(RELEASETOOL_MODE_LISTRELSTACK,     listrelstackMode);
+        MODECASE(RELEASETOOL_MODE_DELETERELSTACK,   deleterelstackMode);
         MODECASE(RELEASETOOL_MODE_SUMMARY,          summaryMode);
 
@@ -817 +823 @@
 
     psFree(output);
+
+    return true;
+}
+
+static bool deleterelexpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(relexp_id, config->args,   "-relexp_id", false, false);
+    // If relexp_id is supplied that is all that we need
+    if (!relexp_id) {
+        // Otherwise we need to identify the release ...
+        PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+        PXOPT_LOOKUP_S32(rel_id, config->args,      "-rel_id", false, false);
+
+        if (!release_name && !rel_id) {
+            psError(PXTOOLS_ERR_CONFIG, true, "at least one of -relexp_id, -release_name, or -rel_id is required\n");
+            return false;
+        }
+        // ... and the exposure
+        PXOPT_LOOKUP_STR(label, config->args,       "-label", false, false);
+        PXOPT_LOOKUP_S64(cam_id, config->args,      "-cam_id", false, false);
+        PXOPT_LOOKUP_S64(exp_id, config->args,      "-exp_id", false, false);
+        PXOPT_LOOKUP_STR(exp_name, config->args,    "-exp_name", false, false);
+        if (!label && !cam_id && !exp_id && !exp_name) {
+            psError(PXTOOLS_ERR_CONFIG, true, "at least one of -label, -cam_id, -exp_id, -exp_name is required\n");
+            return false;
+        }
+    }
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S32(config->args, where, "-relexp_id", "relExp.relexp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",  "camRun.label", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+
+    PXOPT_COPY_S32(config->args, where, "-rel_id",  "relExp.rel_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "==");
+
+    psString query = pxDataGet("releasetool_deleterelexp.sql");
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, "\n    AND %s", whereClause);
+    psFree(whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("releasetool", PS_LOG_INFO, "Deleted %" PRIu64 " relExp rows", affected);
+
+
+    psFree(query);
 
     return true;
@@ -1028 +1092 @@
     return false;
 }
+static bool setrelstackcalibratedfromskycalMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-skycal_id",  "skycalRun.skycal_id", "==");
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "at least one of -skycal_label, -skycal_data_group, or -skycal_id is required");
+        return false;
+    }
+
+    PXOPT_COPY_STR(config->args, where, "-tess_id",    "stackRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
+    pxskycellAddWhere(config, where);
+
+    // insure that at least one of these is supplied to select the release
+    PXOPT_LOOKUP_S64(rel_id, config->args,      "-rel_id", false, false);
+    PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+    if (!rel_id && !release_name) {
+        psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required");
+        return false;
+    }
+
+    // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match)
+    // note: we add these after above to insure that search args are supplied
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "==");
+    PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(replace, config->args, "-replace", false);
+
+    psString query = pxDataGet("releasetool_setrelstackcalibratedfromskycal.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (replace) {
+        // don't cut on relStack.state
+        psStringAppend(&query, "\nWHERE ");;
+    } else {
+        // normal path only update rows in state processed
+        psStringAppend(&query, "\nWHERE relStack.state = 'processed'\nAND ");;
+    }
+        
+    // Add in the where conditions
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, "%s", whereClause);
+    psFree(whereClause);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " relStack rows", affected);
+
+    return true;
+}
+
+static bool deleterelstackMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(relstack_id, config->args,   "-relstack_id", false, false);
+    // If relexp_id is supplied that is all that we need
+    if (!relstack_id) {
+        // Otherwise we need to identify the release ...
+        PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false);
+        PXOPT_LOOKUP_S32(rel_id, config->args,      "-rel_id", false, false);
+
+        if (!release_name && !rel_id) {
+            psError(PXTOOLS_ERR_CONFIG, true, "at least one of -relstack_id, -release_name, or -rel_id is required\n");
+            return false;
+        }
+        // ... and the Stack
+        PXOPT_LOOKUP_STR(label, config->args,       "-label", false, false);
+        PXOPT_LOOKUP_S64(stack_id, config->args,    "-stack_id", false, false);
+        if (!label && !stack_id) {
+            psError(PXTOOLS_ERR_CONFIG, true, "at least one of -label and  -stack_id is required\n");
+            return false;
+        }
+    }
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S32(config->args, where, "-relstack_id", "relStack.relstack_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
+    PXOPT_COPY_S64(config->args, where, "-stack_id", "relStack.stack_id", "==");
+
+    PXOPT_COPY_S32(config->args, where, "-rel_id", "relStack.rel_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "==");
+
+    psString query = pxDataGet("releasetool_deleterelstack.sql");
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, "\n    AND %s", whereClause);
+    psFree(whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    psU64 affected = psDBAffectedRows(config->dbh);
+    psLogMsg("releasetool", PS_LOG_INFO, "Deleted %" PRIu64 " relStack rows", affected);
+
+
+    psFree(query);
+
+    return true;
+}
 
 static bool listrelstackMode(pxConfig *config)
@@ -1052 +1242 @@
 
     PXOPT_COPY_STR(config->args, where, "-surveyName",  "survey.surveyName", "LIKE");
-    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relExp.rel_id", "==");
+    PXOPT_COPY_S32(config->args, where, "-rel_id",      "relStack.rel_id", "==");
     pxskycellAddWhere(config, where);
 

branches/eam_branches/ipp-20140717/ippTools/src/releasetool.h

@@ -34 +34 @@
     RELEASETOOL_MODE_TOCALIBEXP,
     RELEASETOOL_MODE_LISTRELEXP,
+    RELEASETOOL_MODE_DELETERELEXP,
     RELEASETOOL_MODE_DEFINERELSTACK,
     RELEASETOOL_MODE_UPDATERELSTACK,
+    RELEASETOOL_MODE_SETRELSTACKCALIBRATEDFROMSKYCAL,
+    RELEASETOOL_MODE_DELETERELSTACK,
     RELEASETOOL_MODE_TOCALIBSTACK,
     RELEASETOOL_MODE_LISTRELSTACK,

branches/eam_branches/ipp-20140717/ippTools/src/releasetoolConfig.c

@@ -67 +67 @@
     psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_ubercal_file", 0,  "define ubercal file name", NULL);
 
-    // -uptdaterelease
+    // -updaterelease
     psMetadata *updatereleaseArgs = psMetadataAlloc();
 
@@ -183 +183 @@
     psMetadataAddBool(listrelexpArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
 
+    // -deleterelexp
+    psMetadata *deleterelexpArgs = psMetadataAlloc();
+    psMetadataAddS64(deleterelexpArgs, PS_LIST_TAIL,  "-relexp_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddS64(deleterelexpArgs, PS_LIST_TAIL,  "-exp_id", 0,      "select by exposure ID", 0);
+    psMetadataAddStr(deleterelexpArgs,  PS_LIST_TAIL, "-exp_name", 0,    "select by exposure name", NULL);
+    psMetadataAddS32(deleterelexpArgs, PS_LIST_TAIL,  "-rel_id", 0,      "select by release ID", 0);
+    psMetadataAddS32(deleterelexpArgs, PS_LIST_TAIL,  "-cam_id", 0,      "select by cam run ID", 0);
+    psMetadataAddStr(deleterelexpArgs,  PS_LIST_TAIL, "-label", 0,       "select by camRun.label", NULL);
+    psMetadataAddStr(deleterelexpArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+
+
     // -definerelstack
     psMetadata *definerelstackArgs = psMetadataAlloc();
@@ -248 +259 @@
     psMetadataAddBool(listrelstackArgs, PS_LIST_TAIL, "-simple",      0, "use the simple output format", false);
 
+    // -updaterelstack
+    psMetadata *updaterelstackArgs = psMetadataAlloc();
+
+    psMetadata *calibraterelstackArgs = psMetadataAlloc();
+
+        // set the target release and collection of relStacks (one of these is required)
+    psMetadataAddStr(calibraterelstackArgs,  PS_LIST_TAIL, "-release_name", 0, "define release name", NULL);
+    psMetadataAddS32(calibraterelstackArgs, PS_LIST_TAIL,  "-rel_id", 0,      "define release ID", 0);
+
+        // select the processing
+    psMetadataAddStr(calibraterelstackArgs,  PS_LIST_TAIL, "-skycal_label",  0, "select by skycalRun.label", NULL);
+    psMetadataAddStr(calibraterelstackArgs,  PS_LIST_TAIL, "-skycal_data_group", 0,  "select by skycalRun.data_group", NULL);
+    psMetadataAddS64(calibraterelstackArgs, PS_LIST_TAIL,  "-skycal_id", 0,      "select by skycalRun.skycal_id", 0);
+
+    psMetadataAddStr(calibraterelstackArgs,  PS_LIST_TAIL, "-tess_id", 0,  "select by tess_id", NULL);
+    psMetadataAddStr(calibraterelstackArgs,  PS_LIST_TAIL, "-skycell_id", 0,  "select by skycell_id", NULL);
+    pxskycellAddArguments(calibraterelstackArgs);
+
+    psMetadataAddBool(calibraterelstackArgs, PS_LIST_TAIL, "-replace",   0, "replace any existing calibrations", false);
+    psMetadataAddU64(calibraterelstackArgs, PS_LIST_TAIL,  "-limit",   0, "limit result set to N items", 0);
+
+    // -deleterelstack
+    psMetadata *deleterelstackArgs = psMetadataAlloc();
+    psMetadataAddS64(deleterelstackArgs, PS_LIST_TAIL,  "-relstack_id", 0,   "select by released exposure ID", 0);
+    psMetadataAddS64(deleterelstackArgs, PS_LIST_TAIL,  "-stack_id", 0,   "select by stack ID", 0);
+//    psMetadataAddS64(deleterelstackArgs, PS_LIST_TAIL,  "-skycal_id", 0,   "select by skycal ID", 0);
+    psMetadataAddStr(deleterelstackArgs,  PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL);
+    psMetadataAddS32(deleterelstackArgs, PS_LIST_TAIL,  "-rel_id", 0,   "select by release ID", 0);
+
+    psMetadataAddStr(deleterelstackArgs,  PS_LIST_TAIL, "-label",  0,      "select by stackRun.label", NULL);
+//    psMetadataAddStr(deleterelstackArgs,  PS_LIST_TAIL, "-data_group", 0,  "select by stackRun.data_group", NULL);
+//    psMetadataAddStr(deleterelstackArgs,  PS_LIST_TAIL, "-skycal_label",  0, "select by skycalRun.label", NULL);
+//    psMetadataAddStr(deleterelstackArgs,  PS_LIST_TAIL, "-skycal_data_group", 0,  "select by skycalRun.data_group", NULL);
+
     // -summary
     psMetadata *summaryArgs = psMetadataAlloc();
@@ -348 +393 @@
     PXOPT_ADD_MODE("-tocalibexp",       "list relExps pending calibration",  RELEASETOOL_MODE_TOCALIBEXP,  tocalibexpArgs);
     PXOPT_ADD_MODE("-listrelexp",       "list released exposures",      RELEASETOOL_MODE_LISTRELEXP,    listrelexpArgs);
+    PXOPT_ADD_MODE("-deleterelexp",     "delete a released exposure",   RELEASETOOL_MODE_DELETERELEXP,  deleterelexpArgs);
 
     PXOPT_ADD_MODE("-definerelstack",     "define a released stack",    RELEASETOOL_MODE_DEFINERELSTACK,  definerelstackArgs);
     PXOPT_ADD_MODE("-listrelstack",       "list released stacks",      RELEASETOOL_MODE_LISTRELSTACK,    listrelstackArgs);
+    PXOPT_ADD_MODE("-updaterelstack",     "update parameters of released stacks",      RELEASETOOL_MODE_LISTRELSTACK,    updaterelstackArgs);
+    PXOPT_ADD_MODE("-setrelstackcalibratedfromskycal", "update parameters of using skycalResults",      RELEASETOOL_MODE_SETRELSTACKCALIBRATEDFROMSKYCAL,    calibraterelstackArgs);
+    PXOPT_ADD_MODE("-deleterelstack",     "update parameters of released stacks",      RELEASETOOL_MODE_DELETERELSTACK,  deleterelstackArgs);
+
     PXOPT_ADD_MODE("-summary",            "list stackSummaryes for released stacks", RELEASETOOL_MODE_SUMMARY,    summaryArgs);
 

branches/eam_branches/ipp-20140717/ippTools/src/stacktool.c

@@ -260 +260 @@
     // these are used to build the HAVING restriction
     PXOPT_COPY_S32(config->args, having, "-min_num", "num_warp", ">=");
+    PXOPT_COPY_S32(config->args, having, "-max_num", "num_warp", "<=");
+    
     PXOPT_LOOKUP_S32(min_num,     config->args, "-min_num",  false, false);
     if (min_num < 2) {

branches/eam_branches/ipp-20140717/ippTools/src/stacktoolConfig.c

@@ -98 +98 @@
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0);
+    psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-max_num", 0, "maximum number of inputs", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0);
     psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-min_frac", 0, "minumum fraction of new inputs", NAN);

branches/eam_branches/ipp-20140717/ippTools/src/staticskytool.c

@@ -1057 +1057 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
     psString query = pxDataGet("staticskytool_defineskycalrun.sql");
@@ -1084 +1085 @@
     }
 
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, "\n%s", limitString);
+        psFree(limitString);
+    }
 
     if (!p_psDBRunQueryF(config->dbh, query, labelHook)) {
@@ -1171 +1177 @@
     PXOPT_COPY_S64(config->args, where, "-sky_id",   "sky_id",   "==");
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "stackRun.skycell_id",    "==");
     PXOPT_COPY_STR(config->args, where, "-label",   "skycalRun.label",    "==");
     PXOPT_COPY_STR(config->args, where, "-state",   "skycalRun.state",    "==");

branches/eam_branches/ipp-20140717/ippTools/src/staticskytoolConfig.c

@@ -181 +181 @@
     psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs",  0, "list inputs, do not modify database", false);
     psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddU64(defineskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
 
     // -updateskycalrun
@@ -188 +189 @@
     psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-stack_id", 0,  "search by stack ID", 0);
     psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
+    psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", 0);
     psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
     pxskycellAddArguments(updateskycalrunArgs);

branches/eam_branches/ipp-20140717/ippconfig/gpc1/psastro.config

@@ -379 +379 @@
 #  PSASTRO.CATDIR                STR     PS1.REF.20120503
 #  PSASTRO.CATDIR               STR     PS1.REF.20120524
-   PSASTRO.CATDIR               STR     PS1.REF.20140605
+#   PSASTRO.CATDIR               STR     PS1.REF.20140605
+   PSASTRO.CATDIR                STR     PS1.REF.20140713
    ZERO.POINT.USE.MEAN          BOOL    TRUE  
 END