Changeset 36370 for branches/eam_branches/ipp-20130904

Timestamp:

Dec 10, 2013, 2:14:21 PM (13 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20130904

Files:

• . (modified) (1 prop)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• ippScripts/scripts/lap_science.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/rawcheck.pl (modified) (7 diffs)
• ippScripts/scripts/skycell_jpeg.pl (modified) (2 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (1 prop)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/camtool_find_pendingimfile.sql (modified) (1 prop)
• ippTools/share/chiptool_setimfiletoupdate.sql (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 prop)
• ippTools/share/stacktool_addsummary.sql (modified) (1 diff)
• ippTools/share/stacktool_revertsummary.sql (copied) (copied from trunk/ippTools/share/stacktool_revertsummary.sql )
• ippTools/share/warptool_towarped.sql (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/camtool.c (modified) (1 prop)
• ippTools/src/camtoolConfig.c (modified) (1 prop)
• ippTools/src/magictool.c (modified) (1 prop)
• ippTools/src/regtoolConfig.c (modified) (1 diff)
• ippTools/src/stacktool.c (modified) (3 diffs)
• ippTools/src/stacktool.h (modified) (1 diff)
• ippTools/src/stacktoolConfig.c (modified) (2 diffs)
• ippconfig (modified) (1 prop)
• ippconfig/gpc1 (modified) (1 prop)
• ippconfig/recipes/filerules-split.mdc (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 prop)
• ippconfig/recipes/reductionClasses.mdc (modified) (1 prop)

branches/eam_branches/ipp-20130904/ippScripts/scripts/rawcheck.pl

@@ -81 +81 @@
 my $regtool_cmd = "$regtool -processedimfile -exp_id $exp_id -dbname $dbname";
 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
-    run(command => $regtool_cmd, verbose => 1);
+    run(command => $regtool_cmd, verbose => 0);
 unless ($success) {
     $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -133 +133 @@
             $existance[$i] = 1;
             $existing_copies++;
-            my $fh;
-            open($fh,$files[$i]);
-            $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest;
+            $md5sums[$i] = local_md5sum($files[$i]);
+#           my $fh;
+#           open($fh,$files[$i]);
+#           $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest;
+#           close($fh);
             $md5sum_uniq{$md5sums[$i]} = 1;
-            close($fh);
+
         }
         else {
@@ -223 +225 @@
 # Begin my best validation thought
         {
-            my $fh;
-            open($fh,$files[0]);
-            my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
-            close($fh);
+            my $tmpmd5 = local_md5sum($files[0]);
+#           my $fh;
+#           open($fh,$files[0]);
+#           my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
+#           close($fh);
             if ($tmpmd5 ne $md5sum) { 
                 die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
@@ -257 +260 @@
                 my $uris = $neb->find_instances($key,$rep_vol);
                 @$uris = map {URI->new($_)->file if $_} @$uris;
-                my $fh;
-                open($fh,${ $uris }[0]);
-                my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
-                close($fh);
+                my $tmpmd5 = local_md5sum(${ $uris }[0]);
+#               my $fh;
+#               open($fh,${ $uris }[0]);
+#               my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
+#               close($fh);
                 if ($tmpmd5 ne $md5sum) { 
                     die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
@@ -282 +286 @@
 
 #               print $uris . " " . ${ $uris }[0] . " \n";
-                my $fh;
-                open($fh,${ $uris }[0]);
-
-                my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
-                close($fh);
+                my $tmpmd5 = local_md5sum(${ $uris }[0]);
+#               my $fh;
+#               open($fh,${ $uris }[0]);
+#               my $tmpmd5 = Digest::MD5->new->addfile($fh)->hexdigest;
+#               close($fh);
                 if ($tmpmd5 ne $md5sum) { 
                     die "Post-replication md5sum does not match! $tmpmd5 != $md5sum";
@@ -318 +322 @@
 }
 
+sub local_md5sum {
+    my $filename = shift;
+    my $volume   = (split /\//, $filename)[2];
+    my $host     = $volume;
+    $host =~ s/\.\d//;
+#    print "$filename $host $volume\n";
+    my $response = `ssh $host md5sum $filename`;
+    chomp($response);
+    my ($sum, undef) = split /\s+/, $response;
+    unless(defined($sum)) {
+        my_die("Failed to calculate md5sum locally. $filename $host $volume");
+    }
+    return($sum);
+}
+
+
 sub parse_volume {
     my $filename = shift(@_);
@@ -341 +361 @@
 }
 
-        
-
-
 sub my_die {
     my $msg = shift(@_);

branches/eam_branches/ipp-20130904/ippScripts/scripts/skycell_jpeg.pl

@@ -271 +271 @@
 
     my %products = ('image' => "PPSTACK.UNCONV",
-                    'mask'  => "PPSTACK.UNCONV.MASK",
+#                   'mask'  => "PPSTACK.UNCONV.MASK",
                     'variance' => "PPSTACK.UNCONV.VARIANCE",
                     'exp'   => "PPSTACK.UNCONV.EXP",
@@ -327 +327 @@
                 $command .= " -Di BIN1 1 -Di BIN2 1 ";
             }
-#           elsif ($key eq 'image') {
-#               $command .= " -masks $tangents{$projection_cell}{mask}{NAME} ";
-#           }
-#           elsif ($key eq 'mask') { 
-#               next; # This should be made with the images.
-#           }
-            else {
-                # Append the image list to other objects, in case the WCS information is unpopulated
-                $command .= " -wcsref $tangents{$projection_cell}{image}{NAME} ";
-            }
+            elsif ($key eq 'image') {
+                $command .= " -exptimeOrder 1 ";
+            }
+            elsif ($key eq 'variance') { 
+                $command .= " -exptimeOrder 2 ";
+            }
+            # Append the image list to other objects, in case the WCS information is unpopulated
+            $command .= " -wcsref $tangents{$projection_cell}{image}{NAME} ";
+
             if ($tangents{$projection_cell}{$key}{N} > 0) {
                 print "$command\n";

branches/eam_branches/ipp-20130904/ippTools/share/Makefile.am

@@ -400 +400 @@
         stacktool_tosummary.sql \
         stacktool_addsummary.sql \
+        stacktool_revertsummary.sql \
         stacktool_summary.sql \
         staticskytool_definebyquery_select.sql \

branches/eam_branches/ipp-20130904/ippTools/share/stacktool_addsummary.sql

@@ -1 +1 @@
 INSERT INTO stackSummary (sass_id,projection_cell,path_base) VALUES (
-       %lld,   -- warp_id
+       %lld,   -- sass_id
        '%s',   -- projection_cell
        '%s'    -- path_base

branches/eam_branches/ipp-20130904/ippTools/src/regtoolConfig.c

@@ -292 +292 @@
     psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-state",         0,            "search by exposure state", NULL);
     psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-exp_tag",       0,            "search by exp_tag", NULL);
-    psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-exp_type",      0,            "search by exp_type", "object");
+    psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-exp_type",      0,            "search by exp_type", NULL);
     psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-filelevel",     0,            "search by filelevel", NULL);
     psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-reduction",     0,            "search by reduction class", NULL);

branches/eam_branches/ipp-20130904/ippTools/src/stacktool.c

@@ -44 +44 @@
 static bool tosummaryMode(pxConfig *config);
 static bool addsummaryMode(pxConfig *config);
+static bool revertsummaryMode(pxConfig *config);
 static bool summaryMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
@@ -85 +86 @@
         MODECASE(STACKTOOL_MODE_TOSUMMARY,             tosummaryMode);
         MODECASE(STACKTOOL_MODE_ADDSUMMARY,            addsummaryMode);
+        MODECASE(STACKTOOL_MODE_REVERTSUMMARY,         revertsummaryMode);
         MODECASE(STACKTOOL_MODE_SUMMARY,               summaryMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -1570 +1572 @@
 }
 
+static bool revertsummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", true, false);
+
+  psString query = pxDataGet("stacktool_revertsummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+    return(false);
+  }
+  if (!p_psDBRunQueryF(config->dbh, query, sass_id)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return(false);
+  }
+  psS64 numUpdated = psDBAffectedRows(config->dbh);
+
+  if (numUpdated != 1) {
+    psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+    psFree(query);
+    return(false);
+  }
+  psLogMsg("stacktool", PS_LOG_INFO, "Deleted %ld rows", numUpdated);
+  
+  psFree(query);
+
+  return(true);
+}
+
 static bool summaryMode(pxConfig *config)
 {

branches/eam_branches/ipp-20130904/ippTools/src/stacktool.h

@@ -38 +38 @@
     STACKTOOL_MODE_TOSUMMARY,
     STACKTOOL_MODE_ADDSUMMARY,
+    STACKTOOL_MODE_REVERTSUMMARY,
     STACKTOOL_MODE_SUMMARY,
     STACKTOOL_MODE_PENDINGCLEANUPRUN,

branches/eam_branches/ipp-20130904/ippTools/src/stacktoolConfig.c

@@ -265 +265 @@
     psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0,     "set summary path base", NULL);
 
+    // -revertsummary
+    psMetadata *revertsummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(revertsummaryArgs, PS_LIST_TAIL, "-sass_id", 0,   "search by stack association ID", 0);
+    
     // -summary
     psMetadata *summaryArgs = psMetadataAlloc();
@@ -335 +339 @@
     PXOPT_ADD_MODE("-addsumskyfile",   "", STACKTOOL_MODE_ADDSUMSKYFILE,   addsumskyfileArgs);
     PXOPT_ADD_MODE("-sumskyfile",      "list results of stackRun", STACKTOOL_MODE_SUMSKYFILE,      sumskyfileArgs);
-    PXOPT_ADD_MODE("-sassskyfile",      "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE,      sassskyfileArgs);
+
     PXOPT_ADD_MODE("-revertsumskyfile","", STACKTOOL_MODE_REVERTSUMSKYFILE,      revertsumskyfileArgs);
+    PXOPT_ADD_MODE("-updatesumskyfile",      "update fault code for sumskyfile",     STACKTOOL_MODE_UPDATESUMSKYFILE,          updatesumskyfileArgs);
+
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
     PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     STACKTOOL_MODE_DONECLEANUP,          donecleanupArgs);
-    PXOPT_ADD_MODE("-updatesumskyfile",      "update fault code for sumskyfile",     STACKTOOL_MODE_UPDATESUMSKYFILE,          updatesumskyfileArgs);
+
+    PXOPT_ADD_MODE("-sassskyfile",      "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE,      sassskyfileArgs);
     PXOPT_ADD_MODE("-tosummary",            "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs);
     PXOPT_ADD_MODE("-summary",              "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs);
     PXOPT_ADD_MODE("-addsummary",           "add entry to the summary table", STACKTOOL_MODE_ADDSUMMARY, addsummaryArgs);
+    PXOPT_ADD_MODE("-revertsummary",        "revert entry in the summary table", STACKTOOL_MODE_REVERTSUMMARY, revertsummaryArgs);
+    
     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs);
     PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           STACKTOOL_MODE_IMPORTRUN, importrunArgs);

branches/eam_branches/ipp-20130904/ippconfig/recipes/filerules-split.mdc

@@ -382 +382 @@
 PPSKYCELL.JPEG1              OUTPUT {OUTPUT}.0.b1.jpeg     JPEG            NONE       CHIP       TRUE      NONE
 PPSKYCELL.JPEG2              OUTPUT {OUTPUT}.0.b2.jpeg     JPEG            NONE       CHIP       TRUE      NONE
-PPSKYCELL.BIN1               OUTPUT {OUTPUT}.b1.fits     IMAGE     COMP_IMG       FPA        TRUE      NONE
-PPSKYCELL.BIN2               OUTPUT {OUTPUT}.b2.fits     IMAGE     COMP_IMG       FPA       TRUE      NONE
+PPSKYCELL.BIN1               OUTPUT {OUTPUT}.b1.fits     IMAGE     NONE       FPA        TRUE      NONE
+PPSKYCELL.BIN2               OUTPUT {OUTPUT}.b2.fits     IMAGE     NONE       FPA       TRUE      NONE
 PPSKYCELL.BIN1.MASK          OUTPUT {OUTPUT}.b1.mk.fits     MASK     COMP_MASK       FPA        TRUE      NONE
 PPSKYCELL.BIN2.MASK          OUTPUT {OUTPUT}.b2.mk.fits     MASK     COMP_MASK       FPA       TRUE      NONE