Changeset 29515 for branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl

Timestamp:

Oct 21, 2010, 2:45:13 PM (16 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl (modified) (13 diffs)

branches/eam_branches/ipp-20100823/ippToPsps/perl/checkOdmStatus.pl

@@ -10 +10 @@
 use ippToPsps::IppToPspsDb;
 use ippToPsps::Datastore;
-use ippToPsps::BatchManager;
+use ippToPsps::Batch;
 
 my $singleBatch = undef;
@@ -19 +19 @@
 my $product = undef;
 my $filePath = undef;
+my $removeUnprocessed = undef;
 
 
@@ -27 +28 @@
         'product|p=s' => \$product,
         'location|l=s' => \$filePath,
+        'remove|r' => \$removeUnprocessed,
         'verbose|v' => \$verbose,
         'save_temps|s' => \$save_temps
@@ -32 +34 @@
 
 if (!defined $product) {
-    print "* OPTIONAL: a datastore product name        -p <name>\n";
+    print "* OPTIONAL: a datastore product name            -p <name>\n";
 }
 if (!defined $singleBatch) {
-    print "* OPTIONAL: a single batch                  -b <batchNum>        (default = none)\n";
+    print "* OPTIONAL: a single batch                      -b <batchNum>        (default = none)\n";
 }
 if (!defined $fromTime) {
     $fromTime = "2010-01-01";
-    print "* OPTIONAL: from time                       -f <dateTime>        (default = $fromTime)\n";
+    print "* OPTIONAL: from time                           -f <dateTime>        (default = $fromTime)\n";
 }
 if (!defined $toTime) {
     $toTime = "2099-12-31";
-    print "* OPTIONAL: to time                         -t <dateTime>        (default = $toTime)\n";
+    print "* OPTIONAL: to time                             -t <dateTime>        (default = $toTime)\n";
 }
 if (!defined $filePath) {
     print "* OPTIONAL: location for files to be deleted    -l <path>\n";
 }
+if (!defined $removeUnprocessed) {
+    $removeUnprocessed = 0;
+    print "* OPTIONAL: remove unprocessed files            -r                   (default = $removeUnprocessed)\n";
+}
 if (!defined $verbose) {
     $verbose = 0;
-    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
+    print "* OPTIONAL: run in verbose mode                 -v                   (default = $verbose)\n";
 }
 if (!defined $save_temps) {
     $save_temps = 0;
-    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
+    print "* OPTIONAL: keep temp files                     -t                   (default = $save_temps)\n";
 }
 
@@ -60 +66 @@
 my $datastore = new ippToPsps::Datastore($product, 0, 0);
 my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
-my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $filePath, $verbose, $save_temps);
 my $odmUrl = "http://web01.psps.ifa.hawaii.edu/a01/OdmWebService/OdmWebService.asmx/GetBatchStatus";
 my $ua = LWP::UserAgent->new;
@@ -85 +90 @@
 
     print "\n";
-    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
-    printf("|      Timestamp       |    Batch     |  Exposure ID | Loaded to ODM? | Merge worthy? | Merged? | Deleted? |\n");
-    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
+    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
+    printf("|      Timestamp       |    Batch     |  Exposure ID | Loaded to ODM? | Load failed? |Merge worthy? | Merged? | Deleted? |\n");
+    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
 
     # loop round batches
@@ -95 +100 @@
     my $numRemovedFromDatastore = 0;
     my $numDeleted = 0;
+    my $newMerged = 0;
     foreach $batch ( @{$batches} ) {
-        my ($timestamp, $expId, $batchId, $surveyType, $deleted, $dvoDb, $processed, $onDatastore, $loadedToOdm, $mergeWorthy, $merged) =  @{$batch};
-
-        if (!$processed) {next;}
+        my ($timestamp, 
+                $type, 
+                $expId, 
+                $batchId, 
+                $surveyType,
+                $deleted,
+                $dvoDb, 
+                $processed, 
+                $onDatastore, 
+                $loadedToOdm, 
+                $loadFailed, 
+                $mergeWorthy, 
+                $merged) = @{$batch};
+
+        if (!$onDatastore) {next;}
+
+        my $batch = ippToPsps::Batch->existing(
+                undef,
+                undef,
+                $batchId, 
+                $ippToPspsDb, 
+                $filePath, 
+                $verbose, 
+                $save_temps);
+
+        if (!$processed) {
+
+            if ($removeUnprocessed && defined $filePath) {
+
+                if ($batch->deleteDir()) {$numDeleted++;}
+            }
+            next;
+        }
 
         $numBatchesToCheck++;
 
-        my $batchName = $batchManager->getBatchName($batchId);
-
         # if not merged then update by polling ODM for status
-        if (!$merged) {
-
-            if (checkODM($batchName, \$loadedToOdm, \$mergeWorthy, \$merged)) {$numChecked++;}
+        if (!$merged && !$loadFailed) {
+
+            if (checkODM($batch->getName(), \$loadedToOdm, \$loadFailed, \$mergeWorthy, \$merged)) {
+                
+                $numChecked++;
+                if ($merged) {$newMerged++;}
+            }
+            else {next;}
         }
 
         # delete from datastore
-        if (defined $product && !$deleted && $loadedToOdm && $mergeWorthy) {
-
-            $deleted = $datastore->remove($batchName);
+        if (defined $product && !$deleted && $loadedToOdm && ($mergeWorthy || $loadFailed)) {
+
+
+            $deleted = $datastore->remove($batch->getName());
             if ($deleted) {
                 $ippToPspsDb->setBatchAsDeleted($batchId, $expId);
@@ -122 +162 @@
         if (defined $filePath && $merged && $deleted) {
 
-            if($batchManager->deleteBatch($batchId)) {$numDeleted++;}
+            if($batch->deleteFile($batchId)) {$numDeleted++;}
         }
 
         # update database
-        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $mergeWorthy, $merged, $deleted);
-        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s     | %6s  |  %5s   |\n", 
+        $ippToPspsDb->updateODMStatus($batchId, $expId, $loadedToOdm, $loadFailed, $mergeWorthy, $merged, $deleted);
+        printf( "| %18s  | %11s  | %10d   |    %6s      |    %6s    |   %6s     | %6s  |  %5s   |\n", 
                 $timestamp, 
-                $batchName, 
+                $batch->getName(), 
                 $expId, 
                 $loadedToOdm ? "yes" : "no", 
+                $loadFailed ? "yes" : "no", 
                 $mergeWorthy ? "yes" : "no", 
                 $merged ? "yes" : "no",
@@ -137 +178 @@
     }
 
-    printf("+----------------------+--------------+--------------+----------------+---------------+---------+----------+\n");
-
+    printf("+----------------------+--------------+--------------+----------------+--------------+--------------+---------+----------+\n");
     printf( "* Successfully checked %d batch%s out of %d\n", $numChecked, ($numChecked==1) ? "" : "es", $numBatchesToCheck);
+    printf( "* %d newly merged batches\n", $newMerged);
     printf( "* Successfully removed %d batch%s from the datastore\n", $numRemovedFromDatastore, ($numRemovedFromDatastore==1) ? "" : "es");
     printf( "* Successfully deleted %d batch%s from local file system\n", $numDeleted, ($numDeleted==1) ? "" : "es");
@@ -151 +192 @@
 ########################################################################################
 sub checkODM {
-    my ($batchName, $loadedToOdm, $mergeWorthy, $merged) = @_;
+    my ($batchName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_;
 
     my $statusFilter = "*";
@@ -171 +212 @@
 
     my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_odmXml.XXXX", UNLINK => !$save_temps);
+    #print $response->content. "\n";
     print $tempFile $response->content;
     close($tempFile);
 
     ${$loadedToOdm} = 0;
+    ${$loadFailed} = 0;
     ${$mergeWorthy} = 0;
     ${$merged} = 0;
 
-    parseXml($tempName, $loadedToOdm, $mergeWorthy, $merged);
+    parseXml($tempName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged);
+
+    unlink($tempFile);
 
     return 1;
 }
 
-######################################################################################y
+########################################################################################
 # 
 # Parses ODM XML
@@ -189 +234 @@
 ########################################################################################
 sub parseXml {
-    my ($xmlFile, $loadedToOdm, $mergeWorthy, $merged) = @_;
+    my ($xmlFile, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_;
 
     my $parser = XML::LibXML->new;
@@ -195 +240 @@
     my $xc = XML::LibXML::XPathContext->new($doc);
     $xc->registerNs('ArrayOfOdmBatchState', 'PanSTARRS.Services.OdmWebService');
-    my $result = $xc->findvalue('//ArrayOfOdmBatchState:Message');
+    my $result = $xc->findvalue('//ArrayOfOdmBatchState:BatchState');
+
+    if ($result =~ m/failed/) { 
+        ${$loadFailed} = 1;
+        ${$loadedToOdm} = 1;
+    return;
+    }
+
+    $result = $xc->findvalue('//ArrayOfOdmBatchState:Message');
     ${$loadedToOdm} = 0;
     ${$mergeWorthy} = 0;