Changeset 28980 for branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl

Timestamp:

Aug 19, 2010, 6:50:13 PM (16 years ago)

Author:

eugene

Message:

updates from trunk

File:

• branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl (modified) (14 diffs)

branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl

@@ -6 +6 @@
 use PS::IPP::Config 1.01 qw( :standard );
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw(tempfile);
 use XML::LibXML;
+use File::Basename;
 
 # local classes
 use ippToPsps::Gpc1Db;
 use ippToPsps::IppToPspsDb;
+use ippToPsps::Datastore;
 
 # globals
 my $camera = 'GPC1';
 my $batchType = undef;
-my $survey = undef;
-my $dvodb = undef;
+my $dvoLocation = undef;
+my $dvoDb = undef;
+my $fullDvoPath = undef;
 my $verbose = undef;
 my $save_temps = undef; 
-my $no_update = undef;
 my $output = undef;
 my $singleExpId = undef;
@@ -34 +35 @@
         'output|o=s' => \$output,
         'batch|b=s' => \$batchType,
-        'dvodb|d=s' => \$dvodb,
-        'survey|s=s' => \$survey,
+        'dvo|d=s' => \$fullDvoPath,
         'expid|e=s' => \$singleExpId,
         'product|p=s' => \$datastoreProduct,
         'verbose|v' => \$verbose,
         'save_temps|t' => \$save_temps,
-        'no-update|u' => \$no_update,
         'force|f' => \$force,
         'tarnzip|z' => \$dontTarNZip,
@@ -49 +48 @@
 print "* \n";
 if (@ARGV) {
-    print "* UNKNKOWN: option:                         @ARGV\n"; $quit=1;}
+    $quit=1;
+    print "* UNKNKOWN: option                          @ARGV\n";
+}
 if (!defined $output) {
-    print "* REQUIRED: need to provide an output path: -o <path>\n"; $quit=1;}
+    $quit=1;
+    print "* REQUIRED: need to provide an output path  -o <path>\n";
+}
 if (!defined $batchType) {
-    print "* REQUIRED: need to define a batch type:    -b <init|det|diff|stack>\n"; $quit=1;}
-if (!defined $dvodb) {
-    print "* REQUIRED: need to provide a DVO Db path:  -d <path>\n"; $quit=1;}
-if (!defined $survey and !defined $singleExpId) {
-    print "* REQUIRED: need to provide a survey type:  -s <ThreePi|STS|SAS|M31|MD01|MD02|etc> ***OR***\n";
-    print "*           an exposure ID                  -e <expID>\n"; $quit=1;}
+    $quit=1;
+    print "* REQUIRED: need to define a batch type     -b <init|det|diff|stack>\n";
+}
+if (!defined $fullDvoPath) {
+    $quit=1;
+    print "* REQUIRED: need to provide a DVO Db        -d <pathToDVO>\n";
+}
+if (!defined $singleExpId) {
+
+    print "* OPTIONAL: a single exposure ID            -e <expID>           (default = none)\n";
+}
 if (!defined $datastoreProduct) {
-    print "* OPTIONAL: datastore product:              -p <product>\n";}
+
+    print "* OPTIONAL: datastore product               -p <product>         (default = none, i.e. data will not be published)\n";
+}
 if (!defined $verbose) {
-    print "* OPTIONAL: run in verbose mode:            -v\n"; $verbose = 0;}
+    $verbose = 0;
+    print "* OPTIONAL: run in verbose mode             -v                   (default = $verbose)\n";
+}
 if (!defined $save_temps) {
-    print "* OPTIONAL: keep temp files:                -t\n"; $save_temps = 0;}
-if (!defined $no_update) {
-    print "* OPTIONAL: don't update database:          -u\n"; $no_update = 0;}
+    $save_temps = 0;
+    print "* OPTIONAL: keep temp files                 -t                   (default = $save_temps)\n";
+}
 if (!defined $force) {
-    print "* OPTIONAL: force if already processed :    -f\n"; $force = 0;}
+    $force = 0;
+    print "* OPTIONAL: force if already processed      -f                   (default = $force)\n";
+}
 if (!defined $dontTarNZip) {
-    print "* OPTIONAL: don't tar and zip output :      -z\n"; $dontTarNZip = 0;}
-    print "*\n*******************************************************************************\n";
+    $dontTarNZip = 0;
+    print "* OPTIONAL: don't tar and zip output        -z                   (default = $dontTarNZip)\n";
+}
+print "*\n*******************************************************************************\n";
 
 if ($quit) { exit; }
+
+# determine PSPS batch 'type'
+my $pspsBatchType;
+if ($batchType eq 'init') {$pspsBatchType = "IN";}
+elsif ($batchType eq 'det') {$pspsBatchType = "P2";}
+elsif ($batchType eq 'stack') {$pspsBatchType = "ST";}
+elsif ($batchType eq 'diff') {$pspsBatchType = "OB";}
+else {$pspsBatchType = "UNKNOWN";}
+
+# spilt full DVO path into Db and path
+$fullDvoPath =~ s/\/$//; # strip off trailing '\'
+($dvoDb, $dvoLocation) = fileparse($fullDvoPath);
 
 my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
 my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+my $datastore = undef;
+if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);}
 
 # check we can run programs and get camera config
 my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR));
-my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR));
 my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); 
 
 if ($batchType eq "init") {$initBatch = 1;}
 else {$initBatch = 0;}
-process();
+
+if (!process()) {print "* Finished unsuccessfully\n";}
+else {print "* Finished successfully\n";}
 
 #######################################################################################
@@ -114 +145 @@
     my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
     my $lastExpId = $ippToPspsDb->getLastExpId();
-    my $rows = undef;
+    my $exposures;
 
     my $query;
     if ($initBatch) {
-   
-        my @row = (0, 'NULL', 'ThreePi');
-        @{$rows} = (\@row);
-    }
-    elsif ($singleExpId) { $rows = $gpc1Db->getExposureListFromExpId($singleExpId); }
-    else { $rows = $gpc1Db->getExposureListFromSurvey($survey); }
-
-# TODO check if there are no exposures and give a warning
+
+        my @exposure = (0, 'NULL', 'ThreePi');
+        @{$exposures} = (\@exposure);
+    }
+    # get single exposure
+    elsif ($singleExpId) {
+       
+        if (!$gpc1Db->getSingleExposureFromDvoDb($dvoDb, $singleExpId, \$exposures)) {return 0;}
+    }
+    # get all exposures in this DVO Db
+    else {
+        
+        if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures)) {return 0;}
+    }
 
     my $batchId = 0;
@@ -132 +169 @@
 
     #my $batchId = -1;
-    my $row;
-    foreach $row ( @{$rows} ) {
-        my ($expId, $expName, $distGroup) = @{$row};
-    #        print "JHGHGHGHGH2 $expId, $expName, $distGroup\n";
-
-    # loop round exposures
-    #while (my @row = $query->fetchrow_array()) {
-     #   my ($expId, $expName, $distGroup) = @row;
+    my $exposure;
+    foreach $exposure ( @{$exposures} ) {
+        my ($expId, $expName, $distGroup) = @{$exposure};
 
         if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) {
-                
-                if ($force) {print "* Forcing....\n";} 
-                else {next};
+
+            if ($force) {print "* Forcing....\n";} 
+            else {next};
         }
 
@@ -150 +182 @@
         if (!$surveyType) {next;}
 
-        $batchId = $ippToPspsDb->getNewBatchId($expId, $surveyType, $batchType);
+        $batchId = $ippToPspsDb->createNewBatch($expId, $distGroup, $pspsBatchType, $dvoDb, (defined $datastore) ? $datastoreProduct : "NONE");
 
         # TODO quit here if no sensible batch ID
@@ -156 +188 @@
         # generate batch path from batch IDs
         my $batch = sprintf("B%08d", $batchId);
-        my $batchDir = sprintf("$output/$batch");
+        my $dvoDir = "$output/$dvoDb";
+        my $batchDir = "$dvoDir/$batch";
+
+        # make directories
+        unless(-d $dvoDir) {mkdir($dvoDir, 0777);}
         mkdir($batchDir, 0777);
+
         $published = 0;
 
@@ -171 +208 @@
             if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) {
 
+                # tar n' zip
                 if (!$dontTarNZip) {
-                    my $tarball = tarAndZipBatch($output, $batch);
-                    if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;}
+                    my $tarball = tarAndZipBatch($dvoDir, $batch);
+
+                    # and publish
+                    if ($tarball && defined $datastore ) {
+                        
+                        $published = $datastore->register($batch, $dvoDir, $tarball, "IPP_PSPS", "tgz");
+                    }
                 }
             }
 
-            $ippToPspsDb->updateDb($batchId, $expId, 1, $published, $totalDetections);
+            $ippToPspsDb->updateBatch($batchId, $expId, 1, $published, $totalDetections, $minObjId, $maxObjId);
         }
 
@@ -330 +373 @@
     my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
 
-    # determine batch 'type'
-    my $type;
-    if ($batchType eq 'init') {$type = "IN";}
-    elsif ($batchType eq 'det') {$type = "P2";}
-    elsif ($batchType eq 'stack') {$type = "ST";}
-    elsif ($batchType eq 'diff') {$type = "OB";}
-    else {$type = "UNKNOWN";}
-
     # create XML file
     my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
@@ -372 +407 @@
                 $writer->startTag('manifest',
                         "name" => "$batch",
-                        "type" => $type,
+                        "type" => $pspsBatchType,
                         "survey" => $pspsSurvey,
                         "timestamp" => "$timeStamp",
@@ -382 +417 @@
                 $writer->startTag('manifest',
                         "name" => "$batch",
-                        "type" => $type,
+                        "type" => $pspsBatchType,
                         "timestamp" => "$timeStamp");
             }
@@ -404 +439 @@
 
 #######################################################################################
-#
-# register new job with the datastore
-#
-########################################################################################
-sub publishToDatastore {
-    my ($batch, $path, $tarball) = @_;
-
-    my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps);
-
-    print $tempFile $tarball . '|||tgz' . "\n";
-
-    # build dsreg command command
-    my $command  = "$dsreg";
-    $command .= " --add $batch";
-    $command .= " --copy";
-    $command .= " --datapath $path";
-    $command .= " --type IPP_PSPS";
-    $command .= " --product $datastoreProduct";
-    $command .= " --list $tempName";
-
-    # run command
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 };
-
-    print "* Successfully published $tarball to datastore\n";
-
-    close($tempFile);
-
-    return 1;
-}
-
-#######################################################################################
 # 
 # runs ippToPsps to produce init batch 
@@ -457 +458 @@
     my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
 
-    my $nebPath = $gpc1Db->getCameraStageSmf($expId);
+    my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId);
     if (!$nebPath) { return 0; }
 
@@ -527 +528 @@
     $command .= " -input $input";
     $command .= " -output $output";
-    $command .= " -D CATDIR $dvodb";
+    $command .= " -D CATDIR $fullDvoPath";
     $command .= " -config ../config"; # TODO
-    $command .= " -expid $expid";
+        $command .= " -expid $expid";
     $command .= " -expname $expName";
     $command .= " -survey $surveyType";