Changeset 28304 for branches/czw_branch/20100519/ippToPsps/scripts/ippToPsps_run.pl

Timestamp:

Jun 10, 2010, 6:28:51 PM (16 years ago)

Author:

watersc1

Message:

Skycell Summary and stack Association stuff should be finished. I'll merge and do final tests on monday.

Location:

branches/czw_branch/20100519

Files:

• . (modified) (1 prop)
• ippToPsps/scripts/ippToPsps_run.pl (modified) (26 diffs)

branches/czw_branch/20100519/ippToPsps/scripts/ippToPsps_run.pl

@@ -12 +12 @@
 
 # globals
-my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $label, $singleExpId, $no_publish);
-
-# TODO temporary until we use database to store current jobid
-my $jobIdFilePath = "./config/currentJobId.txt";
-my $logFile = "./config/logfile.txt";
+my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $no_publish, $force, $initBatch);
 
 # default values for certain globals
@@ -28 +24 @@
 $singleExpId = undef;
 $datastoreProduct = "PSPS_test";
-
+$force = 0;
+$initBatch = 0;
 
 # get user args
@@ -35 +32 @@
         'batch|b=s' => \$batchType,
         'dvodb|d=s' => \$dvodb,
-        'label|l=s' => \$label,
+        'survey|s=s' => \$survey,
         'expid|e=s' => \$singleExpId,
         'product|p=s' => \$datastoreProduct,
@@ -43 +40 @@
         'no-op' => \$no_op,
         'no-update' => \$no_update,
+        'force|f' => \$force,
         ) or pod2usage( 2 );
 
@@ -52 +50 @@
         "--batch <init|det|diff|stack>\n".
         "--output <path>\n".
-        "--label <label> or --expid <expid>\n" .
+        "--survey <ThreePi|MD01,2 etc|STS|SAS|SweetSpot> or --expid <expid>\n" .
         "--dvodb <path>\n".
         "\n   Optional:\n\n".
@@ -65 +63 @@
 defined $output and
 defined $dvodb and
-( defined $label or defined $singleExpId );
+(( defined $survey or defined $singleExpId ) or ( $batchType eq "init"));
+
+if ($batchType eq "init") {$initBatch = 1;}
 
 # check we can run programs and get camera config
@@ -73 +73 @@
 
 # make a temporary file for saving results
-my ($resultsFile, $resultsFileName) = tempfile( "/tmp/results.XXXX", UNLINK => !$save_temps);
+my ($resultsFile, $resultsFileName) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
 
 process();
@@ -89 +89 @@
     if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";}
     if ($distGroup =~ m/^M31$/i) {return "M31";}
-    if ($distGroup =~ m/^sts$/i) {return "STS";}
+    if ($distGroup =~ m/^sts$/i) {return "STS1";}
     if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";}
     if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";}
@@ -97 +97 @@
 }
 
+
 #######################################################################################
 # 
-# Finds all exposures for the provided label and generates PSPS FITS data for them
-#
-#######################################################################################
-sub process {
-    my $lastExpId = getLastExpId();
-
-    print "*******************************************************************************\n*\n";
-    print "* Last successfully processed exposure ID was $lastExpId\n";
-
-    # Set up the database
-    my $dbname = 'gpc1';
-    my $dbserver = 'ippdb01';
-    my $dbuser = 'ippuser';
-    my $dbpass = 'ippuser';
+# Connects to a db
+# 
+########################################################################################
+sub connectToDb {
+    my ($dbname,$dbserver,$dbuser,$dbpass) = @_;
+
+
     my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
             "mysql_socket=" . DB_SOCKET(),
             ${dbuser},${dbpass},
             { RaiseError => 1, AutoCommit => 1}
-            ) or die "Unable to connect to database $DBI::errstr\n";
+            );
+    
+    printf("* Connection to '$dbname' on '$dbserver': %s\n", $db ? "success" : "FAILED ($DBI::errstr)");
+
+    return $db;
+}
+
+
+#######################################################################################
+# 
+# Finds all exposures for the provided TODO and generates PSPS FITS data for them
+#
+#######################################################################################
+sub process {
+
+    print "*******************************************************************************\n*\n";
+
+    my $gpc1Db = connectToDb("gpc1", "ippdb01", "ippuser", "ippuser") or return 0;
+    my $ippToPspsDb = connectToDb("ippToPsps", "ippdb01", "ipp", "ipp") or return 0;
+    my $lastExpId = getLastExpId($ippToPspsDb);
 
     my $query;
-    if (!$singleExpId) {
-
-        # query to retrieve all exposures with the provided label
-        $query = $db->prepare(<<SQL);
-
-        SELECT DISTINCT rawExp.exp_id, camRun.dist_group
-            FROM camRun, chipRun, rawExp
-            WHERE camRun.chip_id = chipRun.chip_id
-            AND chipRun.exp_id = rawExp.exp_id
-            AND camRun.label LIKE '%$label%'
-            AND rawExp.exp_id > $lastExpId
-            ORDER BY rawExp.exp_id ASC
+    if ($initBatch) {
+    
+        $query = $gpc1Db->prepare(<<SQL);
+
+        SELECT 0, 'NULL', 'ThreePi';
 SQL
+    }
+    elsif ($singleExpId) {
+
+        $query = $gpc1Db->prepare(<<SQL);
+
+        SELECT DISTINCT rawExp.exp_id,  rawExp.exp_name, dist_group 
+            FROM rawExp, chipRun 
+            WHERE chipRun.exp_id = rawExp.exp_id 
+            AND rawExp.exp_id = $singleExpId
+SQL
+    }
+    else {
+
+        $query = $gpc1Db->prepare(<<SQL);
+
+        SELECT rawExp.exp_id, rawExp.exp_name, camRun.dist_group 
+            FROM camRun, chipRun, rawExp 
+            WHERE camRun.state = 'full' 
+            AND camRun.chip_id = chipRun.chip_id 
+            AND chipRun.exp_id = rawExp.exp_id 
+            AND camRun.dist_group = '$survey'   
+            GROUP BY  rawExp.exp_id 
+            ORDER BY rawExp.exp_id ASC;
+SQL
+
+            #AND rawExp.dateobs <= '2010-03-12'
+            #AND rawExp.exp_id > $lastExpId
+            #AND camRun.label LIKE '%$label%'
             #AND rawExp.exp_id > 136561
             #AND camRun.dist_group LIKE 'sas'
             #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
     }
-    else {
-
-        $query = $db->prepare(<<SQL);
-
-        SELECT DISTINCT rawExp.exp_id, dist_group FROM rawExp, chipRun WHERE chipRun.exp_id = rawExp.exp_id AND rawExp.exp_id = $singleExpId
-SQL
-    }
 
     my $batchId = 0;
     my $batchesPerJob = 1;
     my $anyBatches = 0;
+    my $published = 0;
 
     $query->execute;
-
-    open (LOGFILE, ">> $logFile");
 
     my $jobId = -1;
@@ -157 +183 @@
     # loop round exposures
     while (my @row = $query->fetchrow_array()) {
-        my ($expId, $distGroup) = @row;
-
-        $jobId = getNewJobId();
-
-        print "* Exposure ID = $expId with dist group = '$distGroup' as job ID $jobId\n";
+        my ($expId, $expName, $distGroup) = @row;
+
+        if (!$initBatch && isExposureAlreadyProcessed($ippToPspsDb, $expId)) {
+                
+                if ($force) {print "* Forcing....\n";} 
+                else {next};
+        }
 
         my $surveyType = getSurveyTypeFromDistGroup($distGroup);
-
         if (!$surveyType) {next;}
-        print "* Using survey type of $surveyType\n";
+
+        $jobId = getNewBatchId($ippToPspsDb, $expId, $surveyType);
+
+        # TODO quit here if no sensible job ID
 
         # generate batch and job paths from job and batch IDs
@@ -173 +203 @@
         my $batchDir = sprintf("B%03d", $batchId);
         my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir);
-
-        print "* Preparing exposure $expId as job '$job'...\n";
-
-        # make directories
         mkdir($jobOutputPath, 0777);
         mkdir($batchOutputPath, 0777);
 
+        print "* Preparing exposure $expId as job '$job'...\n";
+
         # run IppToPsps program
-        if (runIppToPsps($db, $expId, $batchOutputPath, $batchType)) {
+        if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType)) {
 
             # write batch manifest and tar and zip up
@@ -198 +226 @@
             elsif ($anyBatches) {
 
+                $published = 0;
                 if (writeJobManifest($output, $job, $surveyType)) {
 
-                    if (!$no_publish && !publishToDatastore($output, $job)) { 
-
+                    if (!$no_publish && publishToDatastore($output, $job)) { 
+                        $published = 1;
                     }
                 }
 
-                # write to the log
-                print LOGFILE "$jobId $expId\n";
+                updateDb($ippToPspsDb, $jobId, $expId, 1, $published);
 
                 $batchId = 0;
@@ -214 +242 @@
 
                 print "* No batches to publish\n";
-                print LOGFILE "$jobId $expId ERROR\n";
             }
         }
-        else {
-            print LOGFILE "$jobId $expId ERROR\n";
-        }
-
-        if ($batchType eq "init" ) {print "* Wrote initialisation batch\n"; last;}
-
+
+        if ($initBatch) {print "* Wrote initialisation batch\n";}
 
         print "*\n*******************************************************************************\n\n";
     }
 
-    $db->disconnect();
-    close (LOGFILE);
+    $gpc1Db->disconnect();
+    $ippToPspsDb->disconnect();
 
     return 1;
@@ -239 +262 @@
 ########################################################################################
 sub getLastExpId {
-
-    my $lastExpId = `tail $logFile -n 1 | awk '{print \$2}'`;
-    chomp($lastExpId);
-    return $lastExpId;
-}
-
-#######################################################################################
-#
-# Gets the current jobId (ini file for now - will be Db eventually TODO) 
+    my ($db) = @_;
+
+    my $query = $db->prepare(<<SQL);
+
+    SELECT exp_id 
+        FROM batches 
+        WHERE processed = 1
+        ORDER BY exp_id 
+        DESC LIMIT 1;
+SQL
+
+    $query->execute;
+
+    my $expId = $query->fetchrow_array();
+
+    print "* Last successfully processed exposure ID was $expId\n";
+
+    return $expId; 
+}
+
+#######################################################################################
+#
+# Updates an existing database record 
 #
 ########################################################################################
-sub getNewJobId {
-
-    my $jobId = -1;
-
-    open (MYFILE, $jobIdFilePath);
-    while (<MYFILE>) {
-        chomp;
-        $jobId = $_;
-    }
-    close (MYFILE);
-
-    $jobId++;
-
-    open (MYFILE, "> $jobIdFilePath");
-    print MYFILE "$jobId";
-    close (MYFILE);
-
-    return $jobId;
+sub updateDb {
+    my ($db, $batchId, $expId, $processed, $published) = @_;
+
+    my $query = $db->prepare(<<SQL);
+
+    UPDATE batches 
+        SET processed = $processed, on_datastore = $published 
+        WHERE batch_id = $batchId 
+        AND exp_id = $expId;
+SQL
+
+    $query->execute; # TODO check response of these
+}
+
+#######################################################################################
+#
+# Checks whether we have successfully processed this exposure 
+#
+########################################################################################
+sub isExposureAlreadyProcessed {
+    my ($db, $expId) = @_;
+
+    my $query = $db->prepare(<<SQL);
+
+SELECT COUNT(*) 
+    FROM batches 
+    WHERE exp_id = $expId 
+    AND processed = 1;
+
+SQL
+
+    $query->execute;
+
+    my $processed = $query->fetchrow_array();
+
+    if ($processed) {print "* Exposure ID '$expId' has already been processed\n";}
+
+    return $processed;
+}
+
+#######################################################################################
+#
+# Generates a new record in the database with a new jobId 
+#
+########################################################################################
+sub getNewBatchId {
+    my ($db, $expId,$surveyType) = @_;
+
+    my $query = $db->prepare(<<SQL);
+
+    SELECT batch_id 
+        FROM batches 
+        ORDER BY batch_id 
+        DESC LIMIT 1;
+SQL
+
+    $query->execute;
+
+    my $batchId = $query->fetchrow_array();
+
+    $batchId++;
+
+    $query = $db->prepare(<<SQL);
+    INSERT INTO batches
+        (batch_id, exp_id, survey_id)
+        VALUES
+        ($batchId, $expId, '$surveyType');
+
+SQL
+
+    $query->execute;
+
+    print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n";
+
+    return $batchId;
 }
 
@@ -276 +370 @@
 #######################################################################################
 sub runIppToPsps {
-    my ($db, $expId, $batchOutputPath, $batchType) = @_;
+    my ($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType) = @_;
 
     my $success = 0;
 
-    # before anything else, we can publish the init batch
     if ($batchType =~ /init/) {
         $success = produceInit($batchOutputPath);
     }
     elsif ($batchType =~ /det/) {
-        $success = produceDetections($db, $expId, $batchOutputPath);
+        $success = produceDetections($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath);
     }
     elsif ($batchType =~ /diff/) {
-        $success = produceDiffs($db, $expId, $batchOutputPath);
+        $success = produceDiffs($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath);
     }
 
@@ -532 +625 @@
     my $fullJobPath = "$path/$job";
 
-    my ($tempFile, $tempName) = tempfile( "/tmp/dsregList.XXXX", UNLINK => !$save_temps);
+    my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps);
 
     # loop through all batch files in this job directory
@@ -582 +675 @@
     my ($outputPath) = @_;
 
-    my $success = runProgram( "NULL", $outputPath, 0, $batchType);
+    my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType);
     return $success;
 }
@@ -591 +684 @@
 #######################################################################################
 sub produceDetections {
-    my ($db,$expId,$outputPath) = @_;
+    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_;
 
     # query to retrieve nebulous key of camera smf file for this exposure
@@ -602 +695 @@
         "JOIN chipRun USING(chip_id) ".
         "JOIN rawExp USING (exp_id) ".
-        "WHERE rawExp.exp_id = $expId AND camRun.magicked";
+        "WHERE rawExp.exp_id = $expId AND camRun.magicked ".
+        "ORDER BY camRun.cam_id DESC LIMIT 1";  # make sure we get most recently processed
 
     if ($verbose) { print"* $query\n";}
@@ -608 +702 @@
 
     # execute query and check results
-    my $results = $db->selectall_arrayref( $query );
+    my $results = $gpc1Db->selectall_arrayref( $query );
     my $size = @$results;
     if ($size < 1) {print "* No smf files found for this exposure\n"; return 0;}
@@ -622 +716 @@
 
         # now write the path to this file out to temp file
-        my ($tempFile, $tempName) = tempfile( "/tmp/inputFileList.XXXX", UNLINK => !$save_temps);
+        my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$save_temps);
         print $tempFile $realFile . "\n";
         close $tempFile;
 
-        $success = runProgram($tempName, $outputPath, $expId, $batchType);
+        $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType);
         return $success;
     }
@@ -637 +731 @@
 #######################################################################################
 sub produceDiffs {
-    my ($db,$expId,$outputPath) = @_;
+    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_;
 
     my $query = 
@@ -653 +747 @@
 
     # execute query and check results
-    my $results = $db->selectall_arrayref( $query );
+    my $results = $gpc1Db->selectall_arrayref( $query );
     my $size = @$results;
     if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;}
@@ -675 +769 @@
     }
 
-    my $ret = runProgram($tempName, $outputPath, $expId, $batchType);
+    my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType);
 
     close $tempFile;
@@ -688 +782 @@
 #######################################################################################
 sub runProgram {
-    my ($input, $output, $expid, $batchType) = @_;
+    my ($input, $output, $expid, $expName, $surveyType, $batchType) = @_;
 
     # build command
@@ -697 +791 @@
     $command .= " -config config"; # TODO
     $command .= " -expid $expid";
+    $command .= " -expname $expName";
+    $command .= " -survey $surveyType";
     $command .= " -batch $batchType";
     $command .= " -results $resultsFileName";