Index: branches/czw_branch/20101203/ippToPsps/docs/loadingSummary.txt =================================================================== --- branches/czw_branch/20101203/ippToPsps/docs/loadingSummary.txt (revision 30118) +++ branches/czw_branch/20101203/ippToPsps/docs/loadingSummary.txt (revision 30118) @@ -0,0 +1,21 @@ +*********************************************************** +* Summary of IPP->PSPS loading as of 2010-12-09 12:05:49 +* DVO Db = 'ThreePi.V1' +* Exposures between 105437 and 213141 +* +* total Exposures = 35973 +* merged into PSPS = 32082 +* un-merged = 3890 +* unprocessed by ippToPsps = 3351 +* missing SMF files = 518 +* available SMF files = 2833 +* processed by ippToPsps = 539 +* not on datastore = 4 +* on datastore = 535 +* not loaded to ODM = 19 +* loaded to ODM but rejected = 516 +* under -30 dec limit = 493 +* remaining ODM failures = 23 +* +*********************************************************** + Index: branches/czw_branch/20101203/ippToPsps/perl/checkOdmStatus.pl =================================================================== --- branches/czw_branch/20101203/ippToPsps/perl/checkOdmStatus.pl (revision 29907) +++ branches/czw_branch/20101203/ippToPsps/perl/checkOdmStatus.pl (revision 30118) @@ -39,4 +39,7 @@ ); +if (@ARGV) { + print "* UNKNKOWN: option @ARGV\n"; +} if (!defined $product) { print "* OPTIONAL: a datastore product name -p \n"; @@ -159,7 +162,6 @@ $numBatchesToCheck++; - # if not merged then update by polling ODM for status + # if not merged or failed load, then update by polling ODM for status if (!$merged && !$loadFailed) { - if (checkODM($batch->getName(), \$loadedToOdm, \$loadFailed, \$mergeWorthy, \$merged)) { Index: branches/czw_branch/20101203/ippToPsps/perl/exposureSummary.pl =================================================================== --- branches/czw_branch/20101203/ippToPsps/perl/exposureSummary.pl (revision 30118) +++ branches/czw_branch/20101203/ippToPsps/perl/exposureSummary.pl (revision 30118) @@ -0,0 +1,213 @@ +#!/usr/bin/env perl + +# script to provide a summary of loading status given an upper and lower exposure ID limit + +use warnings; +use strict; + +use LWP::UserAgent; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use File::Temp qw(tempfile); +use ippToPsps::IppToPspsDb; +use ippToPsps::Gpc1Db; +use ippToPsps::DetectionBatch; + +my $dvoDb = undef; +my $beginExp = undef; +my $endExp = undef; +my $verbose = undef; +my $checkNeb = undef; +my $save_temps = undef; + +GetOptions( + 'startexp|b=s' => \$beginExp, + 'endexpi|e=s' => \$endExp, + 'dvodb|d=s' => \$dvoDb, + 'checkNeb|n' => \$checkNeb, + 'verbose|v' => \$verbose, + 'save_temps|s' => \$save_temps + ); + + +my $quit = 0; +print "\n*******************************************************************************\n"; +print "* \n"; +if (@ARGV) { + $quit=1; + print "* UNKNKOWN: option @ARGV\n"; +} +if (!defined $beginExp) { + $quit = 1; + print "* REQUIRED: a begin exp_id -b \n"; +} +if (!defined $endExp) { + $quit = 1; + print "* REQUIRED: an end exp_id -e \n"; +} +if (!defined $dvoDb) { + $quit = 1; + print "* REQUIRED: a dvo Db name -d \n"; +} +if (!defined $checkNeb) { + $checkNeb = 0; + print "* OPTIONAL: check nebulous for unprocessed exposures -n (default = $checkNeb)\n"; +} +if (!defined $verbose) { + $verbose = 0; + print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; +} +if (!defined $save_temps) { + $save_temps = 0; + print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; +} +print "*\n*******************************************************************************\n"; + +if ($quit) { exit; } + +my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps); +my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); + +my $exposures; +#if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures, 0)) {exit;} + +my $totalExposures = $ippToPspsDb->countExposures($beginExp, $endExp, "PSPS_test", "P2", $dvoDb); +my $totalMerged = $ippToPspsDb->countMergedExposures($beginExp, $endExp, "PSPS_test", "P2", $dvoDb); + +if(!$ippToPspsDb->getUnmergedExposures(\$exposures, $beginExp, $endExp, "PSPS_test", "P2", $dvoDb)) { + + print "No unmerged exposures\n"; + exit; +} + +my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); +my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; + +my $exposure; +my $totalUnmerged = 0; +my $totalMergeWorthy = 0; +my $totalProcessed = 0; +my $totalOnDatastore = 0; +my $totalLoadedToOdm = 0; +my $totalLoadFailed = 0; +my $totalUnderObjIdLimit = 0; +my $totalUnprocessed = 0; +my $totalUnprocessedAvailableSmfs = 0; +my $totalOdmFailures = 0; +my $totalUnprocessedMissingSmfs = 0; + + +open (UNPRO, ">unprocessedExposures.txt") or print "* Problem opening data file for unprocessed exposures\n"; +open (ODMFAIL, ">odmFailures.txt") or print "* Problem opening data file for ODM failures\n"; + +if ($checkNeb) { + + open (MISSSMF, ">missingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; + open (NOTMISSSMF, ">notMissingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; +} + +foreach $exposure ( @{$exposures} ) { + my ($batchId, $expId) = @{$exposure}; + + if ($ippToPspsDb->isExposureMerged($expId)) {next;} + + $totalUnmerged++; + if ($verbose) {print "$totalUnmerged | $expId \n"; } + + if ($ippToPspsDb->isExposureProcessed($expId)) { + + $totalProcessed++; + + if ($ippToPspsDb->isExposureAlreadyPublished($expId)) { + + $totalOnDatastore++; + + if ($ippToPspsDb->isExposureLoadedToOdm($expId)) { + + $totalLoadedToOdm++; + + if ($ippToPspsDb->didLoadFail($expId)) { + + $totalLoadFailed++; + } + if ($ippToPspsDb->isMinObjIdUnderLimit($expId)) { + + $totalUnderObjIdLimit++; + } + else { + + $totalOdmFailures++; + print ODMFAIL "$expId\n"; + } + } + } + } + else { + + + $totalUnprocessed++; + print UNPRO "$expId\n"; + + # if flag is set, then attempt to get smf file from nebulous for this unprocessed exposure + if ($checkNeb) { + + my $detectionBatch = ippToPsps::DetectionBatch->existing( + "GPC1", + $gpc1Db, + $batchId, + $ippToPspsDb, + ".", + $verbose, + $save_temps); + + + my $path = $detectionBatch->getSmfFile(); + if (!$path) { + print "no smf file for $expId\n"; + + $totalUnprocessedMissingSmfs++; + print MISSSMF "$expId\n"; + } + else { + + $totalUnprocessedAvailableSmfs++; + print NOTMISSSMF "$expId\n"; + + } + } + + } +} + +close(UNPRO); +close(ODMFAIL); +if ($checkNeb) { + + close(MISSSMF); + close(NOTMISSSMF); +} +my $totalProcessedNotOnDatastore = $totalProcessed - $totalOnDatastore; +my $totalOnDatastoreNotLoaded = $totalOnDatastore - $totalLoadedToOdm; +print "***********************************************************\n"; +print "* Summary of IPP->PSPS loading as of $timeStamp\n"; +print "* DVO Db = '$dvoDb'\n"; +print "* Exposures between $beginExp and $endExp\n"; +print "*\n"; +print "* total Exposures = $totalExposures\n"; +print "* merged into PSPS = $totalMerged\n"; +print "* un-merged = $totalUnmerged\n"; +print "* unprocessed by ippToPsps = $totalUnprocessed\n"; +if ($checkNeb) { + print "* missing SMF files = $totalUnprocessedMissingSmfs\n"; + print "* available SMF files = $totalUnprocessedAvailableSmfs\n"; +} +print "* processed by ippToPsps = $totalProcessed\n"; +print "* not on datastore = $totalProcessedNotOnDatastore\n"; +print "* on datastore = $totalOnDatastore\n"; +print "* not loaded to ODM = $totalOnDatastoreNotLoaded\n"; +print "* loaded to ODM but rejected = $totalLoadedToOdm\n"; +print "* under -30 dec limit = $totalUnderObjIdLimit\n"; +print "* remaining ODM failures = $totalOdmFailures\n"; +print "*\n"; +print "***********************************************************\n"; + + Index: branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/DetectionBatch.pm =================================================================== --- branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/DetectionBatch.pm (revision 29907) +++ branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/DetectionBatch.pm (revision 30118) @@ -78,5 +78,5 @@ # get neb path of smf file my $nebPath = $self->{_gpc1Db}->getCameraStageSmfForThisDvoDb($self->{_dvoDb}, $self->{_expId}); - if (!$nebPath) { return 0; } + if (!$nebPath) { return undef; } # get real filename from neb 'key' Index: branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/IppToPspsDb.pm =================================================================== --- branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 29907) +++ branches/czw_branch/20101203/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 30118) @@ -11,5 +11,5 @@ ########################################################################### # -# Returns a list of batches that have been processed and loaded to datastore +# Returns a list of batches created within the provided dates # ########################################################################### @@ -39,7 +39,5 @@ $query->execute; ${$batches} = $query->fetchall_arrayref(); - my $count = scalar @{${$batches}}; - - return $count; + return scalar @{${$batches}}; } @@ -72,7 +70,75 @@ $query->execute; ${$batches} = $query->fetchall_arrayref(); - my $count = scalar @{${$batches}}; - - return $count; + return scalar @{${$batches}}; +} + +########################################################################### +# +# Returns a count of all distinct exposures between provided limits +# +########################################################################### +sub countExposures { + my ($self, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; + + my $query = $self->{_db}->prepare(<= $fromExp + AND exp_id <= $toExp + AND dvo_db = '$dvoDb' + AND batch_type = '$batchType' + AND datastore_product = '$datastoreProduct' +SQL + + $query->execute; + return scalar $query->fetchrow_array(); +} + +########################################################################### +# +# Returns a count of exposures (between provided limits) that have been merged +# +########################################################################### +sub countMergedExposures { + my ($self, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; + + my $query = $self->{_db}->prepare(<= $fromExp + AND exp_id <= $toExp + AND dvo_db = '$dvoDb' + AND batch_type = '$batchType' + AND datastore_product = '$datastoreProduct' + AND merged +SQL + + $query->execute; + return scalar $query->fetchrow_array(); +} + +########################################################################### +# +# Returns a list of exposures (between provided limits) that have not been merged +# +########################################################################### +sub getUnmergedExposures { + my ($self, $exposures, $fromExp, $toExp, $datastoreProduct, $batchType, $dvoDb) = @_; + + my $query = $self->{_db}->prepare(<= $fromExp + AND exp_id <= $toExp + AND dvo_db = '$dvoDb' + AND batch_type = '$batchType' + AND datastore_product = '$datastoreProduct' + AND !merged + GROUP BY exp_id +SQL + + $query->execute; + ${$exposures} = $query->fetchall_arrayref(); + return scalar @{${$exposures}}; } @@ -166,4 +232,103 @@ ####################################################################################### # +# Checks whether this exposure failed to load to the ODM +# +######################################################################################## +sub didLoadFail { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether this exposure has a min object ID under old PSPS hard limit of 72010000000000001 +# +######################################################################################## +sub isMinObjIdUnderLimit { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} +####################################################################################### +# +# Checks whether this exposure has been processed +# +######################################################################################## +sub isExposureProcessed { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether this exposure has been merged +# +######################################################################################## +sub isExposureMerged { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether this exposure has been loaded to ODM +# +######################################################################################## +sub isExposureLoadedToOdm { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# # Checks whether we have successfully processed this exposure and loaded it to the datastore # @@ -184,4 +349,5 @@ my $processed = $query->fetchrow_array(); + # TODO can use return scalar $query->fetchrow_array(); return $processed; } Index: branches/czw_branch/20101203/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- branches/czw_branch/20101203/ippToPsps/perl/ippToPsps_run.pl (revision 29907) +++ branches/czw_branch/20101203/ippToPsps/perl/ippToPsps_run.pl (revision 30118) @@ -15,4 +15,5 @@ use ippToPsps::IppToPspsDb; use ippToPsps::Datastore; +#use ippToPsps::BatchManager; # globals @@ -112,4 +113,5 @@ my $datastore = undef; if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);} +#my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $output, $verbose, $save_temps); # check we can run programs and get camera config