Index: trunk/ippToPsps/perl/exposureSummary.pl =================================================================== --- trunk/ippToPsps/perl/exposureSummary.pl (revision 29998) +++ trunk/ippToPsps/perl/exposureSummary.pl (revision 29999) @@ -11,5 +11,5 @@ use ippToPsps::IppToPspsDb; use ippToPsps::Gpc1Db; -use ippToPsps::Batch; +use ippToPsps::DetectionBatch; my $dvoDb = undef; @@ -17,4 +17,5 @@ my $endExp = undef; my $verbose = undef; +my $checkNeb = undef; my $save_temps = undef; @@ -23,4 +24,5 @@ 'endexpi|e=s' => \$endExp, 'dvodb|d=s' => \$dvoDb, + 'checkNeb|n' => \$checkNeb, 'verbose|v' => \$verbose, 'save_temps|s' => \$save_temps @@ -37,21 +39,25 @@ if (!defined $beginExp) { $quit = 1; - print "* REQUIRED: a begin exp_id -b \n"; + print "* REQUIRED: a begin exp_id -b \n"; } if (!defined $endExp) { $quit = 1; - print "* REQUIRED: an end exp_id -e \n"; + print "* REQUIRED: an end exp_id -e \n"; } if (!defined $dvoDb) { $quit = 1; - print "* REQUIRED: a dvo Db name -d \n"; + print "* REQUIRED: a dvo Db name -d \n"; +} +if (!defined $checkNeb) { + $checkNeb = 0; + print "* OPTIONAL: check nebulous for unprocessed exposures -n (default = $checkNeb)\n"; } if (!defined $verbose) { $verbose = 0; - print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; + print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; } if (!defined $save_temps) { $save_temps = 0; - print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; + print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; } print "*\n*******************************************************************************\n"; @@ -73,4 +79,7 @@ exit; } + +my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); +my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; my $exposure; @@ -82,9 +91,21 @@ my $totalLoadFailed = 0; my $totalUnderObjIdLimit = 0; -my @unprocessedExposures; -my @odmFailures; +my $totalUnprocessed = 0; +my $totalUnprocessedAvailableSmfs = 0; +my $totalOdmFailures = 0; +my $totalUnprocessedMissingSmfs = 0; + + +open (UNPRO, ">unprocessedExposures.txt") or print "* Problem opening data file for unprocessed exposures\n"; +open (ODMFAIL, ">odmFailures.txt") or print "* Problem opening data file for ODM failures\n"; + +if ($checkNeb) { + + open (MISSSMF, ">missingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; + open (NOTMISSSMF, ">notMissingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; +} foreach $exposure ( @{$exposures} ) { - my ($expId) = @{$exposure}; + my ($batchId, $expId) = @{$exposure}; if ($ippToPspsDb->isExposureMerged($expId)) {next;} @@ -114,6 +135,7 @@ } else { - - push(@odmFailures, $expId); + + $totalOdmFailures++; + print ODMFAIL "$expId\n"; } } @@ -122,14 +144,52 @@ else { - push(@unprocessedExposures, $expId); + + $totalUnprocessed++; + print UNPRO "$expId\n"; + + # if flag is set, then attempt to get smf file from nebulous for this unprocessed exposure + if ($checkNeb) { + + my $detectionBatch = ippToPsps::DetectionBatch->existing( + "GPC1", + $gpc1Db, + $batchId, + $ippToPspsDb, + ".", + $verbose, + $save_temps); + + + my $path = $detectionBatch->getSmfFile(); + if (!$path) { + print "no smf file for $expId\n"; + + $totalUnprocessedMissingSmfs++; + print MISSSMF "$expId\n"; + } + else { + + $totalUnprocessedAvailableSmfs++; + print NOTMISSSMF "$expId\n"; + + } + } + } } -my $totalUnprocessed = @unprocessedExposures; -my $totalOdmFailures = @odmFailures; +close(UNPRO); +close(ODMFAIL); +if ($checkNeb) { + + close(MISSSMF); + close(NOTMISSSMF); +} my $totalProcessedNotOnDatastore = $totalProcessed - $totalOnDatastore; my $totalOnDatastoreNotLoaded = $totalOnDatastore - $totalLoadedToOdm; print "***********************************************************\n"; -print "* Summary of IPP->PSPS loading from the '$dvoDb' DVO database for exposures between $beginExp and $endExp:\n"; +print "* Summary of IPP->PSPS loading as of $timeStamp\n"; +print "* DVO Db = '$dvoDb'\n"; +print "* Exposures between $beginExp and $endExp\n"; print "*\n"; print "* total Exposures = $totalExposures\n"; @@ -137,4 +197,8 @@ print "* un-merged = $totalUnmerged\n"; print "* unprocessed by ippToPsps = $totalUnprocessed\n"; +if ($checkNeb) { + print "* missing SMF files = $totalUnprocessedMissingSmfs\n"; + print "* available SMF files = $totalUnprocessedAvailableSmfs\n"; +} print "* processed by ippToPsps = $totalProcessed\n"; print "* not on datastore = $totalProcessedNotOnDatastore\n"; @@ -147,63 +211,3 @@ print "***********************************************************\n"; -my $gnuplotDat = "gnuplot.dat"; -open (FILE, ">$gnuplotDat") or print "* Problem opening data file for gnuplot\n"; - -print FILE "totalExposures $totalExposures"; -print FILE "mergedintoPSPS $totalMerged\n"; -print FILE "un-merged $totalUnmerged\n"; -print FILE "unprocessedbyippToPsps $totalUnprocessed\n"; -print FILE "processedbyippToPsps $totalProcessed\n"; -print FILE "notondatastore $totalProcessedNotOnDatastore\n"; -print FILE "ondatastore $totalOnDatastore\n"; -print FILE "notloadedtoODM $totalOnDatastoreNotLoaded\n"; -print FILE "loadedtoODMbutrejected $totalLoadedToOdm\n"; -print FILE "ODMfailures $totalLoadFailed\n"; -print FILE "under-30declimit $totalUnderObjIdLimit\n"; -print FILE "remainingODMfailures $totalOdmFailures\n"; -close (FILE); - - -# save list of unprocessed exposures to file -open (FILE, ">unprocessedExposures.txt") or print "* Problem opening data file for unprocessed exposures\n"; -foreach my $exp_id (@unprocessedExposures) { - - print FILE "$exp_id\n"; -} -close (FILE); - -# save list of ODM failed exposures to file -open (FILE, ">odmFailures.txt") or print "* Problem opening data file for ODM failures\n"; -foreach my $exp_id (@odmFailures) { - - print FILE "$exp_id\n"; -} -close (FILE); - - -sub plot { - - open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot"; - use FileHandle; - GP->autoflush(1); - - print GP - #"set term png font \"/usr/share/fonts/corefonts/arial.ttf\" 8;" . - "set term X11;" . - "set output \"summary.png\";" . - "set title \"Summary of IPP->PSPS loading from the '$dvoDb' DVO database\nfor exposures between $beginExp and $endExp\";" . - "set grid;" . - "set boxwidth;" . - "set style data histogram;" . - "set style fill solid border -1;" . - "set ylabel \"Exposures\";" . - "set boxwidth 0.75;" . - "plot '$gnuplotDat' using 2:xtic(1) notitle, '' using 3 notitle, '' using 4 notitle;" . - "\n"; - - close GP; - - -} - - + Index: trunk/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- trunk/ippToPsps/perl/ippToPsps_run.pl (revision 29998) +++ trunk/ippToPsps/perl/ippToPsps_run.pl (revision 29999) @@ -15,4 +15,5 @@ use ippToPsps::IppToPspsDb; use ippToPsps::Datastore; +#use ippToPsps::BatchManager; # globals @@ -112,4 +113,5 @@ my $datastore = undef; if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);} +#my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $output, $verbose, $save_temps); # check we can run programs and get camera config