Index: trunk/ippToPsps/scripts/ippToPsps_run.pl =================================================================== --- trunk/ippToPsps/scripts/ippToPsps_run.pl (revision 28660) +++ trunk/ippToPsps/scripts/ippToPsps_run.pl (revision 28661) @@ -13,5 +13,5 @@ # globals -my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $no_publish, $force, $initBatch); +my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $force, $initBatch, $help); # default values for certain globals @@ -20,11 +20,11 @@ $no_op = 0; $no_update = 0; -$no_publish = 0; $camera = 'GPC1'; $output = undef; $singleExpId = undef; -$datastoreProduct = "PSPS_test"; +$datastoreProduct = undef; $force = 0; $initBatch = 0; +$help = 0; # get user args @@ -36,5 +36,4 @@ 'expid|e=s' => \$singleExpId, 'product|p=s' => \$datastoreProduct, - 'no_publish' => \$no_publish, 'verbose|v' => \$verbose, 'save_temps' => \$save_temps, @@ -42,29 +41,20 @@ 'no-update' => \$no_update, 'force|f' => \$force, - ) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; - -# tell off user for not providing proper args -pod2usage( - -msg => "\n Required options:\n\n". - "--batch \n". - "--output \n". - "--survey or --expid \n" . - "--dvodb \n". - "\n Optional:\n\n". - "--product \n". - "--no_publish - won't publish to datastore\n". - "--save_temps - will save temporary files\n". - "--no-op - ?\n". - "--no-update - will not update database\n", - -exitval => 3 - ) unless -defined $batchType and -defined $output and -defined $dvodb and -(( defined $survey or defined $singleExpId ) or ( $batchType eq "init")); - -if ($batchType eq "init") {$initBatch = 1;} + 'help|h' => \$help, + ); + +if ($help) {printUsage(); exit;} +if (@ARGV) {print "* Unknown option: @ARGV\n"; printUsage(); exit;} + +my $quit = 0; +print "\n"; +if (!defined $batchType) {print "* ERROR: need to define a batch type (-b)\n"; $quit=1;} +if (!defined $output) {print "* ERROR: need to provide an output path (-o)\n"; $quit=1;} +if (!defined $dvodb) {print "* ERROR: need to provide a DVO Db path (-d)\n"; $quit=1;} +if (!defined $survey and !defined $singleExpId) {print "* ERROR: need to provide a survey type (-s) or an exposure ID (-e)\n"; $quit=1;} +print "\n\n"; + +if ($quit) { exit; } + # check we can run programs and get camera config @@ -74,9 +64,35 @@ # make a temporary file for saving results -my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); - + +if ($batchType eq "init") {$initBatch = 1;} process(); -close $resultsFile; + +####################################################################################### +# +# Generates a PSPS-suitable survey 'type' from the IPP distribution group +# +####################################################################################### +sub printUsage { + + print "\n" . + " *** Required options:\n". + "\n" . + " --batch \n". + " --output \n". + " --survey \n" . + " --expid \n" . + " --dvodb \n". + "\n" . + " *** Optional:\n". + "\n" . + " --product if not provided, data will NOT be published\n". + " --save_temps will save temporary files\n". + " --no-op ?\n". + " --no-update will not update database\n" . + " --force will force an already processed exposure\n" . + " --help show this message\n\n"; + +} ####################################################################################### @@ -128,4 +144,5 @@ print "*******************************************************************************\n*\n"; + my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); my $gpc1Db = connectToDb("gpc1", "ippdb01", "ippuser", "ippuser") or return 0; my $ippToPspsDb = connectToDb("ippToPsps", "ippdb01", "ipp", "ipp") or return 0; @@ -160,9 +177,10 @@ AND chipRun.exp_id = rawExp.exp_id AND camRun.dist_group = '$survey' - AND rawExp.dateobs >= '2010-02-20' GROUP BY rawExp.exp_id ORDER BY rawExp.exp_id ASC; SQL + #AND rawExp.exp_id > 133887 + #AND filter = 'r.00000' #AND rawExp.dateobs <= '2010-03-12' #AND rawExp.exp_id > $lastExpId @@ -173,12 +191,13 @@ } +# TODO check if there are no exposures and give a warning + my $batchId = 0; my $batchesPerJob = 1; - my $anyBatches = 0; my $published = 0; $query->execute; - my $jobId = -1; + #my $batchId = -1; # loop round exposures @@ -195,53 +214,30 @@ if (!$surveyType) {next;} - $jobId = getNewBatchId($ippToPspsDb, $expId, $surveyType); - - # TODO quit here if no sensible job ID - - # generate batch and job paths from job and batch IDs - my $job = sprintf("J%06d", $jobId); - my $jobOutputPath = sprintf("$output/$job"); - my $batchDir = sprintf("B%03d", $batchId); - my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir); - mkdir($jobOutputPath, 0777); - mkdir($batchOutputPath, 0777); - - print "* Preparing exposure $expId as job '$job'...\n"; + $batchId = getNewBatchId($ippToPspsDb, $expId, $surveyType); + + # TODO quit here if no sensible batch ID + + # generate batch path from batch IDs + my $batch = sprintf("B%08d", $batchId); + my $batchDir = sprintf("$output/$batch"); + mkdir($batchDir, 0777); + $published = 0; + + print "* Preparing exposure $expId as batch '$batch'...\n"; # run IppToPsps program - if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType)) { + if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath)) { + + my ($filename, $minObjId, $maxObjId, $totalDetections); + parseResults($resultsFilePath, \$filename, \$minObjId, \$maxObjId, \$totalDetections); # write batch manifest and tar and zip up - if (writeBatchManifest($batchOutputPath, $batchDir, $batchType)) { - - if (tarAndZipDirectory($jobOutputPath, $batchDir)) { - - $anyBatches = 1 - } + if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) { + + my $tarball = tarAndZipBatch($output, $batch); + if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;} } - if (($batchId+1) < $batchesPerJob) { - $batchId++; - } - # job is complete, now publish - elsif ($anyBatches) { - - $published = 0; - if (writeJobManifest($output, $job, $surveyType)) { - - if (!$no_publish && publishToDatastore($output, $job)) { - $published = 1; - } - } - - updateDb($ippToPspsDb, $jobId, $expId, 1, $published); - - $batchId = 0; - $anyBatches = 0; - } - else { - - print "* No batches to publish\n"; - } + updateDb($ippToPspsDb, $batchId, $expId, 1, $published, $totalDetections); } @@ -253,4 +249,5 @@ $gpc1Db->disconnect(); $ippToPspsDb->disconnect(); + close $resultsFile; return 1; @@ -289,10 +286,10 @@ ######################################################################################## sub updateDb { - my ($db, $batchId, $expId, $processed, $published) = @_; + my ($db, $batchId, $expId, $processed, $published, $totalDetections) = @_; my $query = $db->prepare(< $command, verbose => $verbose); - - if (!$success) { print "* Unable to tar up dir: $batch\n" and return 0 }; + if (!$success) { print "* Unable to tar up contents of dir: $batchDir\n" and return undef; } # zip @@ -411,21 +409,74 @@ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - - if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0 }; - + if (!$success) { print "* Unable to gzip: $batchTar\n" and return undef; } + + # remove tar file $command = "rm $batchTar"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to remove: $batchTar\n" }; + # remove batch dir $command = "rm -r $batchDir"; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to remove: $batch\n"}; - return 1; -} + return $tarball; +} + +####################################################################################### +# +# zip a file then delete it +# +####################################################################################### +sub zipFile { + my ($path,$origFile) = @_; + + my $file; + + # zip + my $zippedName = "$origFile.gz"; + my $command = "gzip -c $path/$origFile > $path/$zippedName"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $command, verbose => $verbose); + + if (!$success) { + + $file = $origFile; + print "* Unable to gzip: $path/$origFile\n"; + } + else { + + $file = $zippedName; + + # now delete original + $command = "rm -r $path/$origFile"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if (!$success) { print "* Unable to remove: $path/$origFile\n"}; + } + + return $file; +} + + +####################################################################################### +# +# parse results XML +# +####################################################################################### +sub parseResults { + my ($resultsFilePath, $filename, $minObjId, $maxObjId, $totalDetections) = @_; + + my $parser = XML::LibXML->new; + my $doc = $parser->parse_file($resultsFilePath); + ${$filename} = $doc->findvalue('//filename'); + ${$minObjId} = $doc->findvalue('//minObjID'); + ${$maxObjId} = $doc->findvalue('//maxObjID'); + ${$totalDetections} = $doc->findvalue('//totalDetections'); +} + ####################################################################################### @@ -435,6 +486,5 @@ ####################################################################################### sub writeBatchManifest { - - my ($path,$batch,$batchType) = @_; + my ($path,$batch,$batchType,$surveyType,$filename,$minObjId,$maxObjId) = @_; use XML::Writer; @@ -451,16 +501,9 @@ # determine batch 'type' my $type; - if($batchType eq 'init') {$type = "IN";} - elsif($batchType eq 'det') {$type = "P2";} + if ($batchType eq 'init') {$type = "IN";} + elsif ($batchType eq 'det') {$type = "P2";} elsif ($batchType eq 'stack') {$type = "ST";} elsif ($batchType eq 'diff') {$type = "OB";} else {$type = "UNKNOWN";} - - # read results - my $parser = XML::LibXML->new; - my $doc = $parser->parse_file($resultsFilePath); - my $filename = $doc->findvalue('//filename'); - my $minObjId = $doc->findvalue('//minObjID'); - my $maxObjId = $doc->findvalue('//maxObjID'); # create XML file @@ -475,5 +518,4 @@ my $foundFits = 0; - foreach my $f (@thefiles) { @@ -481,4 +523,6 @@ if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;} + + # my $file = zipFile($path, $f); # get md5sum of file @@ -494,4 +538,5 @@ "name" => "$batch", "type" => $type, + "survey" => $surveyType, "timestamp" => "$timeStamp", "minObjId" => $minObjId, @@ -511,16 +556,5 @@ "md5" => $md5sum); - #$writer->startTag('file' ); - #$writer->startTag('name'); $writer->characters("$f"); $writer->endTag(); - #$writer->startTag('bytes'); $writer->characters("$filesize"); $writer->endTag(); - #$writer->startTag('md5'); $writer->characters("$md5sum"); $writer->endTag(); - - #if($batchType eq 'det' && $filename eq $f) { - - # $writer->startTag('minObjId'); $writer->characters("$minObjId"); $writer->endTag(); - # $writer->startTag('maxObjId'); $writer->characters("$maxObjId"); $writer->endTag(); - #} $writer->endTag(); - # finish up $writer->endTag(); $writer->end(); @@ -530,5 +564,4 @@ } - if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;} return 1; @@ -537,105 +570,19 @@ ####################################################################################### # -# write job manifest -# -####################################################################################### -sub writeJobManifest { - my ($path, $job, $surveyType) = @_; - - use XML::Writer; - use Digest::MD5::File qw( file_md5_hex ); - - # sort out paths - my $outputPath = "$path/$job/JobManifest.xml"; - my $output = new IO::File(">$outputPath"); - - # create a timestamp - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); - my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; - - # create XML file - my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); - $writer->xmlDecl('UTF-8'); - $writer->doctype('manifest', "", "psps-manifest.dtd"); - $writer->startTag('manifest', - "name" => "$job", - "type" => $surveyType, - "timestamp" => "$timeStamp"); - - my $jobPath = "$path/$job"; - - # open directory to hunt for FITS files - opendir(DIR, $jobPath) or print "* Cannot open '$path' to write job manifest\n" and return 0; - my @thefiles= readdir(DIR); - closedir(DIR); - - my $foundBatch = 0; - - foreach my $f (@thefiles) { - - if (($f =~ /\.tar\.gz/)) { - - my $filePath = "$jobPath/$f"; - - # get md5sum of file - my $md5sum = file_md5_hex($filePath); - - # get size of file in bytes - my $filesize = -s $filePath; - - # write the tag for this batch - $writer->startTag('file', - "name" => $f, - "bytes" => "$filesize", - "md5" => "$md5sum", - ); - $writer->endTag(); - $foundBatch = 1; - } - } - - # finish up - $writer->endTag(); - $writer->end(); - - if (!$foundBatch) { print "* Could not find any batches for this job\n"; return 0;} - return 1; -} - -####################################################################################### -# # register new job with the datastore # ######################################################################################## sub publishToDatastore { - my ($path, $job) = @_; - - my $fullJobPath = "$path/$job"; + my ($batch, $path, $tarball) = @_; my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps); - # loop through all batch files in this job directory - opendir(DIR, $fullJobPath) or print "* Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0; - my @thefiles= readdir(DIR); - closedir(DIR); - - my $foundBatch = 0; - - foreach my $f (@thefiles) { - - # get all batch tgz files - if ($f =~ /\.tar.gz/) {print $tempFile $f . '|||tgz' . "\n"; $foundBatch = 1;} - - # get JobManifest XML file - elsif ($f =~ /\.xml/) {print $tempFile $f . '|||xml' . "\n";} - } - - if (!$foundBatch) { print "Could not find any batches to publish\n"; return 0;} + print $tempFile $tarball . '|||tgz' . "\n"; # build dsreg command command my $command = "$dsreg"; - $command .= " --add $job"; + $command .= " --add $batch"; $command .= " --copy"; - $command .= " --datapath $fullJobPath"; + $command .= " --datapath $path"; $command .= " --type IPP_PSPS"; $command .= " --product $datastoreProduct"; @@ -646,7 +593,7 @@ run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to publish $job to datastore\n" and return 0 }; - - print "* Successfully published $job to datastore\n"; + if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 }; + + print "* Successfully published $tarball to datastore\n"; close($tempFile); @@ -661,9 +608,10 @@ ####################################################################################### sub produceInit { - my ($outputPath) = @_; - - my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType); + my ($outputPath, $resultsFilePath) = @_; + + my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType, $resultsFilePath); return $success; } + ####################################################################################### # @@ -672,5 +620,5 @@ ####################################################################################### sub produceDetections { - my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_; + my ($gpc1Db, $expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; # query to retrieve nebulous key of camera smf file for this exposure @@ -708,5 +656,5 @@ close $tempFile; - $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType); + $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); return $success; } @@ -719,5 +667,5 @@ ####################################################################################### sub produceDiffs { - my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_; + my ($gpc1Db, $expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; my $query = @@ -757,5 +705,5 @@ } - my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType); + my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); close $tempFile; @@ -770,5 +718,5 @@ ####################################################################################### sub runProgram { - my ($input, $output, $expid, $expName, $surveyType, $batchType) = @_; + my ($input, $output, $expid, $expName, $surveyType, $batchType, $resultsFilePath) = @_; # build command @@ -790,5 +738,2 @@ return $success; } - - -