Index: trunk/ippToPsps/perl/ippToPsps/Gpc1Db.pm =================================================================== --- trunk/ippToPsps/perl/ippToPsps/Gpc1Db.pm (revision 28698) +++ trunk/ippToPsps/perl/ippToPsps/Gpc1Db.pm (revision 28698) @@ -0,0 +1,120 @@ +#!/usr/bin/perl i-w + +package ippToPsps::Gpc1Db; + +use warnings; +use strict; + +use ippToPsps::MySQLDb; +our @ISA = qw(ippToPsps::MySQLDb); # inherits from MySQLDb + +########################################################################### +# +# Returns diff-stage cmf files for this exposure +# +########################################################################### +sub getDiffStageCmfs { + my ($self, $expId) = @_; + + # DESC and LIMIT 1 is to make sure we get most recently processed + my $query = $self->{_db}->prepare(<execute; + return $query->fetchall_arrayref(); +} + +########################################################################### +# +# Returns camera-stage smf files for this exposure +# +########################################################################### +sub getCameraStageSmf { + my ($self, $expId) = @_; + + # DESC and LIMIT 1 is to make sure we get most recently processed + my $query = $self->{_db}->prepare(<execute; + return $query->fetchrow_array(); +} + +########################################################################### +# +# Returns a list of exposure IDs for this survey +# +########################################################################### +sub getExposureListFromSurvey { + my ($self, $survey) = @_; + + my $query = $self->{_db}->prepare(< 136561 + GROUP BY rawExp.exp_id + ORDER BY rawExp.exp_id ASC; +SQL + + + #AND rawExp.exp_id > 133887 + #AND filter = 'r.00000' + #AND rawExp.dateobs <= '2010-03-12' + #AND rawExp.exp_id > $lastExpId + #AND camRun.label LIKE '%$label%' + #AND rawExp.exp_id > 136561 + #AND camRun.dist_group LIKE 'sas' + #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range + + $query->execute; + return $query->fetchall_arrayref(); +} + +########################################################################### +# +# Returns info for this exposure ID +# +########################################################################### +sub getExposureListFromExpId { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + return $query->fetchall_arrayref(); +} + + + +1; Index: trunk/ippToPsps/perl/ippToPsps/IppToPsps.pm =================================================================== --- trunk/ippToPsps/perl/ippToPsps/IppToPsps.pm (revision 28698) +++ trunk/ippToPsps/perl/ippToPsps/IppToPsps.pm (revision 28698) @@ -0,0 +1,43 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use DBI; + +package ippToPsps::IppToPsps; + +########################################################################### +# +# Constructor +# +########################################################################### +sub new { + + my $class = shift; + my $self = { + _dbName => shift, + _dbHost => shift, + _dbUser => shift, + _dbPass => shift, + _verbose => shift, + }; + + my $dbname = $self->{_dbName}; + my $dbhost = $self->{_dbHost}; + my $dbuser = $self->{_dbUser}; + my $dbpass = $self->{_dbPass}; + + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + + $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ); + + if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");} + + bless $self, $class; + return $self; +} + Index: trunk/ippToPsps/perl/ippToPsps/IppToPspsDb.pm =================================================================== --- trunk/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 28698) +++ trunk/ippToPsps/perl/ippToPsps/IppToPspsDb.pm (revision 28698) @@ -0,0 +1,114 @@ +#!/usr/bin/perl i-w + +package ippToPsps::IppToPspsDb; + +use warnings; +use strict; + +use ippToPsps::MySQLDb; +our @ISA = qw(ippToPsps::MySQLDb); # inherits from MySQLDb + +####################################################################################### +# +# Gets the last processed exposure ID (ini file for now - will be Db eventually TODO) +# +######################################################################################## +sub getLastExpId { + my ($self) = @_; + + my $query = $self->{_db}->prepare(<execute; + + my $expId = $query->fetchrow_array(); + + if ($self->{_verbose}) {print "* Last successfully processed exposure ID was $expId\n";} + + return $expId; +} + +####################################################################################### +# +# Updates an existing database record +# +######################################################################################## +sub updateDb { + my ($self, $batchId, $expId, $processed, $published, $totalDetections) = @_; + + my $query = $self->{_db}->prepare(<execute; # TODO check response of this +} + +####################################################################################### +# +# Checks whether we have successfully processed this exposure +# +######################################################################################## +sub isExposureAlreadyProcessed { + my ($self, $expId) = @_; + + my $query = $self->{_db}->prepare(<execute; + + my $processed = $query->fetchrow_array(); + + if ($processed) {print "* Exposure ID '$expId' has already been processed\n";} + + return $processed; +} + +####################################################################################### +# +# Generates a new record in the database with a new batch ID +# +######################################################################################## +sub getNewBatchId { + my ($self, $expId, $surveyType) = @_; + + my $query = $self->{_db}->prepare(<execute; + + my $batchId = $query->fetchrow_array(); + + $batchId++; + + my $query = $self->{_db}->prepare(<execute; + + print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n"; + + return $batchId; +} + + Index: trunk/ippToPsps/perl/ippToPsps/MySQLDb.pm =================================================================== --- trunk/ippToPsps/perl/ippToPsps/MySQLDb.pm (revision 28698) +++ trunk/ippToPsps/perl/ippToPsps/MySQLDb.pm (revision 28698) @@ -0,0 +1,134 @@ +#!/usr/bin/perl -w + +use warnings; +use strict; +use DBI; + +package ippToPsps::MySQLDb; + +########################################################################### +# +# Constructor +# +########################################################################### +sub new { + + my $class = shift; + my $self = { + _dbName => shift, + _dbHost => shift, + _dbUser => shift, + _dbPass => shift, + _verbose => shift, + }; + + my $dbname = $self->{_dbName}; + my $dbhost = $self->{_dbHost}; + my $dbuser = $self->{_dbUser}; + my $dbpass = $self->{_dbPass}; + + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + + $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ); + + if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");} + + bless $self, $class; + return $self; +} + +########################################################################### +# +# Sets the date format to be used +# +########################################################################### +sub setDateFormat { + my ($self, $dateFormat) = @_; + $self->{_dateFormat} = $dateFormat if defined($dateFormat); + return $self->{_dateFormat}; +} + +sub setDbHost { + my ( $self, $dbHost ) = @_; + $self->{_dbHost} = $dbHost if defined($dbHost); + return $self->{_dbHost}; +} + +sub getDbHost { + my( $self ) = @_; + return $self->{_dbHost}; +} + + +########################################################################### +# +# Returns 'now' as a timestamp +# +########################################################################### +sub getIntervalInPast { + my ($self, $interval) = @_; + + my $query = $self->{_db}->prepare(<execute; + +return scalar $query->fetchrow_array(); +} + + +########################################################################### +# +# Returns 'now' as a timestamp +# +########################################################################### +sub getNowTimestamp { + my ($self) = @_; + + my $query = $self->{_db}->prepare(<execute; + +return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Checks whether a certain table exists +# +####################################################################################### +sub doesTableExist { + my ($self, $table) = @_; + + my $query = $self->{_db}->prepare(<execute; + + return scalar $query->fetchrow_array(); +} + +####################################################################################### +# +# Destructor +# +####################################################################################### +sub DESTROY { + my($self) = @_; + + if ($self->{_verbose}) { + print "* " . ref($self) . " is disconnecting from " . $self->{_dbName} . "@" . $self->{_dbHost} . "\n"; + } + #$self->{_db}->disconnect(); +} +1; + Index: trunk/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- trunk/ippToPsps/perl/ippToPsps_run.pl (revision 28698) +++ trunk/ippToPsps/perl/ippToPsps_run.pl (revision 28698) @@ -0,0 +1,536 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw(tempfile); +use XML::LibXML; + +# local classes +use ippToPsps::Gpc1Db; +use ippToPsps::IppToPspsDb; + + +# globals +my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $force, $initBatch, $help); + +# default values for certain globals +$verbose = undef; +$save_temps = undef; +$no_update = undef; +$camera = 'GPC1'; +$output = undef; +$singleExpId = undef; +$datastoreProduct = undef; +$force = undef; +$initBatch = 0; + +# get user args +GetOptions( + 'output|o=s' => \$output, + 'batch|b=s' => \$batchType, + 'dvodb|d=s' => \$dvodb, + 'survey|s=s' => \$survey, + 'expid|e=s' => \$singleExpId, + 'product|p=s' => \$datastoreProduct, + 'verbose|v' => \$verbose, + 'save_temps|t' => \$save_temps, + 'no-update|u' => \$no_update, + 'force|f' => \$force, + ); + +my $quit = 0; +print "\n*******************************************************************************\n"; +print "* \n"; +if (@ARGV) { + print "* UNKNKOWN: option: @ARGV\n"; $quit=1;} +if (!defined $output) { + print "* REQUIRED: need to provide an output path: -o \n"; $quit=1;} +if (!defined $batchType) { + print "* REQUIRED: need to define a batch type: -b \n"; $quit=1;} +if (!defined $dvodb) { + print "* REQUIRED: need to provide a DVO Db path: -d \n"; $quit=1;} +if (!defined $survey and !defined $singleExpId) { + print "* REQUIRED: need to provide a survey type: -s ***OR***\n"; + print "* an exposure ID -e \n"; $quit=1;} +if (!defined $datastoreProduct) { + print "* OPTIONAL: datastore product: -p \n";} +if (!defined $verbose) { + print "* OPTIONAL: run in verbose mode: -v\n"; $verbose = 0;} +if (!defined $save_temps) { + print "* OPTIONAL: keep temp files: -t\n"; $save_temps = 0;} +if (!defined $no_update) { + print "* OPTIONAL: don't update database: -u\n"; $no_update = 0;} +if (!defined $force) { + print "* OPTIONAL: force if already processed : -f\n"; $force = 0;} + print "*\n*******************************************************************************\n"; + +if ($quit) { exit; } + +my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose); +my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose); + +# check we can run programs and get camera config +my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR)); +my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR)); +my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); + +if ($batchType eq "init") {$initBatch = 1;} +process(); + +####################################################################################### +# +# Generates a PSPS-suitable survey 'type' from the IPP distribution group +# +####################################################################################### +sub getSurveyTypeFromDistGroup { + my ($distGroup) = @_; + + if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";} + if ($distGroup =~ m/^M31$/i) {return "M31";} + if ($distGroup =~ m/^sts$/i) {return "STS1";} + if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";} + if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";} + + print "* Unknown distribution group: '$distGroup'\n"; + return undef; +} + +####################################################################################### +# +# Finds all exposures for the provided TODO and generates PSPS FITS data for them +# +####################################################################################### +sub process { + + my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); + my $lastExpId = $ippToPspsDb->getLastExpId(); + my $rows = undef; + + my $query; + if ($initBatch) { + + my @row = (0, 'NULL', 'ThreePi'); + @{$rows} = (\@row); + } + elsif ($singleExpId) { $rows = $gpc1Db->getExposureListFromExpId($singleExpId); } + else { $rows = $gpc1Db->getExposureListFromSurvey($survey); } + +# TODO check if there are no exposures and give a warning + + my $batchId = 0; + my $batchesPerJob = 1; + my $published = 0; + + #my $batchId = -1; + my $row; + foreach $row ( @{$rows} ) { + my ($expId, $expName, $distGroup) = @{$row}; + # print "JHGHGHGHGH2 $expId, $expName, $distGroup\n"; + + # loop round exposures + #while (my @row = $query->fetchrow_array()) { + # my ($expId, $expName, $distGroup) = @row; + + if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) { + + if ($force) {print "* Forcing....\n";} + else {next}; + } + + my $surveyType = getSurveyTypeFromDistGroup($distGroup); + if (!$surveyType) {next;} + + $batchId = $ippToPspsDb->getNewBatchId($expId, $surveyType); + + # TODO quit here if no sensible batch ID + + # generate batch path from batch IDs + my $batch = sprintf("B%08d", $batchId); + my $batchDir = sprintf("$output/$batch"); + mkdir($batchDir, 0777); + $published = 0; + + print "* Preparing exposure $expId as batch '$batch'...\n"; + + # run IppToPsps program + if (runIppToPsps($expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath)) { + + my ($filename, $minObjId, $maxObjId, $totalDetections); + parseResults($resultsFilePath, \$filename, \$minObjId, \$maxObjId, \$totalDetections); + + # write batch manifest and tar and zip up + if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) { + + my $tarball = tarAndZipBatch($output, $batch); + if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;} + } + + $ippToPspsDb->updateDb($batchId, $expId, 1, $published, $totalDetections); + } + + if ($initBatch) {print "* Wrote initialisation batch\n";} + + print "*\n*******************************************************************************\n\n"; + } + + close $resultsFile; + + return 1; +} + +####################################################################################### +# +# run IppToPsps +# +####################################################################################### +sub runIppToPsps { + my ($expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath) = @_; + + my $success = 0; + + if ($batchType =~ /init/) { + $success = produceInit($batchDir, $resultsFilePath); + } + elsif ($batchType =~ /det/) { + $success = produceDetections($expId, $expName, $surveyType, $batchDir, $resultsFilePath); + } + elsif ($batchType =~ /diff/) { + $success = produceDiffs($expId, $expName, $surveyType, $batchDir, $resultsFilePath); + } + elsif ($batchType =~ /test/) { + $success = produceDetections($expId, $expName, $surveyType, $batchDir, $resultsFilePath); + } + + return $success; +} + +####################################################################################### +# +# tar 'n zip a batch TODO handle errors +# +####################################################################################### +sub tarAndZipBatch { + my ($path,$batch) = @_; + + my $batchDir = "$path/$batch"; + my $tar = "$batch.tar"; + my $tarball = "$tar.gz"; + my $batchTar = "$path/$tar"; + my $batchZip = "$path/$tarball"; + + # tar + my $command = "tar -cvf $batchTar -C $path $batch"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { print "* Unable to tar up contents of dir: $batchDir\n" and return undef; } + + # zip + $command = "gzip -c $batchTar > $batchZip"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { print "* Unable to gzip: $batchTar\n" and return undef; } + + # remove tar file + $command = "rm $batchTar"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { print "* Unable to remove: $batchTar\n" }; + + # remove batch dir + $command = "rm -r $batchDir"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + if (!$success) { print "* Unable to remove: $batch\n"}; + + return $tarball; +} + +####################################################################################### +# +# zip a file then delete it +# +####################################################################################### +sub zipFile { + my ($path,$origFile) = @_; + + my $file; + + # zip + my $zippedName = "$origFile.gz"; + my $command = "gzip -c $path/$origFile > $path/$zippedName"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $command, verbose => $verbose); + + if (!$success) { + + $file = $origFile; + print "* Unable to gzip: $path/$origFile\n"; + } + else { + + $file = $zippedName; + + # now delete original + $command = "rm -r $path/$origFile"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if (!$success) { print "* Unable to remove: $path/$origFile\n"}; + } + + return $file; +} + + +####################################################################################### +# +# parse results XML +# +####################################################################################### +sub parseResults { + my ($resultsFilePath, $filename, $minObjId, $maxObjId, $totalDetections) = @_; + + my $parser = XML::LibXML->new; + my $doc = $parser->parse_file($resultsFilePath); + ${$filename} = $doc->findvalue('//filename'); + ${$minObjId} = $doc->findvalue('//minObjID'); + ${$maxObjId} = $doc->findvalue('//maxObjID'); + ${$totalDetections} = $doc->findvalue('//totalDetections'); +} + + +####################################################################################### +# +# writes a batch manifest file for the one FITS file that should be in the path passed in +# +####################################################################################### +sub writeBatchManifest { + my ($path,$batch,$batchType,$surveyType,$filename,$minObjId,$maxObjId) = @_; + + use XML::Writer; + use Digest::MD5::File qw( file_md5_hex ); + + # sort out paths + my $outputPath = "$path/BatchManifest.xml"; + my $output = new IO::File(">$outputPath"); + + # create a timestamp + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); + my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; + + # determine batch 'type' + my $type; + if ($batchType eq 'init') {$type = "IN";} + elsif ($batchType eq 'det') {$type = "P2";} + elsif ($batchType eq 'stack') {$type = "ST";} + elsif ($batchType eq 'diff') {$type = "OB";} + else {$type = "UNKNOWN";} + + # create XML file + my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); + $writer->xmlDecl('UTF-8'); + $writer->doctype('manifest', "", "psps-manifest.dtd"); + + # open directory to hunt for FITS files + opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0; + my @thefiles= readdir(DIR); + closedir(DIR); + + my $foundFits = 0; + foreach my $f (@thefiles) { + + if ($f =~ /\.FITS/) { + + if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;} + + # my $file = zipFile($path, $f); + + # get md5sum of file + my $md5sum = file_md5_hex("$path/$f"); + + # get size of file in bytes + my $filesize = -s "$path/$f"; + + # write manifest element TODO untidy + if($batchType eq 'det' && $filename eq $f) { + + my $pspsSurvey = undef; + if ($surveyType =~ m/^MD[0-1][0-9]$/i) {$pspsSurvey = "MDF";} + else {$pspsSurvey = $surveyType;} + + $writer->startTag('manifest', + "name" => "$batch", + "type" => $type, + "survey" => $pspsSurvey, + "timestamp" => "$timeStamp", + "minObjId" => $minObjId, + "maxObjId" => $maxObjId); + } + else { + + $writer->startTag('manifest', + "name" => "$batch", + "type" => $type, + "timestamp" => "$timeStamp"); + } + # write the tag for this batch + $writer->startTag("file", + "name" => $f, + "bytes" => $filesize, + "md5" => $md5sum); + + $writer->endTag(); + $writer->endTag(); + $writer->end(); + $foundFits = 1; + } + + } + + if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;} + return 1; +} + +####################################################################################### +# +# register new job with the datastore +# +######################################################################################## +sub publishToDatastore { + my ($batch, $path, $tarball) = @_; + + my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps); + + print $tempFile $tarball . '|||tgz' . "\n"; + + # build dsreg command command + my $command = "$dsreg"; + $command .= " --add $batch"; + $command .= " --copy"; + $command .= " --datapath $path"; + $command .= " --type IPP_PSPS"; + $command .= " --product $datastoreProduct"; + $command .= " --list $tempName"; + + # run command + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 }; + + print "* Successfully published $tarball to datastore\n"; + + close($tempFile); + + return 1; +} + +####################################################################################### +# +# runs ippToPsps to produce init batch +# +####################################################################################### +sub produceInit { + my ($outputPath, $resultsFilePath) = @_; + + my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType, $resultsFilePath); + return $success; +} + +####################################################################################### +# +# runs ippToPsps for detections for this exposure +# +####################################################################################### +sub produceDetections { + my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; + + my $nebPath = $gpc1Db->getCameraStageSmf($expId); + if (!$nebPath) { return 0; } + + my $success = 0; + + # get real filename from neb 'key' + my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0; + my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; + + # now write the path to this file out to temp file + my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$save_temps); + print $tempFile $realFile . "\n"; + close $tempFile; + + $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); + return $success; +} + +####################################################################################### +# +# runs ippToPsps for diffs for this exposure +# +####################################################################################### +sub produceDiffs { + my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; + + my $rows = $gpc1Db->getDiffStageCmfs($expId); + + my $size = @{$rows}; + if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;} + + my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps); + + # loop round db results should be one file per skycell + my $success = 0; + + my $row; + foreach $row ( @{$rows} ) { + my ($nebPath) = @{$row}; + + print "NEB PATH $nebPath\n"; + + # get real filename from neb 'key' + my $fpaObjects = $ipprc->filename("PPSUB.OUTPUT.SOURCES",$nebPath) or return 0; + my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; + #print "REALFILE $realFile\n"; + + # now write the path to this file out to temp file + print $tempFile $realFile . "\n"; + } + + my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); + + close $tempFile; + + return $ret; +} + +####################################################################################### +# +# runs the ippToPsps program +# +####################################################################################### +sub runProgram { + my ($input, $output, $expid, $expName, $surveyType, $batchType, $resultsFilePath) = @_; + + # build command + my $command = "$ippToPsps"; + $command .= " -input $input"; + $command .= " -output $output"; + $command .= " -D CATDIR $dvodb"; + $command .= " -config ../config"; # TODO + $command .= " -expid $expid"; + $command .= " -expname $expName"; + $command .= " -survey $surveyType"; + $command .= " -batch $batchType"; + $command .= " -results $resultsFilePath"; + + # run command + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $verbose); + + return $success; +} Index: trunk/ippToPsps/scripts/ippToPsps_run.pl =================================================================== --- trunk/ippToPsps/scripts/ippToPsps_run.pl (revision 28696) +++ (revision ) @@ -1,736 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; -use PS::IPP::Config 1.01 qw( :standard ); -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); -use IPC::Cmd 0.36 qw( can_run run ); -use DBI; -use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; -use File::Temp qw(tempfile); -use XML::LibXML; - -# globals -my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $force, $initBatch, $help); - -# default values for certain globals -$verbose = 0; -$save_temps = 0; -$no_op = 0; -$no_update = 0; -$camera = 'GPC1'; -$output = undef; -$singleExpId = undef; -$datastoreProduct = undef; -$force = 0; -$initBatch = 0; -$help = 0; - -# get user args -GetOptions( - 'output|o=s' => \$output, - 'batch|b=s' => \$batchType, - 'dvodb|d=s' => \$dvodb, - 'survey|s=s' => \$survey, - 'expid|e=s' => \$singleExpId, - 'product|p=s' => \$datastoreProduct, - 'verbose|v' => \$verbose, - 'save_temps' => \$save_temps, - 'no-op' => \$no_op, - 'no-update' => \$no_update, - 'force|f' => \$force, - 'help|h' => \$help, - ); - -if ($help) {printUsage(); exit;} -if (@ARGV) {print "* Unknown option: @ARGV\n"; printUsage(); exit;} - -my $quit = 0; -print "\n"; -if (!defined $batchType) {print "* REQUIRED: need to define a batch type: -b \n"; $quit=1;} -if (!defined $output) {print "* REQUIRED: need to provide an output path: -o \n"; $quit=1;} -if (!defined $dvodb) {print "* REQUIRED: need to provide a DVO Db path: -d \n"; $quit=1;} -if (!defined $survey and !defined $singleExpId) {print "* REQUIRED: need to provide a survey type (-s ) or an exposure ID (-e )\n"; $quit=1;} -if (!defined $datastoreProduct) {print "* OPTIONAL: datastore producth: -p \n";} -print "\n\n"; - -if ($quit) { exit; } - - -# check we can run programs and get camera config -my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR)); -my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR)); -my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); - -# make a temporary file for saving results - -if ($batchType eq "init") {$initBatch = 1;} -process(); - - -####################################################################################### -# -# Generates a PSPS-suitable survey 'type' from the IPP distribution group -# -####################################################################################### -sub printUsage { - - print "\n" . - " *** Required options:\n". - "\n" . - " --batch \n". - " --output \n". - " --survey \n" . - " --expid \n" . - " --dvodb \n". - "\n" . - " *** Optional:\n". - "\n" . - " --product if not provided, data will NOT be published\n". - " --save_temps will save temporary files\n". - " --no-op ?\n". - " --no-update will not update database\n" . - " --force will force an already processed exposure\n" . - " --help show this message\n\n"; - -} - -####################################################################################### -# -# Generates a PSPS-suitable survey 'type' from the IPP distribution group -# -####################################################################################### -sub getSurveyTypeFromDistGroup { - my ($distGroup) = @_; - - if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";} - if ($distGroup =~ m/^M31$/i) {return "M31";} - if ($distGroup =~ m/^sts$/i) {return "STS1";} - if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";} - if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";} - - print "* Unknown distribution group: '$distGroup'\n"; - return undef; -} - - -####################################################################################### -# -# Connects to a db -# -######################################################################################## -sub connectToDb { - my ($dbname,$dbserver,$dbuser,$dbpass) = @_; - - - my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . - "mysql_socket=" . DB_SOCKET(), - ${dbuser},${dbpass}, - { RaiseError => 1, AutoCommit => 1} - ); - - printf("* Connection to '$dbname' on '$dbserver': %s\n", $db ? "success" : "FAILED ($DBI::errstr)"); - - return $db; -} - -####################################################################################### -# -# Finds all exposures for the provided TODO and generates PSPS FITS data for them -# -####################################################################################### -sub process { - - print "*******************************************************************************\n*\n"; - - my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); - my $gpc1Db = connectToDb("gpc1", "ippdb01", "ippuser", "ippuser") or return 0; - my $ippToPspsDb = connectToDb("ippToPsps", "ippdb01", "ipp", "ipp") or return 0; - my $lastExpId = getLastExpId($ippToPspsDb); - - my $query; - if ($initBatch) { - - $query = $gpc1Db->prepare(<prepare(<prepare(< 133887 - #AND filter = 'r.00000' - #AND rawExp.dateobs <= '2010-03-12' - #AND rawExp.exp_id > $lastExpId - #AND camRun.label LIKE '%$label%' - #AND rawExp.exp_id > 136561 - #AND camRun.dist_group LIKE 'sas' - #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range - } - -# TODO check if there are no exposures and give a warning - - my $batchId = 0; - my $batchesPerJob = 1; - my $published = 0; - - $query->execute; - - #my $batchId = -1; - - # loop round exposures - while (my @row = $query->fetchrow_array()) { - my ($expId, $expName, $distGroup) = @row; - - if (!$initBatch && isExposureAlreadyProcessed($ippToPspsDb, $expId)) { - - if ($force) {print "* Forcing....\n";} - else {next}; - } - - my $surveyType = getSurveyTypeFromDistGroup($distGroup); - if (!$surveyType) {next;} - - $batchId = getNewBatchId($ippToPspsDb, $expId, $surveyType); - - # TODO quit here if no sensible batch ID - - # generate batch path from batch IDs - my $batch = sprintf("B%08d", $batchId); - my $batchDir = sprintf("$output/$batch"); - mkdir($batchDir, 0777); - $published = 0; - - print "* Preparing exposure $expId as batch '$batch'...\n"; - - # run IppToPsps program - if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath)) { - - my ($filename, $minObjId, $maxObjId, $totalDetections); - parseResults($resultsFilePath, \$filename, \$minObjId, \$maxObjId, \$totalDetections); - - # write batch manifest and tar and zip up - if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) { - - my $tarball = tarAndZipBatch($output, $batch); - if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;} - } - - updateDb($ippToPspsDb, $batchId, $expId, 1, $published, $totalDetections); - } - - if ($initBatch) {print "* Wrote initialisation batch\n";} - - print "*\n*******************************************************************************\n\n"; - } - - $gpc1Db->disconnect(); - $ippToPspsDb->disconnect(); - close $resultsFile; - - return 1; -} - -####################################################################################### -# -# Gets the last processed exposure ID (ini file for now - will be Db eventually TODO) -# -######################################################################################## -sub getLastExpId { - my ($db) = @_; - - my $query = $db->prepare(<execute; - - my $expId = $query->fetchrow_array(); - - print "* Last successfully processed exposure ID was $expId\n"; - - return $expId; -} - -####################################################################################### -# -# Updates an existing database record -# -######################################################################################## -sub updateDb { - my ($db, $batchId, $expId, $processed, $published, $totalDetections) = @_; - - my $query = $db->prepare(<execute; # TODO check response of this -} - -####################################################################################### -# -# Checks whether we have successfully processed this exposure -# -######################################################################################## -sub isExposureAlreadyProcessed { - my ($db, $expId) = @_; - - my $query = $db->prepare(<execute; - - my $processed = $query->fetchrow_array(); - - if ($processed) {print "* Exposure ID '$expId' has already been processed\n";} - - return $processed; -} - -####################################################################################### -# -# Generates a new record in the database with a new batch ID -# -######################################################################################## -sub getNewBatchId { - my ($db, $expId,$surveyType) = @_; - - my $query = $db->prepare(<execute; - - my $batchId = $query->fetchrow_array(); - - $batchId++; - - $query = $db->prepare(<execute; - - print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n"; - - return $batchId; -} - -####################################################################################### -# -# run IppToPsps -# -####################################################################################### -sub runIppToPsps { - my ($gpc1Db, $expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath) = @_; - - my $success = 0; - - if ($batchType =~ /init/) { - $success = produceInit($batchDir, $resultsFilePath); - } - elsif ($batchType =~ /det/) { - $success = produceDetections($gpc1Db, $expId, $expName, $surveyType, $batchDir, $resultsFilePath); - } - elsif ($batchType =~ /diff/) { - $success = produceDiffs($gpc1Db, $expId, $expName, $surveyType, $batchDir, $resultsFilePath); - } - elsif ($batchType =~ /test/) { - $success = produceDetections($gpc1Db, $expId, $expName, $surveyType, $batchDir, $resultsFilePath); - } - - return $success; -} - -####################################################################################### -# -# tar 'n zip a batch TODO handle errors -# -####################################################################################### -sub tarAndZipBatch { - my ($path,$batch) = @_; - - my $batchDir = "$path/$batch"; - my $tar = "$batch.tar"; - my $tarball = "$tar.gz"; - my $batchTar = "$path/$tar"; - my $batchZip = "$path/$tarball"; - - # tar - my $command = "tar -cvf $batchTar -C $path $batch"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to tar up contents of dir: $batchDir\n" and return undef; } - - # zip - $command = "gzip -c $batchTar > $batchZip"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to gzip: $batchTar\n" and return undef; } - - # remove tar file - $command = "rm $batchTar"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to remove: $batchTar\n" }; - - # remove batch dir - $command = "rm -r $batchDir"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to remove: $batch\n"}; - - return $tarball; -} - -####################################################################################### -# -# zip a file then delete it -# -####################################################################################### -sub zipFile { - my ($path,$origFile) = @_; - - my $file; - - # zip - my $zippedName = "$origFile.gz"; - my $command = "gzip -c $path/$origFile > $path/$zippedName"; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $command, verbose => $verbose); - - if (!$success) { - - $file = $origFile; - print "* Unable to gzip: $path/$origFile\n"; - } - else { - - $file = $zippedName; - - # now delete original - $command = "rm -r $path/$origFile"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if (!$success) { print "* Unable to remove: $path/$origFile\n"}; - } - - return $file; -} - - -####################################################################################### -# -# parse results XML -# -####################################################################################### -sub parseResults { - my ($resultsFilePath, $filename, $minObjId, $maxObjId, $totalDetections) = @_; - - my $parser = XML::LibXML->new; - my $doc = $parser->parse_file($resultsFilePath); - ${$filename} = $doc->findvalue('//filename'); - ${$minObjId} = $doc->findvalue('//minObjID'); - ${$maxObjId} = $doc->findvalue('//maxObjID'); - ${$totalDetections} = $doc->findvalue('//totalDetections'); -} - - -####################################################################################### -# -# writes a batch manifest file for the one FITS file that should be in the path passed in -# -####################################################################################### -sub writeBatchManifest { - my ($path,$batch,$batchType,$surveyType,$filename,$minObjId,$maxObjId) = @_; - - use XML::Writer; - use Digest::MD5::File qw( file_md5_hex ); - - # sort out paths - my $outputPath = "$path/BatchManifest.xml"; - my $output = new IO::File(">$outputPath"); - - # create a timestamp - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); - my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; - - # determine batch 'type' - my $type; - if ($batchType eq 'init') {$type = "IN";} - elsif ($batchType eq 'det') {$type = "P2";} - elsif ($batchType eq 'stack') {$type = "ST";} - elsif ($batchType eq 'diff') {$type = "OB";} - else {$type = "UNKNOWN";} - - # create XML file - my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); - $writer->xmlDecl('UTF-8'); - $writer->doctype('manifest', "", "psps-manifest.dtd"); - - # open directory to hunt for FITS files - opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0; - my @thefiles= readdir(DIR); - closedir(DIR); - - my $foundFits = 0; - foreach my $f (@thefiles) { - - if ($f =~ /\.FITS/) { - - if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;} - - # my $file = zipFile($path, $f); - - # get md5sum of file - my $md5sum = file_md5_hex("$path/$f"); - - # get size of file in bytes - my $filesize = -s "$path/$f"; - - # write manifest element TODO untidy - if($batchType eq 'det' && $filename eq $f) { - - $writer->startTag('manifest', - "name" => "$batch", - "type" => $type, - "survey" => $surveyType, - "timestamp" => "$timeStamp", - "minObjId" => $minObjId, - "maxObjId" => $maxObjId); - } - else { - - $writer->startTag('manifest', - "name" => "$batch", - "type" => $type, - "timestamp" => "$timeStamp"); - } - # write the tag for this batch - $writer->startTag("file", - "name" => $f, - "bytes" => $filesize, - "md5" => $md5sum); - - $writer->endTag(); - $writer->endTag(); - $writer->end(); - $foundFits = 1; - } - - } - - if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;} - return 1; -} - -####################################################################################### -# -# register new job with the datastore -# -######################################################################################## -sub publishToDatastore { - my ($batch, $path, $tarball) = @_; - - my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps); - - print $tempFile $tarball . '|||tgz' . "\n"; - - # build dsreg command command - my $command = "$dsreg"; - $command .= " --add $batch"; - $command .= " --copy"; - $command .= " --datapath $path"; - $command .= " --type IPP_PSPS"; - $command .= " --product $datastoreProduct"; - $command .= " --list $tempName"; - - # run command - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 }; - - print "* Successfully published $tarball to datastore\n"; - - close($tempFile); - - return 1; -} - -####################################################################################### -# -# runs ippToPsps to produce init batch -# -####################################################################################### -sub produceInit { - my ($outputPath, $resultsFilePath) = @_; - - my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType, $resultsFilePath); - return $success; -} - -####################################################################################### -# -# runs ippToPsps for detections for this exposure -# -####################################################################################### -sub produceDetections { - my ($gpc1Db, $expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; - - # DESC and LIMIT 1 is to make sure we get most recently processed - my $query = $gpc1Db->prepare(<execute; - my $nebPath = $query->fetchrow_array(); - if (!$nebPath) { print "* No smf files found for this exposure\n"; return 0; } - - my $success = 0; - - # get real filename from neb 'key' - my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0; - my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; - - # now write the path to this file out to temp file - my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$save_temps); - print $tempFile $realFile . "\n"; - close $tempFile; - - $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); - return $success; -} - -####################################################################################### -# -# runs ippToPsps for diffs for this exposure -# -####################################################################################### -sub produceDiffs { - my ($gpc1Db, $expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; - - my $query = - "SELECT DISTINCT diffSkyfile.path_base ". - "FROM warpRun ". - "JOIN fakeRun USING(fake_id) ". - "JOIN camRun USING(cam_id) ". - "JOIN chipRun USING(chip_id) ". - "JOIN rawExp USING (exp_id) ". - "JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) ". - "JOIN diffSkyfile USING (diff_id) ". - "WHERE rawExp.exp_id = $expId AND camRun.magicked"; - - if ($verbose) { print"* $query\n";} - - # execute query and check results - my $results = $gpc1Db->selectall_arrayref( $query ); - my $size = @$results; - if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;} - - my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps); - - # loop round db results should be one file per skycell - my $success = 0; - for my $row (@$results) { - - my ($nebPath) = @$row; - #print "NEB PATH $nebPath\n"; - - # get real filename from neb 'key' - my $fpaObjects = $ipprc->filename("PPSUB.OUTPUT.SOURCES",$nebPath) or return 0; - my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; - #print "REALFILE $realFile\n"; - - # now write the path to this file out to temp file - print $tempFile $realFile . "\n"; - } - - my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); - - close $tempFile; - - return $ret; -} - -####################################################################################### -# -# runs the ippToPsps program -# -####################################################################################### -sub runProgram { - my ($input, $output, $expid, $expName, $surveyType, $batchType, $resultsFilePath) = @_; - - # build command - my $command = "$ippToPsps"; - $command .= " -input $input"; - $command .= " -output $output"; - $command .= " -D CATDIR $dvodb"; - $command .= " -config config"; # TODO - $command .= " -expid $expid"; - $command .= " -expname $expName"; - $command .= " -survey $surveyType"; - $command .= " -batch $batchType"; - $command .= " -results $resultsFilePath"; - - # run command - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) - = run(command => $command, verbose => $verbose); - - return $success; -} Index: trunk/ippToPsps/src/ippToPspsBatchDetection.c =================================================================== --- trunk/ippToPsps/src/ippToPspsBatchDetection.c (revision 28696) +++ trunk/ippToPsps/src/ippToPspsBatchDetection.c (revision 28698) @@ -14,7 +14,9 @@ // Gets uncalibrated instrumental flux from magnitude -static __inline bool ippToPsps_getFlux(const float zeroPoint, const float exposureTime, const float magnitude, float* flux) { +static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) { *flux = powf(10.0, -0.4*magnitude) / exposureTime; + if (fluxErr) *fluxErr = (magnitudeErr * *flux)/1.085736; +//if (fluxErr) printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr); return (!isfinite(*flux) || *flux < 0.000001) ? false : true; } @@ -46,5 +48,5 @@ fits_get_hdrspace(fitsIn, &nKeys, NULL, &status); - float zptObs, zeroPoint, exposureTime; + float zptObs, exposureTime; char filterType[20]; double obsTime; @@ -52,5 +54,4 @@ double expTime; status=0; fits_read_key(fitsIn, TFLOAT, "ZPT_OBS", &zptObs, NULL, &status); - status=0; fits_read_key(fitsIn, TFLOAT, "ZPT_REF", &zeroPoint, NULL, &status); status=0; fits_read_key(fitsIn, TFLOAT, "EXPREQ", &exposureTime, NULL, &status); status=0; fits_read_key(fitsIn, TSTRING, "FILTERID", filterType, NULL, &status); @@ -138,10 +139,10 @@ long removeList[MAXDETECT]; long thisObjId; - bool firstTimeIn = true; + bool firstTimeIn = true, peakFluxOk, instFluxOk; int ippIDetNum, instMagNum, instMagErrNum, peakMagNum; // loop round all 60 chips - for (int x=0; x<8; x++) { - for (int y=0; y<8; y++) { + for (int x=3; x<4; x++) { + for (int y=3; y<4; y++) { // dodge the corners @@ -293,9 +294,12 @@ obsTimes[s] = obsTime; - ippToPsps_getFlux(zeroPoint, exposureTime, instMagErr[s], &instFluxErr[s]); +// ippToPsps_getFlux(exposureTime, instMagErr[s], &instFluxErr[s]); + + // calculate flux and flux errors from magnitudes + peakFluxOk = ippToPsps_getFlux(exposureTime, peakMag[s], &peakFlux[s], 0.0, NULL); + instFluxOk = ippToPsps_getFlux(exposureTime, instMag[s], &instFlux[s], instMagErr[s], &instFluxErr[s]); // check for invalid flux values (if not already labelled as a duplicate) - if ((!ippToPsps_getFlux(zeroPoint, exposureTime, peakMag[s], &peakFlux[s]) || - !ippToPsps_getFlux(zeroPoint, exposureTime, instMag[s], &instFlux[s])) && !isDuplicate) { + if ( (!peakFluxOk || !instFluxOk) && !isDuplicate) { removeList[numOfDuplicates+numInvalidFlux] = s+1; numInvalidFlux++;