Index: branches/pap/Nebulous-Server/bin/neb-host =================================================================== --- branches/pap/Nebulous-Server/bin/neb-host (revision 28506) +++ branches/pap/Nebulous-Server/bin/neb-host (revision 28534) @@ -159,4 +159,5 @@ neb-host [ ] + [--note ''] [--host ] [--state ] [--dbhost ] [--db ] @@ -174,4 +175,8 @@ =over 4 +=item * --note + +Set a status note for the specified volume. + =item * --host Index: branches/pap/Nebulous-Server/bin/nebdiskd =================================================================== --- branches/pap/Nebulous-Server/bin/nebdiskd (revision 28506) +++ branches/pap/Nebulous-Server/bin/nebdiskd (revision 28534) @@ -53,5 +53,9 @@ # set the process name for easy pgrep-ability $0 = 'nebdiskd'; -my $rcfile = "$ENV{HOME}/.nebdiskrc"; +#my $rcfile = "$ENV{HOME}/.nebdiskrc"; +my $rcfile = $ENV{NEBDISKDRC}; +unless (defined($rcfile)) { + $rcfile = '/etc/nebdiskd.rc'; +} $rcfile = File::Spec->canonpath($rcfile); @@ -71,6 +75,4 @@ my %host_removed = (); my $failure_limit = 5; - - #my $mounts = $c->get_mounts; @@ -143,4 +145,6 @@ $SIG{HUP} = sub { $c = read_rcfile($rcfile) }; +my $date = localtime(); +$log->warn("BEGIN: $date with RCFILE: $rcfile"); while (1) { Index: branches/pap/Nebulous-Server/lib/Nebulous/Server.pm =================================================================== --- branches/pap/Nebulous-Server/lib/Nebulous/Server.pm (revision 28506) +++ branches/pap/Nebulous-Server/lib/Nebulous/Server.pm (revision 28534) @@ -9,5 +9,5 @@ no warnings qw( uninitialized ); -our $VERSION = '0.17'; +our $VERSION = '0.18'; use base qw( Class::Accessor::Fast ); @@ -1508,4 +1508,104 @@ return \@keys; +} + +sub find_objects_wildcard +{ + my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); + + my ($pattern) = validate_pos( @_, + { + type => SCALAR, + optional => 1, + }, + ); + + my $sql = $self->sql; + my $db = $self->_db_for_index(0); + + # validate that we have a key to deal with + eval { + $pattern = parse_neb_key($pattern) if defined $pattern; + }; + $log->logdie("$@") if $@; + + unless (defined $pattern) { + $log->debug( "leaving" ); + $log->logdie("no keys found"); + } + + # parse out the directory we're working in, and decide if we are a directory + my $dir_id = $self->_resolve_dir_parent_id(key => $pattern, dir_id => 1); + my $work_dir; + my $file_pattern; + if (defined $dir_id) { + $work_dir = $pattern; + $file_pattern = '%'; + } + else { + my $dir_plain = dirname($pattern); + if ($dir_plain eq 'neb:') { + $dir_plain = 'neb:///'; + } + if ($dir_plain) { + $work_dir = parse_neb_key($dir_plain); + } + else { + $work_dir = parse_neb_key('/'); + } + $file_pattern = basename $pattern; + unless ($file_pattern) { + $file_pattern = '%'; + } + + $dir_id = $self->_resolve_dir_parent_id(key => $work_dir, dir_id => 1); + unless (defined $dir_id) { + $log->logdie("pattern $work_dir does not match any key or directory"); + } + } + + # find dirs under dir + my @dir_keys; + + eval { + $log->debug("looking for directories under dir: $dir_id"); + my $query = $db->prepare_cached( $sql->find_dir_by_parent_id . " AND dirname LIKE ? "); + $query->execute( $dir_id, $file_pattern ); + + while ( my $row = $query->fetchrow_hashref ) { + next if $row->{'dir_id'} == 1; + my $dir = $row->{'dirname'}; + if ($dir_id == 1) { + push @dir_keys, $dir . '/' if $dir; + } else { + push @dir_keys, $work_dir->path . '/' . $dir . '/' if $dir; + } + $log->debug( "matched $dir" ) if $dir; + } + }; + $log->logdie("database error: $@") if $@; + + # find files under dir + my @keys; + eval { + $log->debug("looking for objects under dir: $dir_id"); + my $query = $db->prepare_cached( $sql->find_object_by_dir_id . " AND ext_id_basename LIKE ? "); + $query->execute( $dir_id , $file_pattern); + + while ( my $row = $query->fetchrow_hashref ) { + my $key = $row->{ 'ext_id' }; + push @keys, $key if $key; + $log->debug( "matched $key" ) if $key; + } + }; + $log->logdie("database error: $@") if $@; + + $log->debug( "leaving" ); + + return [sort(@dir_keys), sort(@keys)]; + } Index: branches/pap/Nebulous/bin/neb-ls =================================================================== --- branches/pap/Nebulous/bin/neb-ls (revision 28506) +++ branches/pap/Nebulous/bin/neb-ls (revision 28534) @@ -1,5 +1,5 @@ #!/usr/bin/env perl -# Copyright (C) 2007-2009 Joshua Hoblitt +# Copyright (C) 2007-2009 Joshua Hoblitt, 2010 Chris Waters use strict; @@ -16,18 +16,20 @@ my ( $server, - $long, -# $recursive, + $path, + $column, ); $server = $ENV{'NEB_SERVER'} unless $server; -# make --long the default -$long = 1; GetOptions( 'server|s=s' => \$server, -# 'recursive|r' => \$recursive, - 'l|1' => \$long, + 'path|p' => \$path, + 'column|c' => \$column, ) || pod2usage( 2 ); + +# twiddle column so that it does it by default. + +$column = not $column; my $pattern = shift; @@ -47,19 +49,27 @@ $pattern ||= "/"; -#if ($recursive) { -# $pattern = "^" . $pattern . ".*"; -#} else { -# $pattern = "^" . $pattern . "\$"; -#} +my $keys = $neb->find_objects_wildcard($pattern); -my $keys = $neb->find_objects($pattern); - +if ($path) { + my @files; + foreach my $key (@{ $keys }) { + my $uris = $neb->find_instances($key); + if (defined $uris) { + push @files, URI->new(@$uris[0])->file; + } + } + @{ $keys } = @files; +} if ($keys) { - if ($long) { - print join("\n", @{ $keys }), "\n"; - } else { - print join(" ", @{ $keys }), "\n"; + if ($column) { + open(COLUMN,"|column") || die "Cannot open column command."; + print COLUMN join("\n", @{ $keys }), "\n"; + close(COLUMN); + } + else { + print join("\n", @{ $keys }), "\n"; } } + __END__ @@ -73,11 +83,11 @@ =head1 SYNOPSIS - neb-ls [--server ] [-l|-1] [--recursive] + neb-ls [--server ] [-c] [-p] =head1 DESCRIPTION This program list Nebulous keys matched by C<>. Call it with no -arguments is equivalanet to searching with the pattern C<.*>. Where -C<> is a POSIX 1003.2 compatable regular repression. +arguments is equivalent to searching with the pattern C. SQL like +wildcards are supported to select out certain files. =head1 OPTIONS @@ -85,19 +95,14 @@ =over 4 -=item * -l|-1 +=item * --path|-p -Use a long listing format. +Find the disk files corresponding to the keys found. + +=item * --column|-c + +Disable column formatting, and output results one to a line. Optional -=cut -#=item * --recursive|-r -# -#By default C will only try to match the exact string provided to it. -#With this option set all keys which match C<> as a REGEX or substring -#will be returned. -# -#Optional -# =item * --server|-s @@ -122,8 +127,4 @@ =back -=head1 CREDITS - -Just me, myself, and I. - =head1 SUPPORT @@ -132,9 +133,9 @@ =head1 AUTHOR -Joshua Hoblitt +Joshua Hoblitt , Chris Waters =head1 COPYRIGHT -Copyright (C) 2007-2009 Joshua Hoblitt. All rights reserved. +Copyright (C) 2007-2010 Joshua Hoblitt / Chris Waters. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/pap/Nebulous/lib/Nebulous/Client.pm =================================================================== --- branches/pap/Nebulous/lib/Nebulous/Client.pm (revision 28506) +++ branches/pap/Nebulous/lib/Nebulous/Client.pm (revision 28534) @@ -9,5 +9,5 @@ no warnings qw( uninitialized ); -our $VERSION = '0.17'; +our $VERSION = '0.18'; use Digest::MD5; @@ -741,4 +741,46 @@ } +sub find_objects_wildcard +{ + my $self = shift; + + my @args = validate_pos( @_, + { + type => SCALAR, + optional => 1, + }, + ); + + $log->debug( "entered - @_" ); + + my $response = $self->{ 'server' }->find_objects_wildcard( @args ); + if ( $response->fault ) { + $self->set_err($response->faultstring); + + if ($response->faultstring =~ /no keys found/) { + $log->debug( "leaving" ); + return; + } + if ($response->faultstring =~ /does not match any key or directory/) { + $log->debug( "leaving" ); + return; + } + + $log->logdie("unhandled fault - ", $self->err); + } + + my $keys = $response->result; + + $log->debug( "server found: @$keys" ); + +# foreach my $path ( @{ $uris } ) { +# $path = _get_file_path( $path ); +# } + + $log->debug( "leaving" ); + + return $keys; +} + sub find_instances Index: branches/pap/ippScripts/scripts/automate_stacks.pl =================================================================== --- branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28506) +++ branches/pap/ippScripts/scripts/automate_stacks.pl (revision 28534) @@ -766,4 +766,5 @@ $cmd .= " -workdir $workdir "; $cmd .= " -label $label "; + $cmd .= " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 "; if ($maxN > 0) { $cmd .= " -random_subset -random_limit $maxN "; Index: branches/pap/ippScripts/scripts/dqstats_bundle.pl =================================================================== --- branches/pap/ippScripts/scripts/dqstats_bundle.pl (revision 28506) +++ branches/pap/ippScripts/scripts/dqstats_bundle.pl (revision 28534) @@ -73,5 +73,5 @@ # Output products unless (defined($uri)) { - $uri = "/data/${host}.0/tmp/dqstats.${dqstats_id}.fits"; + $uri = "/tmp/dqstats.${dqstats_id}.fits"; } #$ipprc->outroot_prepare($uri); # hm....need to think more here. @@ -90,4 +90,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + unlink($uri); &my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code); } @@ -106,4 +107,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + unlink($uri); &my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code); } Index: branches/pap/ippTasks/dqstats.pro =================================================================== --- branches/pap/ippTasks/dqstats.pro (revision 28506) +++ branches/pap/ippTasks/dqstats.pro (revision 28534) @@ -37,4 +37,16 @@ task dqstats.revert active false + end +end + +macro dqstats.revert.on + task dqstats.revert + active true + end +end + +macro dqstats.revert.off + task dqstats.revert + active true end end @@ -109,5 +121,4 @@ ## we want only a single outstanding dqstats job. - host local npending 1 @@ -187,4 +198,5 @@ periods -timeout 120.0 npending 1 + active false stdout NULL Index: branches/pap/ippTasks/ipphosts.mhpcc.config =================================================================== --- branches/pap/ippTasks/ipphosts.mhpcc.config (revision 28506) +++ branches/pap/ippTasks/ipphosts.mhpcc.config (revision 28534) @@ -253,36 +253,34 @@ END -# this list is no longer used -# XXX : delete if this does not cause trouble -## ipphosts METADATA -## camera STR distribution -## count S32 26 -## 0 STR ipp021 -## 1 STR ipp023 -## 2 STR ipp024 -## 3 STR ipp026 -## 4 STR ipp028 -## 5 STR ipp029 -## 6 STR ipp030 -## 7 STR ipp031 -## 8 STR ipp032 -## 9 STR ipp033 -## 10 STR ipp034 -## 11 STR ipp035 -## 12 STR ipp036 -## 13 STR ipp038 -## 14 STR ipp039 -## 15 STR ipp040 -## 16 STR ipp041 -## 17 STR ipp043 -## 18 STR ipp044 -## 19 STR ipp045 -## 20 STR ipp046 -## 21 STR ipp047 -## 22 STR ipp048 -## 23 STR ipp050 -## 24 STR ipp051 -## 25 STR ipp052 -## END +ipphosts METADATA + camera STR distribution + count S32 26 + 0 STR ipp021 + 1 STR ipp023 + 2 STR ipp024 + 3 STR ipp026 + 4 STR ipp028 + 5 STR ipp029 + 6 STR ipp030 + 7 STR ipp031 + 8 STR ipp032 + 9 STR ipp033 + 10 STR ipp034 + 11 STR ipp035 + 12 STR ipp036 + 13 STR ipp038 + 14 STR ipp039 + 15 STR ipp040 + 16 STR ipp041 + 17 STR ipp043 + 18 STR ipp044 + 19 STR ipp045 + 20 STR ipp046 + 21 STR ipp047 + 22 STR ipp048 + 23 STR ipp050 + 24 STR ipp051 + 25 STR ipp052 +END ipphosts METADATA Index: branches/pap/ippToPsps/scripts/createDb.pl =================================================================== --- branches/pap/ippToPsps/scripts/createDb.pl (revision 28506) +++ branches/pap/ippToPsps/scripts/createDb.pl (revision 28534) @@ -25,12 +25,23 @@ print "* Connected to '$dbname' on 'dbserver'\n"; -if (!doesTableExist("revision")) { - createRevision_1(); +my $currentRevision = -1; + +my $latestRevision = 3; + print "* Latest revision = $latestRevision\n"; + +while ($currentRevision != $latestRevision) { + + $currentRevision = getRevision(); + print "* Current revision = $currentRevision\n"; + + if ($currentRevision == 0) {createRevision_1();} + elsif ($currentRevision == 1) {createRevision_2();} + elsif ($currentRevision == 2) {createRevision_3();} + } - - $db->disconnect(); print "* Disconnected from '$dbname' on 'dbserver'\n"; print "*\n*******************************************************************************\n\n"; + ####################################################################################### @@ -50,9 +61,4 @@ primary key (revision) ); -SQL - $query->execute; - - $query = $db->prepare(<execute; @@ -77,4 +83,59 @@ $query->execute; +setRevision(1); + +} + +####################################################################################### +# +# Create revision 2 of the database +# +####################################################################################### +sub createRevision_2 { + + print "* Creating revision 2 of '$dbname'\n"; + + my $query = $db->prepare(<execute; + +setRevision(2); +} + +####################################################################################### +# +# Create revision 3 of the database +# +####################################################################################### +sub createRevision_3 { + + print "* Creating revision 3 of '$dbname'\n"; + + my $query = $db->prepare(<execute; + +setRevision(3); +} + + +####################################################################################### +# +# Sets current revision of ippToPsps database +# +####################################################################################### +sub setRevision { + my ($revision) = @_; + + my $query = $db->prepare(<execute; } @@ -82,4 +143,25 @@ # # Gets current revision of ippToPsps database +# +####################################################################################### +sub getRevision { + + if (!doesTableExist("revision")) {return 0;} + + my $query = $db->prepare(<execute; + my @row = $query->fetchrow_array(); + + return $row[0]; +} + +####################################################################################### +# +# Checks whether a certain table exists # ####################################################################################### @@ -99,6 +181,4 @@ my $count = $query->fetchrow_array(); - printf( "* Table '$table' %s\n", $count ? "exists" : "does not exist"); - return $count; } Index: branches/pap/ippToPsps/scripts/ippToPsps_run.pl =================================================================== --- branches/pap/ippToPsps/scripts/ippToPsps_run.pl (revision 28506) +++ branches/pap/ippToPsps/scripts/ippToPsps_run.pl (revision 28534) @@ -10,4 +10,5 @@ use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; use File::Temp qw(tempfile); +use XML::LibXML; # globals @@ -73,5 +74,5 @@ # make a temporary file for saving results -my ($resultsFile, $resultsFileName) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); +my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); process(); @@ -118,5 +119,4 @@ } - ####################################################################################### # @@ -160,4 +160,5 @@ AND chipRun.exp_id = rawExp.exp_id AND camRun.dist_group = '$survey' + AND rawExp.dateobs >= '2010-02-20' GROUP BY rawExp.exp_id ORDER BY rawExp.exp_id ASC; @@ -457,22 +458,9 @@ # read results - open (MYFILE, $resultsFileName); - my $i = 0; - - # detection results - my $minObjId; - my $maxObjId; - my $filename; - if($batchType eq 'det') { - - while () { - chomp; - if ($i == 0) { $filename = $_; } - if ($i == 1) { $minObjId = $_; } - if ($i == 2) { $maxObjId = $_; } - $i++; - } - } - close (MYFILE); + my $parser = XML::LibXML->new; + my $doc = $parser->parse_file($resultsFilePath); + my $filename = $doc->findvalue('//filename'); + my $minObjId = $doc->findvalue('//minObjID'); + my $maxObjId = $doc->findvalue('//maxObjID'); # create XML file @@ -794,5 +782,5 @@ $command .= " -survey $surveyType"; $command .= " -batch $batchType"; - $command .= " -results $resultsFileName"; + $command .= " -results $resultsFilePath"; # run command Index: branches/pap/ippToPsps/src/ippToPsps.c =================================================================== --- branches/pap/ippToPsps/src/ippToPsps.c (revision 28506) +++ branches/pap/ippToPsps/src/ippToPsps.c (revision 28534) @@ -53,4 +53,14 @@ } + // save XML document for results + if (this->resultsXmlDoc) { + + xmlSaveFormatFileEnc(this->resultsPath, this->resultsXmlDoc, "UTF-8", 1); + + xmlFreeDoc(this->resultsXmlDoc); + xmlCleanupParser(); + xmlMemoryDump(); + } + psFree(this->fitsInPath); psFree(this->resultsPath); @@ -59,5 +69,4 @@ psFree(this->pmconfig); - if (this->resultsFile) fclose(this->resultsFile); for(uint32_t i=0; inumOfInputFiles; i++) @@ -246,5 +255,5 @@ this->fitsInPath = NULL; this->resultsPath = NULL; - this->resultsFile = NULL; + this->resultsXmlDoc = NULL; this->numOfInputFiles = 0; this->inputFiles = NULL; @@ -304,4 +313,5 @@ } + // create a config object this->config = ippToPspsConfig_Constructor(this->configsDir); if (this->config == NULL) { @@ -311,18 +321,11 @@ } - // ready results file, if necessary + // create XML document for results if (this->resultsPath) { - this->resultsFile = fopen (this->resultsPath, "w+"); - - if (this->resultsFile == NULL) { - - psError(PS_ERR_UNKNOWN, false, "Unable to open results file at %s", this->resultsPath); - // not essential to write results, so don't bail out - } - else { - if (fprintf(this->resultsFile, "%s\n", outputName) < 1 ) - psError(PS_ERR_IO, false, "Unable to write FITS output filename to'%s'\n", this->resultsPath); - } + this->resultsXmlDoc = xmlNewDoc(BAD_CAST "1.0"); + xmlNodePtr rootNode = xmlNewNode(NULL, BAD_CAST "ippToPsps_Results"); + xmlDocSetRootElement(this->resultsXmlDoc, rootNode); + xmlNewChild(rootNode, NULL, BAD_CAST "filename", BAD_CAST outputName); } Index: branches/pap/ippToPsps/src/ippToPsps.h =================================================================== --- branches/pap/ippToPsps/src/ippToPsps.h (revision 28506) +++ branches/pap/ippToPsps/src/ippToPsps.h (revision 28534) @@ -15,4 +15,6 @@ #include #include "ippToPspsConfig.h" +#include +#include #define MAXDETECT 30000 //TODO limit ok? @@ -20,20 +22,20 @@ typedef struct { - uint32_t expId; // the exposure ID to be used - psString expName; // the exposure name - psString surveyType; // the survey type, eg 3PI, MD01, STS, SSS - uint8_t batchType; // PSPS batch type - psString fitsInPath; // path to FITS input - psString resultsPath; // path to results file - FILE* resultsFile; // file for results - uint16_t numOfInputFiles; // number of input files - char** inputFiles; // array of input file names - psString fitsOutPath; // path to FITS output - fitsfile *fitsOut; // output FITS file - psString configsDir; // path to IPP/PSPS mapping file - pmConfig* pmconfig; // pmConfig - dvoConfig* dvoConfig; // dvo database - IppToPspsConfig* config; // config structure - int exitCode; // ps exit code + uint32_t expId; // the exposure ID to be used + psString expName; // the exposure name + psString surveyType; // the survey type, eg 3PI, MD01, STS, SSS + uint8_t batchType; // PSPS batch type + psString fitsInPath; // path to FITS input + psString resultsPath; // path to results file + xmlDocPtr resultsXmlDoc; // pointer to XML document for results + uint16_t numOfInputFiles; // number of input files + char** inputFiles; // array of input file names + psString fitsOutPath; // path to FITS output + fitsfile *fitsOut; // output FITS file + psString configsDir; // path to IPP/PSPS mapping file + pmConfig* pmconfig; // pmConfig + dvoConfig* dvoConfig; // dvo database + IppToPspsConfig* config; // config structure + int exitCode; // ps exit code int (*run)(); Index: branches/pap/ippToPsps/src/ippToPspsBatchDetection.c =================================================================== --- branches/pap/ippToPsps/src/ippToPspsBatchDetection.c (revision 28506) +++ branches/pap/ippToPsps/src/ippToPspsBatchDetection.c (revision 28534) @@ -386,12 +386,14 @@ // write results - if (this->resultsFile) { - - if (fprintf(this->resultsFile, "%ld\n", minObjID) < 1 ) - psError(PS_ERR_IO, false, "Unable to write min Object ID to'%s'\n", this->resultsPath); - if (fprintf(this->resultsFile, "%ld\n", maxObjID) < 1 ) - psError(PS_ERR_IO, false, "Unable to write max Object ID to'%s'\n", this->resultsPath); - if (fprintf(this->resultsFile, "%ld\n", totalDetectionsOut) < 1 ) - psError(PS_ERR_IO, false, "Unable to write totalDetectionsOut to'%s'\n", this->resultsPath); + if (this->resultsXmlDoc) { + + xmlNodePtr rootNode = xmlDocGetRootElement(this->resultsXmlDoc); + char tmp[100]; + sprintf(tmp, "%ld", minObjID); + xmlNewChild(rootNode, NULL, BAD_CAST "minObjID", BAD_CAST tmp); + sprintf(tmp, "%ld", maxObjID); + xmlNewChild(rootNode, NULL, BAD_CAST "maxObjID", BAD_CAST tmp); + sprintf(tmp, "%ld", totalDetectionsOut); + xmlNewChild(rootNode, NULL, BAD_CAST "totalDetections", BAD_CAST tmp); } Index: branches/pap/ippToPsps/src/ippToPspsBatchTest.c =================================================================== --- branches/pap/ippToPsps/src/ippToPspsBatchTest.c (revision 28506) +++ branches/pap/ippToPsps/src/ippToPspsBatchTest.c (revision 28534) @@ -331,13 +331,4 @@ if (fits_close_file(fitsIn, &status)) fits_report_error(stderr, status); - // write results - if (this->resultsFile) { - - if (fprintf(this->resultsFile, "%ld\n", minObjID) < 1 ) - psError(PS_ERR_IO, false, "Unable to write min Object ID to'%s'\n", this->resultsPath); - if (fprintf(this->resultsFile, "%ld\n", maxObjID) < 1 ) - psError(PS_ERR_IO, false, "Unable to write max Object ID to'%s'\n", this->resultsPath); - } - psLogMsg("ippToPsps", PS_LOG_INFO, "Data written for a total of %d chips/OTAs", nOta); Index: branches/pap/ippTools/src/bgtool.c =================================================================== --- branches/pap/ippTools/src/bgtool.c (revision 28506) +++ branches/pap/ippTools/src/bgtool.c (revision 28534) @@ -1078,5 +1078,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); pxAddLabelSearchArgs(config, where, "-warp_label", "warpRun.label", "=="); - pxAddLabelSearchArgs(config, where, "-bg_label", "chipBackgroundRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-chip_label", "chipBackgroundRun.label", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { Index: branches/pap/ippTools/src/bgtoolConfig.c =================================================================== --- branches/pap/ippTools/src/bgtoolConfig.c (revision 28506) +++ branches/pap/ippTools/src/bgtoolConfig.c (revision 28534) @@ -249,5 +249,5 @@ psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL); - psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-bg_label", PS_META_DUPLICATE_OK, "search on warpBackgroundRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL); psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); Index: branches/pap/ippTools/src/magictool.c =================================================================== --- branches/pap/ippTools/src/magictool.c (revision 28506) +++ branches/pap/ippTools/src/magictool.c (revision 28534) @@ -1365,8 +1365,8 @@ // check that state is a valid string value if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - || (strncmp(state, "drop", 5) == 0) - || (strncmp(state, "reg", 4) == 0) + (strncmp(state, "new", 3) == 0) + || (strncmp(state, "full", 4) == 0) + || (strncmp(state, "drop", 4) == 0) + || (strncmp(state, "reg", 3) == 0) ) ) { Index: branches/pap/magic/remove/src/streaksremove.c =================================================================== --- branches/pap/magic/remove/src/streaksremove.c (revision 28506) +++ branches/pap/magic/remove/src/streaksremove.c (revision 28534) @@ -577,5 +577,5 @@ } -static void +static void setStreakBits(psImage *maskImage, psU32 maskStreak) { @@ -644,5 +644,5 @@ if (exciseAll) { strkGetMaskValues(sf); - + // add the STREAK bit to the mask image pixels setStreakBits(sf->inMask->image, sf->maskStreak); @@ -941,9 +941,5 @@ psArray *inTable = psFitsReadTable(in->fits); if (!inTable) { - psErrorStackPrint(stderr, "failed to read tablle in %s", in->resolved_name); - streaksExit("", PS_EXIT_DATA_ERROR); - } - if (!inTable->n) { - psErrorStackPrint(stderr, "table in %s is empty", in->resolved_name); + psErrorStackPrint(stderr, "failed to read table in %s", in->resolved_name); streaksExit("", PS_EXIT_DATA_ERROR); } @@ -983,4 +979,12 @@ streaksExit("", PS_EXIT_DATA_ERROR); } + } else if (psArrayLength(inTable) == 0) { + printf("No input sources\n"); + // We'd like to write a blank table, but this is as good as it gets without more work to parse the + // header and create dummy columns. + if (!psFitsWriteBlank(out->fits, header, extname)) { + psErrorStackPrint(stderr, "failed to write empty header to %s", out->resolved_name); + streaksExit("", PS_EXIT_DATA_ERROR); + } } else { printf("Censored ALL %d sources\n", numCensored); Index: branches/pap/psLib/src/fits/psFitsImage.c =================================================================== --- branches/pap/psLib/src/fits/psFitsImage.c (revision 28506) +++ branches/pap/psLib/src/fits/psFitsImage.c (revision 28534) @@ -415,5 +415,5 @@ if (fits_read_subset(fits->fd, info->fitsDatatype, info->firstPixel, info->lastPixel, info->increment, nullValue, output->data.V[0], &anynull, &status) != 0) { - psFitsError(status, true, "Reading FITS file failed."); + psFitsError(status, true, "Reading FITS file %s failed.", fits->fd->Fptr->filename); return false; } Index: branches/pap/psLib/src/fits/psFitsTable.c =================================================================== --- branches/pap/psLib/src/fits/psFitsTable.c (revision 28506) +++ branches/pap/psLib/src/fits/psFitsTable.c (revision 28534) @@ -39,4 +39,25 @@ int status = 0; // CFITSIO status + + // Check for empty table, which looks like an image + int hdutype; // Type of HDU + fits_get_hdu_type(fits->fd, &hdutype, &status); + if (psFitsError(status, true, "Could not determine the HDU type.")) { + return -1; + } + if (hdutype == IMAGE_HDU) { + // It could be an empty table + int naxis = 0; // Dimensions of image + if (fits_get_img_dim(fits->fd, &naxis, &status) != 0) { + psFitsError(status, true, "Unable to determine dimension for table."); + return -1; + } + if (naxis != 0) { + psFitsError(PS_ERR_BAD_FITS, true, "Current FITS HDU is not a table."); + return -1; + } + return 0; + } + long numRows; // Number of rows if (fits_get_num_rows(fits->fd, &numRows, &status)) { @@ -48,9 +69,10 @@ } + // Check the FITS file in preparation for reading a table static bool readTableCheck(const psFits *fits // FITS file ) { - PS_ASSERT_FITS_NON_NULL(fits, NULL); + PS_ASSERT_FITS_NON_NULL(fits, false); if (psFitsGetExtNum(fits) == 0 && !psFitsMoveExtNum(fits, 1, false)) { @@ -58,5 +80,4 @@ return false; } - // check that we are positioned on a table HDU @@ -67,6 +88,17 @@ return false; } - if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) { - psError(PS_ERR_IO, true, _("Current FITS HDU is not a table.")); + if (hdutype == IMAGE_HDU) { + // It could be an empty table + int naxis = 0; // Dimensions of image + if (fits_get_img_dim(fits->fd, &naxis, &status) != 0) { + psFitsError(status, true, "Unable to determine dimension for table."); + return false; + } + if (naxis != 0) { + psFitsError(PS_ERR_BAD_FITS, true, "Current FITS HDU is not a table."); + return false; + } + } else if (hdutype != ASCII_TBL && hdutype != BINARY_TBL) { + psError(PS_ERR_BAD_FITS, true, _("Current FITS HDU is not a table.")); return false; } Index: branches/pap/psModules/src/objects/pmSourceMatch.c =================================================================== --- branches/pap/psModules/src/objects/pmSourceMatch.c (revision 28506) +++ branches/pap/psModules/src/objects/pmSourceMatch.c (revision 28534) @@ -227,4 +227,5 @@ } + numMaster = size; xMaster->n = size; yMaster->n = size; Index: branches/pap/psModules/src/objects/pmSourcePhotometry.c =================================================================== --- branches/pap/psModules/src/objects/pmSourcePhotometry.c (revision 28506) +++ branches/pap/psModules/src/objects/pmSourcePhotometry.c (revision 28534) @@ -393,4 +393,8 @@ psImage *mask = source->maskObj; + if (!flux || !variance || !mask) { + return false; + } + for (int iy = 0; iy < flux->numRows; iy++) { for (int ix = 0; ix < flux->numCols; ix++) { Index: branches/pap/psphot/src/psphotApResid.c =================================================================== --- branches/pap/psphot/src/psphotApResid.c (revision 28506) +++ branches/pap/psphot/src/psphotApResid.c (revision 28534) @@ -328,5 +328,5 @@ psMetadataAdd (readout->analysis, PS_LIST_TAIL, "DAPMIFIT", PS_DATA_F32 | PS_META_REPLACE, "ap residual scatter", psf->dApResid); psMetadataAdd (readout->analysis, PS_LIST_TAIL, "NAPMIFIT", PS_DATA_S32 | PS_META_REPLACE, "number of apresid stars", psf->nApResid); - psMetadataAdd (readout->analysis, PS_LIST_TAIL, "APLOSS", PS_DATA_F32 | PS_META_REPLACE, "aperture loss (mag)", psf->growth->apLoss); + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "APLOSS", PS_DATA_F32 | PS_META_REPLACE, "aperture loss (mag)", psf->growth ? psf->growth->apLoss : NAN); psLogMsg ("psphot.apresid", PS_LOG_DETAIL, "aperture residual: %f +/- %f\n", psf->ApResid, psf->dApResid);