Index: branches/eam_branches/ipp-20121219/Nebulous-Server/bin/nebdiskd =================================================================== --- branches/eam_branches/ipp-20121219/Nebulous-Server/bin/nebdiskd (revision 35095) +++ branches/eam_branches/ipp-20121219/Nebulous-Server/bin/nebdiskd (revision 35096) @@ -11,7 +11,5 @@ use Config::YAML; use DBI; -use File::Mountpoint qw( is_mountpoint ); use File::Spec; -use Filesys::Df; use Log::Log4perl; use Nebulous::Server::SQL; @@ -22,12 +20,12 @@ my ( - $debug, - $db, - $dbhost, - $dbuser, - $dbpass, - $pidfile, - $stop, - $restart, + $debug, + $db, + $dbhost, + $dbuser, + $dbpass, + $pidfile, + $stop, + $restart, $verbose, $user, @@ -111,4 +109,5 @@ log4perl.appender.SERVERLOGFILE.layout = Log::Log4perl::Layout::PatternLayout log4perl.appender.SERVERLOGFILE.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n + log4perl.appender.SERVERLOGFILE.size = 10485760 log4perl.filter.MatchWarn = Log::Log4perl::Filter::LevelMatch @@ -143,5 +142,5 @@ $SIG{TERM} = sub { unlink_pid_file($pidfile); exit(); }; $SIG{INT} = $SIG{TERM}; -$SIG{HUP} = sub { $c = read_rcfile($rcfile) }; +$SIG{HUP} = sub { $c = read_rcfile($rcfile) }; my $date = localtime(); @@ -192,5 +191,4 @@ eval { my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ?,note FROM volume WHERE mountpoint = ?"); -# my $d_query = $dbh->prepare_cached("UPDATE mountedvol SET allocate = ?, available = ? WHERE mountpoint = ?"); my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?"); @@ -199,6 +197,6 @@ { # there may be multiple vol_ids per mountpoint but we only need to - # check each mointpont once - my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume"); + # check each mountpoint once + my $query = $dbh->prepare_cached("SELECT DISTINCT host, mountpoint FROM volume"); $query->execute; while (my $row = $query->fetchrow_hashref) { @@ -207,67 +205,19 @@ $query->finish; } - + # determine valid mountpoints foreach my $mnt (@$mounts) { my $mountpoint = $mnt->{'mountpoint'}; - $log->debug("checking $mountpoint"); - # this /SHOULD/ fail if the mount point is handled by the - # automounter and it fails to mount - my $tries = 0; - my $valid_mountpoint = 1; - unless (exists($host_failure_counts{$mountpoint})) { - $host_failure_counts{$mountpoint} = 0; - } - TEST: eval { - $tries++; - unless (is_mountpoint($mountpoint)) { - unless(defined($host_removed{$mountpoint})) { - $log->warn("$mountpoint is not a valid mountpoint ($tries $host_failure_counts{$mountpoint})"); - } - $valid_mountpoint = 0; - } - }; - - # fetch stats on the mounted device. this has to be done AFTER - # we determine if it's a valid mountpoint incase - # is_mountpoint() invokes the automounter - my $dev_info = df($mountpoint, 1024); + my $alt_mountpoint = $mountpoint; + $alt_mountpoint =~ s/data/export/; + my $hostname = $mnt->{'host'}; + my $valid_mountpoint = 1; + + # fetch stats on the device using ssh and df + my $dev_info = df_through_ssh($hostname, $alt_mountpoint, 1024); unless (defined $dev_info) { - $valid_mountpoint = 0; - } - - if (!$valid_mountpoint) { - # try is_mountpoint() again if $retry > 1 - if ($tries < $retry) { - $log->warn("retrying test of $mountpoint"); - goto TEST; - } - $host_failure_counts{$mountpoint}++; - - if (!(defined($host_removed{$mountpoint})) || !($host_removed{$mountpoint})) { - if (($host_failure_counts{$mountpoint} > $failure_limit)) { - $host_removed{$mountpoint} = 1; - $log->warn("Removing $mountpoint from the mountedvol table ($host_failure_counts{$mountpoint} > $failure_limit) No further warnings unless state changes."); - $d_query->execute($mountpoint); - } - else { - $log->warn("Mountpoint $mountpoint has had $host_failure_counts{$mountpoint} failures. Will remove after $failure_limit"); - } - } - next; - - } - if ($host_failure_counts{$mountpoint} != 0) { - $host_removed{$mountpoint} = 0; - $log->warn("Mountpoint $mountpoint failures cleared ($host_failure_counts{$mountpoint})"); - $host_failure_counts{$mountpoint} = 0; - } - - # fetch stats on the mounted device. this has to be done AFTER - # we determine if it's a valid mountpoint incase - # is_mountpoint() invokes the automounter - $dev_info = df($mountpoint, 1024); - unless (defined $dev_info) { - $log->error("can't find device info for $mountpoint"); + $valid_mountpoint = 0; + $log->error("can't find device info for $mountpoint / $alt_mountpoint"); + $d_query->execute($mountpoint); next; } @@ -278,13 +228,6 @@ } - -# $dbh->do("call getmountedvol()"); - -# $dbh->commit; -# $log->debug("commited to database"); }; if ($@) { -# $dbh->rollback; -# $log->debug("rolledback transaction"); $log->logdie($@); } @@ -336,5 +279,5 @@ # $dbh->commit; }; - if ($@) { + if ($@) { # $dbh->rollback; $log->logdie($@); @@ -391,7 +334,6 @@ # this follows BSD syntax ps (BSD's and linux) # this will fail on Unix98 syntax ps (Solaris, etc) - }elsif( `ps h o pid p $$` =~ /^\s*$$\s*$/ ){ # can I play ps on myself ? - $exists = `ps h o pid p $current_pid`; - + } elsif( `ps h o pid p $$` =~ /^\s*$$\s*$/ ){ # can I play ps on myself ? + $exists = `ps h o pid p $current_pid`; } @@ -412,4 +354,46 @@ } +sub df_through_ssh { + my $hostname = shift; + my $mountpoint = shift; + my $units = shift; + $log->debug("Getting info from mountpoint = [$mountpoint] / unit = [$units] / hostname = [$hostname]"); + my $df_data = qx/ssh $hostname 'df $mountpoint' | grep -v Filesystem/; + $df_data =~ s/\s+/:/g; + $log->debug("Received from ssh = [$df_data]"); + my %df_info = (); + $df_info{"blocks"} = 0; + $df_info{"used"} = 0; + unless ($df_data) { + # We cannot connect to the host for some reason However, the + # partition can be mounted on some other hosts (in the case of + # an original host failure or upgrade for instance) Let's look + # for an entry in /etc/autofs/auto.data starting with + # .[012] If it's in there, return (blocks => 1, used + # => 1). The partition is mounted but not available for + # replication (since blocks-used = 0 and used/blocks=1=100%). + my $partition_name = $mountpoint; + $partition_name =~ s/\/export\///; + $log->debug("Looking for $partition_name in /etc/autofs/auto.data"); + open(AUTODATA, ") { + if ($line =~ /^$partition_name/) { + $log->debug("$partition_name found in /etc/autofs/auto.data"); + close(AUTODATA); + $df_info{"blocks"} = 1; + $df_info{"used"} = 1; + return \%df_info; + } + } + $log->debug("$partition_name NOT found in /etc/autofs/auto.data"); + close(AUTODATA); + return undef; + } + my @data = split(/:/, $df_data); + $df_info{"blocks"} = $data[1]; + $df_info{"used"} = $data[2]; + $log->debug("blocks = " . $df_info{"blocks"} . " / used = " . $df_info{"used"}); + return \%df_info; +} __END__ @@ -525,5 +509,5 @@ =item * C -Equivalent to --dbpass|-p +Equivalent to --dbpass|-p =back Index: branches/eam_branches/ipp-20121219/Nebulous/bin/neb-df =================================================================== --- branches/eam_branches/ipp-20121219/Nebulous/bin/neb-df (revision 35095) +++ branches/eam_branches/ipp-20121219/Nebulous/bin/neb-df (revision 35096) @@ -53,5 +53,5 @@ my $path = $vol{host}; if ($names) { - $path = $vol{name}; + $path = $vol{name}; } @@ -61,13 +61,13 @@ $total_used += $used; my $available = $total - $used; - my $usedper = ($used / $total) * 100; + my $usedper = ($used / ($total+1)) * 100; my $mountpoint = $vol{mountpoint}; if ($vol{allocate} == 1) { - $allocated_total += $total; - $allocated_used += $used; + $allocated_total += $total; + $allocated_used += $used; } # indicated the part of the nebulous volume name that matches up with a - # local volume (mountpoint) + # local volume (mountpoint) #$path =~ s/$mountpoint/[$mountpoint]/; @@ -95,5 +95,5 @@ my $total_available = $total_total - $total_used; -my $total_usedper = ($total_used / $total_total) * 100; +my $total_usedper = ($total_used / $total_total) * 100; # summary @@ -112,5 +112,5 @@ my $allocated_available = $allocated_total - $allocated_used; -my $allocated_usedper = ($allocated_used / $allocated_total) * 100; +my $allocated_usedper = ($allocated_used / $allocated_total) * 100; # summary @@ -188,5 +188,5 @@ =item * C -Equivalent to --pass|-p +Equivalent to --pass|-p =back Index: branches/eam_branches/ipp-20121219/Ohana/src/getstar/src/select_by_region.c =================================================================== --- branches/eam_branches/ipp-20121219/Ohana/src/getstar/src/select_by_region.c (revision 35095) +++ branches/eam_branches/ipp-20121219/Ohana/src/getstar/src/select_by_region.c (revision 35096) @@ -53,7 +53,6 @@ gfits_scan (&catalog[0].header, "DEC1", "%lf", 1, &Dmax); - if (VERBOSE) fprintf (stderr, "extracting from catalog covering region %f,%f to %f,%f\n", Rmin, Dmin, Rmax, Dmax); - AREA = fabs(Dmax - Dmin) * fabs(Rmax - Rmin) * cos (0.5*RAD_DEG*(Dmax + Dmin)); + if (VERBOSE) fprintf (stderr, "extracting from catalog covering region %f,%f to %f,%f for area %f\n", Rmin, Dmin, Rmax, Dmax, AREA); assert (AREA > 0); @@ -83,5 +82,4 @@ if (isnan(mag)) continue; if (mag < MinMagValue) continue; - if (mag > MagLimitValue) continue; bin = (mag - MagMin) / dMag; @@ -101,5 +99,5 @@ } MagLimitValue = i*dMag + MagMin; - if (VERBOSE) fprintf (stderr, "using %d stars in mag range %f - %f\n", Nsum, MinMagValue, MagLimitValue); + if (VERBOSE) fprintf (stderr, "using %d (Nmax %d) stars in mag range %f - %f\n", Nsum, Nmax, MinMagValue, MagLimitValue); } Index: branches/eam_branches/ipp-20121219/dbconfig/changes.txt =================================================================== --- branches/eam_branches/ipp-20121219/dbconfig/changes.txt (revision 35095) +++ branches/eam_branches/ipp-20121219/dbconfig/changes.txt (revision 35096) @@ -2265,2 +2265,73 @@ ALTER TABLE skycalResult ADD column n_forced INT after n_extended; UPDATE dbversion set schema_version = '1.1.73', updated= CURRENT_TIMESTAMP(); + + +-- new tables for managing the "release" of ipp data + +CREATE TABLE survey ( + surveyID INT(8), + surveyName VARCHAR(16) NOT NULL, + description VARCHAR(255), + PRIMARY KEY(surveyID), + UNIQUE KEY(surveyName) +) ENGINE=InnoDB CHARSET=latin1; + + +CREATE TABLE ippRelease ( + rel_id INT(8) AUTO_INCREMENT, + surveyID INT(8), + releaseName VARCHAR(64), + release_state VARCHAR(16), -- active, pending archive drop + dataRelease INT(8), -- PSPS dataRelease + PRIMARY KEY(rel_id), + KEY(releaseName), + KEY(state), + FOREIGN KEY(surveyID) REFERENCES survey(surveyID) +) ENGINE=Innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relExp ( + relexp_id BIGINT AUTO_INCREMENT, + rel_id INT(8), + exp_id BIGINT, + chip_id BIGINT, -- links to the runs that supplied the data + cam_id BIGINT, -- for this release + state VARCHAR(16), -- released, pending, archive, drop + flags INT UNSIGNED, -- flags for relphot, relastro, ?? + zpt_obs FLOAT, -- calibrated zero point for this release of + zpt_stdev FLOAT, -- this exposure + path_base VARCHAR(255), -- path_base of any supporting files for this + -- release of this exposure. + registered DATETIME, -- insertion time for this row + time_stamp DATETIME, -- time of last update for this row + PRIMARY KEY (relexp_id), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + KEY (state), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relStack ( + relstack_id BIGINT AUTO_INCREMENT, + rel_id INT(8), + skycal_id BIGINT, -- id of the sky calibration run that supplied the calibration + -- for this release. We get back to stack from here + skycell_id varchar(64), + tess_id varchar(64), + state VARCHAR(16), -- released, pending, archive, drop + flags INT UNSIGNED, + stack_type VARCHAR(16), -- nightly, deep, reference + zpt_obs FLOAT, -- calibrated zero point for this release of this skycell + zpt_stdev FLOAT, + fwhm_major FLOAT, -- seeing as measured by staticsky + path_base VARCHAR(255), -- path_base of any supporting files, + registered DATETIME, + time_stamp DATETIME, + PRIMARY KEY (relstack_id), + KEY (tess_id, skycell_id), + KEY (rel_id, tess_id, skycell_id), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +UPDATE dbversion set schema_version = '1.1.74', updated= CURRENT_TIMESTAMP(); Index: branches/eam_branches/ipp-20121219/dbconfig/ipp.m4 =================================================================== --- branches/eam_branches/ipp-20121219/dbconfig/ipp.m4 (revision 35095) +++ branches/eam_branches/ipp-20121219/dbconfig/ipp.m4 (revision 35096) @@ -39,2 +39,3 @@ include(vp.md) include(skycell.md) +include(release.md) Index: branches/eam_branches/ipp-20121219/dbconfig/release.md =================================================================== --- branches/eam_branches/ipp-20121219/dbconfig/release.md (revision 35096) +++ branches/eam_branches/ipp-20121219/dbconfig/release.md (revision 35096) @@ -0,0 +1,46 @@ +survey METADATA + surveyID S32 0 # Primary Key + surveyName STR 16 + description STR 255 +END + +ippRelease METADATA + rel_id S32 0 # Primary Key AUTO_INCREMENT + surveyID S32 0 # fkey(surveyID) ref survey(surveyID) + releaseName STR 64 + release_state STR 16 + dataRelease S32 0 + priority S32 0 +END + +relExp METADATA + relexp_id S64 0 # Primary Key AUTO_INCREMENT + rel_id S32 0 # fkey(rel_id) ref ippRelease(rel_id) + exp_id S64 0 + chip_id S64 0 + cam_id S64 0 + state STR 16 + flags U32 0 + zpt_obs F32 0 + zpt_stdev F32 0 + path_base STR 255 + registered TAI NULL + time_stamp UTC NULL +END + +relStack METADATA + relstack_id S64 0 # Primary Key AUTO_INCREMENT + rel_id S32 0 # fkey(rel_id) ref ippRelease(rel_id) + skycal_id S64 0 # fkey(skycal_id) ref skycalRun(skycal_id) + skycell_id STR 64 + tess_id STR 64 + state STR 16 + flags U32 0 + stack_type STR 16 + zpt_obs F32 0 + zpt_stdev F32 0 + fwhm_major F32 0 + path_base STR 255 + registered TAI NULL + time_stamp TAI NULL +END Index: branches/eam_branches/ipp-20121219/ippMonitor/Makefile.in =================================================================== --- branches/eam_branches/ipp-20121219/ippMonitor/Makefile.in (revision 35095) +++ branches/eam_branches/ipp-20121219/ippMonitor/Makefile.in (revision 35096) @@ -215,5 +215,4 @@ $(DESTWWW)/simplePlotStackImage.php \ $(DESTWWW)/simplePlotStaticskyImage.php \ -$(DESTWWW)/newsimplePlotStaticskyImage.php \ $(DESTWWW)/simplePlotSkycalImage.php \ $(DESTWWW)/histogramBackgroundImage.php \ Index: branches/eam_branches/ipp-20121219/ippMonitor/czartool/czarDbSchema.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippMonitor/czartool/czarDbSchema.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippMonitor/czartool/czarDbSchema.sql (revision 35096) @@ -78,4 +78,5 @@ `total` float default NULL, `available` float default NULL, + `usable` float default NULL, `used` float default NULL, `hostsOver98` smallint(6) default NULL, Index: branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/CzarDb.pm (revision 35095) +++ branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/CzarDb.pm (revision 35096) @@ -9,11 +9,11 @@ my @stages = ( - "chip", - "cam", - "fake", - "warp", - "stack", - "diff", - "dist", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "dist", "pub", "chipbackground"); # TODO put elsewhere @@ -64,6 +64,6 @@ SELECT DISTINCT label FROM processed - WHERE timestamp >= '$begin' - AND timestamp <= '$end' + WHERE timestamp >= '$begin' + AND timestamp <= '$end' SQL @@ -109,7 +109,7 @@ SELECT DISTINCT label - FROM chip - WHERE timestamp >= '$begin' - AND timestamp <= '$end' + FROM chip + WHERE timestamp >= '$begin' + AND timestamp <= '$end' AND label NOT LIKE '%_ud%' AND label NOT LIKE 'update%' @@ -153,7 +153,7 @@ my $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<{_dateFormat}'), + SELECT + DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'), TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp))) - FROM $stage + FROM $stage WHERE label LIKE '$label' - AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; + AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; SQL @@ -413,5 +413,5 @@ ########################################################################### # -# Analysis this stage between these times +# Analysis this stage between these times # ########################################################################### @@ -429,9 +429,9 @@ my $query = $self->{_db}->prepare(<= '$fromTime' AND timestamp <= '$toTime' + SELECT + timestamp, pending, faults, processed + FROM $stage + WHERE label LIKE '$label' + AND timestamp >= '$fromTime' AND timestamp <= '$toTime' ORDER BY timestamp; SQL @@ -452,5 +452,5 @@ # get linear values if($lastProcessed != -1 && $thisProcessed > $lastProcessed) { - + $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed); } @@ -480,5 +480,5 @@ my $started = undef; my $finished = undef; - + my ($have25, $have50, $have75, $have90, $have95, $have100) = 0; @@ -492,5 +492,5 @@ # get linear values if($lastProcessed != -1 && $thisProcessed > $lastProcessed) { - + $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed); } @@ -508,14 +508,14 @@ my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp) + 0.001; - + if ($thisProcessed >= $lastProcessed){ - $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; + $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; } else {$derivProcessed = 0;} - $derivPending = abs($thisPending - $lastPending)/$timeDiff; - $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff; + $derivPending = abs($thisPending - $lastPending)/$timeDiff; + $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff; } else { - + $derivProcessed = 0; $derivPending = 0; @@ -539,12 +539,12 @@ else { if (defined $notProcessing) { - - $notProcessing = undef; - $stuck = undef; + + $notProcessing = undef; + $stuck = undef; } if (!defined $started) { $started = $lastTimestamp; - $finished = undef; + $finished = undef; } } @@ -564,10 +564,10 @@ # how long not processing? if (defined $notProcessing) { - + $howLongNotProcessing = $self->diffTimes($lastTimestamp, $notProcessing); $notProcessingLongerThanInterval = $self->isBefore($interval, $howLongNotProcessing); } else { - + $howLongNotProcessing = 0; $notProcessingLongerThanInterval = 0; @@ -611,9 +611,9 @@ if (!defined $started){ - + $stageMetrics->setStarted(undef); } else { - + my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0; $stageMetrics->setRate($rate); @@ -625,5 +625,5 @@ } else { - + $stageMetrics->setStuck(undef); } @@ -642,7 +642,7 @@ my $query = $self->{_db}->prepare(<= '$fromTime' AND timestamp <= '$toTime'; + AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; SQL @@ -657,9 +657,9 @@ # grab all the data for the provided time range $query = $self->{_db}->prepare(<= '$fromTime' AND timestamp <= '$toTime' + SELECT + timestamp, pending, faults, processed + FROM $stage + WHERE label LIKE '$label' + AND timestamp >= '$fromTime' AND timestamp <= '$toTime' ORDER BY timestamp; SQL @@ -724,5 +724,5 @@ $diffFaults = $thisFaults - $lastFaults; - # First, look for large positive jumps in processed, most likely a label + # First, look for large positive jumps in processed, most likely a label # added, and store value to subtract from linear value below. If > 1000 # processed per hour, then it must be wrong @@ -741,5 +741,5 @@ } else { - + $diffProcessed = 0; $linearProcessed = $linearProcessed; @@ -747,12 +747,12 @@ # calculate first derivative in units of images per hour - $derivPending = $diffPending/($timeSep/3600); - $derivFaults = $diffFaults/($timeSep/3600); - $derivProcessed = $diffProcessed/($timeSep/3600); + $derivPending = $diffPending/($timeSep/3600); + $derivFaults = $diffFaults/($timeSep/3600); + $derivProcessed = $diffProcessed/($timeSep/3600); } # first time in else { - + $derivPending = 0; $derivFaults = 0; @@ -777,6 +777,6 @@ my $totalPending = 0; my $totalFaults = 0; - for (my $i=$size-1; $i>=($size - $range); $i--) { - + for (my $i=$size-1; $i>=($size - $range); $i--) { + $totalPending = $totalPending + $ratePending[$i]; $totalFaults = $totalFaults + $rateFaults[$i]; @@ -833,16 +833,16 @@ # get earliest time for this label my $query = $self->{_db}->prepare(<execute; - -my $timestamp = $query->fetchrow_array(); + WHERE label LIKE '$label'; +SQL + $query->execute; + +my $timestamp = $query->fetchrow_array(); $query = $self->{_db}->prepare(<{_db}->prepare(<{_db}->prepare(<execute) {return 0;} - (${$pending}, ${$faults}) = $query->fetchrow_array(); + (${$pending}, ${$faults}) = $query->fetchrow_array(); ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime); @@ -912,6 +912,6 @@ my $query = $self->{_db}->prepare(< $limit) { - $underLimit = 0; - } else { - $underLimit = 1; - } + + if (($used+$available) == 0) { + $underLimit = 1; + } elsif (($used/($used+$available))*100 > $limit) { + $underLimit = 0; + } else { + $underLimit = 1; + } # Col 1: host print GNUDAT "$host"; - + # Col 2: available, readable used space under limit if ($readable && $underLimit) {print GNUDAT " $used";} @@ -973,6 +973,6 @@ my $query = $self->{_db}->prepare(<".$tmpFile->filename); my $query = $self->{_db}->prepare(<{_dateFormat}'), + SELECT + MAX(available), MIN(available), MAX(usable), MIN(usable), + DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'), TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp))) - FROM cluster_space - WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'; + FROM cluster_space + WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'; SQL if (!$query->execute) {return undef;} - (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array(); - + ($maxAvailable, $minAvailable, $maxUsable, $minUsable, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array(); + ${$maxY} = ($maxAvailable > $maxUsable ? $maxAvailable : $maxUsable); + ${$minY} = ($minAvailable > $minUsable ? $minAvailable : $minUsable); if (!${$maxY} || !${$minY} || !${$maxX} || !${$minX} || !${$timeDiff}) {return 0;} $query = $self->{_db}->prepare(<{_dateFormat}'), available - FROM cluster_space - WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime' + SELECT + DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available, usable + FROM cluster_space + WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime' ORDER BY timestamp; SQL @@ -1021,6 +1026,6 @@ while (my @row = $query->fetchrow_array()) { - my ($timestamp, $available) = @row; - print GNUDAT "$timestamp $available\n"; + my ($timestamp, $available, $usable) = @row; + print GNUDAT "$timestamp $available $usable\n"; } @@ -1074,5 +1079,5 @@ ########################################################################### # -# Gets count of processed stuff in a given time period +# Gets count of processed stuff in a given time period # ########################################################################### @@ -1081,10 +1086,10 @@ my $query = $self->{_db}->prepare(<= '$begin' - AND timestamp < '$end' + WHERE label LIKE '$label' + AND timestamp >= '$begin' + AND timestamp < '$end' SQL $query->execute; @@ -1168,9 +1173,9 @@ $query = $self->{_db}->prepare(< '$fromTime' AND timestamp <= '$toTime' - AND label = '$label' + AND label = '$label' SQL @@ -1181,7 +1186,7 @@ $query = $self->{_db}->prepare(< '$fromTime' - AND timestamp <= '$toTime' + DELETE FROM $stage + WHERE timestamp > '$fromTime' + AND timestamp <= '$toTime' AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete SQL @@ -1203,9 +1208,9 @@ $query = $self->{_db}->prepare(< '$fromTime' AND timestamp <= '$toTime' - AND server = '$server' + AND server = '$server' SQL $query->execute; @@ -1215,7 +1220,7 @@ $query = $self->{_db}->prepare(< '$fromTime' - AND timestamp <= '$toTime' + DELETE FROM servers + WHERE timestamp > '$fromTime' + AND timestamp <= '$toTime' AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete SQL @@ -1229,6 +1234,6 @@ # now deal with cluster_space table $query = $self->{_db}->prepare(< '$fromTime' AND timestamp <= '$toTime' @@ -1240,7 +1245,7 @@ $query = $self->{_db}->prepare(< '$fromTime' - AND timestamp <= '$toTime' + DELETE FROM cluster_space + WHERE timestamp > '$fromTime' + AND timestamp <= '$toTime' ORDER BY timestamp DESC LIMIT $toDelete SQL @@ -1257,5 +1262,5 @@ ########################################################################### # -# Optimizes all tables that need to be optimized +# Optimizes all tables that need to be optimized # ########################################################################### @@ -1275,5 +1280,5 @@ ########################################################################### # -# Returns if a particular server has been stopped for a certain interval +# Returns if a particular server has been stopped for a certain interval # ########################################################################### @@ -1286,5 +1291,5 @@ ########################################################################### # -# Returns if a particular server has been down for a certain interval +# Returns if a particular server has been down for a certain interval # ########################################################################### @@ -1297,5 +1302,5 @@ ########################################################################### # -# Returns if a particular server has been down for a certain interval +# Returns if a particular server has been down for a certain interval # ########################################################################### @@ -1304,7 +1309,7 @@ my $query = $self->{_db}->prepare(< now() - INTERVAL $interval SQL @@ -1316,7 +1321,7 @@ $query = $self->{_db}->prepare(< now() - INTERVAL $interval AND !$mode @@ -1329,10 +1334,10 @@ $query = $self->{_db}->prepare(<{_db}->prepare(< '$fromTime' + SELECT DISTINCT label + FROM $stage + WHERE timestamp > '$fromTime' AND timestamp <= '$toTime'; SQL Index: branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Nebulous.pm =================================================================== --- branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Nebulous.pm (revision 35095) +++ branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Nebulous.pm (revision 35096) @@ -28,6 +28,6 @@ ########################################################################### # -# Converts Kb to Tb -# +# Converts Kb to Tb +# ########################################################################### sub convertKbToTb { @@ -50,5 +50,5 @@ ########################################################################### # -# Returns total cluster usage in T +# Returns total cluster usage in T # ########################################################################### @@ -61,5 +61,5 @@ ########################################################################### # -# Returns total cluster space in T +# Returns total cluster space in T # ########################################################################### @@ -72,5 +72,5 @@ ########################################################################### # -# Returns total cluster available in T +# Returns total cluster available in T # ########################################################################### @@ -83,5 +83,5 @@ ########################################################################### # -# Runs neb-df and puts output in gnuplot ready format +# Runs neb-df and puts output in gnuplot ready format # ########################################################################### @@ -93,4 +93,5 @@ $self->{_totalHosts} = 0; $self->{_hostsOverNinetyEightPC} = 0; + $self->{_totalUsable} = 0; my $readable; my $writable; @@ -100,6 +101,7 @@ if ($line =~ m/.*Filesystem.*/i) {next;} elsif ($line =~ m/.*summmary.*/i) {next;} - elsif ($line =~ m/.*total.*/i) {next;} - elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) { + elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*total.*/i) { + $self->{_totalUsable} += $self->convertKbToTb($2); + } elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) { $self->{_totalUsed} = $self->convertKbToTb($1); @@ -109,9 +111,8 @@ next; - } - elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)|| - ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) || - ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) { - + } elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)|| + ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) || + ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) { + $self->{_totalHosts}++; if ($4 >= 98) {$self->{_hostsOverNinetyEightPC}++;} @@ -119,8 +120,8 @@ $self->getReadableWritableStatus($1, \$readable, \$writable); - $self->{_czarDb}->updateHost($1, - $self->convertKbToTb($2+$3), - $self->convertKbToTb($3), - $self->convertKbToTb($2), + $self->{_czarDb}->updateHost($1, + $self->convertKbToTb($2+$3), + $self->convertKbToTb($3), + $self->convertKbToTb($2), $readable, $writable); } @@ -128,7 +129,8 @@ $self->{_czarDb}->insertNewClusterSpace( - $self->{_totalCluster}, - $self->{_totalAvail}, - $self->{_totalUsed}, + $self->{_totalCluster}, + $self->{_totalAvail}, + $self->{_totalUsable}, + $self->{_totalUsed}, $self->{_hostsOverNinetyEightPC}); } @@ -136,5 +138,5 @@ ########################################################################### # -# Runs neb-host and readable/writable status for this host +# Runs neb-host and readable/writable status for this host # ########################################################################### Index: branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Plotter.pm =================================================================== --- branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Plotter.pm (revision 35095) +++ branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Plotter.pm (revision 35096) @@ -630,5 +630,5 @@ set xlabel "Time" set ylabel "Available (TB)" - plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2 + plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2, "$datFile" using 1:3 title "Theoretical" with lines lt 3 lw 2 PLOT Index: branches/eam_branches/ipp-20121219/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/eam_branches/ipp-20121219/ippScripts/scripts/camera_exp.pl (revision 35095) +++ branches/eam_branches/ipp-20121219/ippScripts/scripts/camera_exp.pl (revision 35096) @@ -41,5 +41,5 @@ my ( $exp_tag, $cam_id, $camera, $outroot, $dbname, $reduction, $dvodb, $verbose, $no_update, - $no_op, $redirect, $save_temps, $run_state, $skip_binned, $skip_masks); + $no_op, $redirect, $save_temps, $run_state, $skip_binned, $skip_masks, $bkg_only); GetOptions( 'exp_tag=s' => \$exp_tag, # Exposure identifier @@ -53,4 +53,5 @@ 'skip-binned' => \$skip_binned, # override recipe - don't create binned images 'skip-refmask' => \$skip_masks, # override recipe - don't create refmask + 'bkg-only' => \$bkg_only, # override recipe - only do background continuity 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Update the database? @@ -111,4 +112,5 @@ &my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; +my $bkg_recipe = 'PPIMAGE_BKGCONT'; # Add to reduction? my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -215,6 +217,10 @@ my ($list4File, $list4Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b4.list.XXXX", UNLINK => !$save_temps ); # For astrometry +### Create temp file for background models +my ($list5File, $list5Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b5.list.XXXX", UNLINK => !$save_temps ); # For background models + # XXX we perform astrometry iff photometry output exists my @outMasks; # Names of output masks +my @bkg_models; # Names of output background models foreach my $file (@$files) { next if $file->{quality} != 0; @@ -226,11 +232,13 @@ my $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id); my $chipMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id); - + print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n"); print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n"); print $list3File ($chipObjects . "\n"); print $list4File ($chipMask . "\n"); + print $list5File ($ipprc->filename("PSPHOT.BACKMDL", $file->{path_base}, $class_id) . "\n"); push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks; + push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1); } close $list1File; @@ -238,5 +246,5 @@ close $list3File; close $list4File; - +close $list5File; # Output products $ipprc->outroot_prepare($outroot); @@ -291,5 +299,5 @@ } - { + if (!$bkg_only) { # run psastro on the chipObjects, producing fpaObjects my $command; @@ -353,4 +361,22 @@ } } + } + # Construct FPA continuity corrected background images + { + my $command; + $command = "$ppImage"; + $command .= " -list $list5Name $outroot"; + $command .= " -recipe PPIMAGE $bkg_recipe"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppImage to fix background: $error_code", $cam_id, $error_code); + } + foreach my $bkgModel (@bkg_models) { + check_output($bkgModel, $replicateOutputs); + } } } Index: branches/eam_branches/ipp-20121219/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/eam_branches/ipp-20121219/ippScripts/scripts/ipp_cleanup.pl (revision 35095) +++ branches/eam_branches/ipp-20121219/ippScripts/scripts/ipp_cleanup.pl (revision 35096) @@ -65,4 +65,10 @@ &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; + +my $hostname = `hostname`; +chomp $hostname; +print STDERR "starting cleanup for $stage $stage_id on $hostname\n" ; + +my $nebulous_server; my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n'; @@ -2280,9 +2286,24 @@ if (my $resolved = $ipprc->file_resolve($filename)) { my $bzip2_filename = $filename . '.bz2'; - if ($ipprc->file_exists($bzip2_filename)) { - $ipprc->kill_file($bzip2_filename); - } + if (file_scheme($bzip2_filename) eq 'neb') { + # XXX: GRRRR $ipprc->file_exists() returns false if storage object exists but instance has + # size of zero + $nebulous_server = $ipprc->nebulous() if !$nebulous_server; + if ($nebulous_server->storage_object_exists($bzip2_filename)) { + print STDERR "$bzip2_filename exists, killing\n"; + $ipprc->kill_file($bzip2_filename); + } else { + print STDERR "$bzip2_filename does not exist\n" if $very_verbose; + } + } else { + if ($ipprc->file_exists($bzip2_filename)) { + print STDERR "$bzip2_filename exists, killing\n"; + $ipprc->kill_file($bzip2_filename); + } else { + print STDERR "$bzip2_filename does not exist\n" if $very_verbose; + } + } my $bzip2_file = $ipprc->file_create($bzip2_filename); - my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file; + my_die("Unable to create $bzip2_filename", $stage_id, $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file; my $command = "$bzip2 < $resolved > $bzip2_file"; Index: branches/eam_branches/ipp-20121219/ippTasks/survey.pro =================================================================== --- branches/eam_branches/ipp-20121219/ippTasks/survey.pro (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTasks/survey.pro (revision 35096) @@ -1529,5 +1529,5 @@ # note: currently skycal uses the staticskyRun workdir - $run = staticskytool -defineskycalrun -select_label $label -set_dist_group $dist_group + $run = staticskytool -defineskycalrun -set_label $label -select_label $label -set_dist_group $dist_group if ($DB:n == 0) Index: branches/eam_branches/ipp-20121219/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/Makefile.am (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/Makefile.am (revision 35096) @@ -461,4 +461,8 @@ vptool_processedcell.sql \ vptool_revertrun.sql \ - sctool_list.sql + sctool_list.sql \ + releasetool_listsurvey.sql \ + releasetool_listrelease.sql \ + releasetool_listrelexp.sql \ + releasetool_definerelexp.sql Index: branches/eam_branches/ipp-20121219/ippTools/share/bgtool_definechip.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/bgtool_definechip.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/bgtool_definechip.sql (revision 35096) @@ -3,4 +3,8 @@ FROM chipRun JOIN rawExp USING(exp_id) +JOIN camRun USING(chip_id) +JOIN camProcessedExp USING(cam_id) LEFT JOIN chipBackgroundRun USING(chip_id) WHERE chipRun.state = 'full' +AND camRun.state = 'full' +AND camProcessedExp.quality = 0 Index: branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_create_tables.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_create_tables.sql (revision 35096) @@ -2148,5 +2148,74 @@ ) ENGINE=InnoDB DEFAULT CHARSET=latin1; - --- This comment line is here to avoid empty query error. +CREATE TABLE survey ( + surveyID INT(8), + surveyName VARCHAR(16) NOT NULL, + description VARCHAR(255), + PRIMARY KEY(surveyID), + UNIQUE KEY(name) +) ENGINE=InnoDB CHARSET=latin1; + + +CREATE TABLE ippRelease ( + rel_id INT(8) AUTO_INCREMENT, + surveyID INT(8), + releaseName VARCHAR(64), + relase_state VARCHAR(16), -- active, pending, archive, drop + dataRelease INT(8), -- PSPS dataRelease + priority INT(8), + PRIMARY KEY(rel_id), + KEY(releaseName), + KEY(state), + KEY(priority), + FOREIGN KEY(surveyID) REFERENCES survey(surveyID) +) ENGINE=Innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relExp ( + relexp_id BIGINT AUTO_INCREMENT, + rel_id INT(8), + exp_id BIGINT, + chip_id BIGINT, -- links to the runs that supplied the data + cam_id BIGINT, -- for this release + state VARCHAR(16), -- released, pending, archive, drop + flags INT UNSIGNED, -- flags for relphot, relastro, ?? + zpt_obs FLOAT, -- calibrated zero point for this release of + zpt_stdev FLOAT, -- this exposure + path_base VARCHAR(255), -- path_base of any supporting files for this + -- release of this exposure. + registered DATETIME, -- insertion time for this row + time_stamp DATETIME, -- time of last update for this row + PRIMARY KEY (relexp_id), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + KEY (state), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relStack ( + relstack_id BIGINT AUTO_INCREMENT, + rel_id INT(8), + skycal_id BIGINT, -- id of the sky calibration run that supplied the calibration + -- for this release. We get back to stack from here + skycell_id varchar(64), + tess_id varchar(64), + state VARCHAR(16), -- released, pending, archive, drop + flags INT UNSIGNED, + stack_type VARCHAR(16), -- nightly, deep, reference + zpt_obs FLOAT, -- calibrated zero point for this release of this skycell + zpt_stdev FLOAT, + fwhm_major FLOAT, -- seeing as measured by staticsky + path_base VARCHAR(255), -- path_base of any supporting files, + registered DATETIME, + time_stamp DATETIME, + PRIMARY KEY (relstack_id), + KEY (tess_id, skycell_id), + KEY (rel_id, tess_id, skycell_id), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + + +-- These comment lines are here to avoid an empty query error. -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_drop_tables.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_drop_tables.sql (revision 35096) @@ -109,4 +109,7 @@ DROP TABLE IF EXISTS skycalResult; DROP TABLE IF EXISTS skycell; +DROP TABLE IF EXISTS relExp; +DROP TABLE IF EXISTS ippRelease; +DROP TABLE IF EXISTS survey; SET FOREIGN_KEY_CHECKS=1 Index: branches/eam_branches/ipp-20121219/ippTools/share/releasetool_definerelexp.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/releasetool_definerelexp.sql (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/share/releasetool_definerelexp.sql (revision 35096) @@ -0,0 +1,19 @@ +SELECT + ippRelease.rel_id, + ippRelease.releaseName, + rawExp.exp_name, + rawExp.filter, + rawExp.dateobs, + rawExp.exp_id, + camRun.chip_id, + camRun.cam_id, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev +FROM ippRelease +JOIN camRun +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN relExp AS previousrelexp USING(exp_id, rel_id) + +WHERE previousrelexp.relexp_id IS NULL Index: branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listrelease.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listrelease.sql (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listrelease.sql (revision 35096) @@ -0,0 +1,1 @@ +select * from ippRelease JOIN survey USING(surveyID) Index: branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listrelexp.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listrelexp.sql (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listrelexp.sql (revision 35096) @@ -0,0 +1,38 @@ +SELECT + relexp_id, + relExp.state, + ippRelease.releaseName, + ippRelease.release_state, + ippRelease.rel_id, + ippRelease.priority, + survey.surveyName, + rawExp.exp_name, + relExp.flags, + relExp.zpt_obs, + relExp.zpt_stdev, + relExp.path_base, + relExp.registered, + relExp.time_stamp, + relExp.exp_id, + relExp.chip_id, + chipRun.state as chip_state, + relExp.cam_id, + camRun.state as cam_state, + camProcessedExp.path_base as cam_path_base, + camProcessedExp.fwhm_major, + warpRun.warp_id, + warpRun.state as warp_state, + rawExp.filter, + rawExp.dateobs, + TRUNCATE(DEGREES(rawExp.ra), 4) AS radeg, + TRUNCATE(DEGREES(rawExp.decl), 4) AS decdeg, + rawExp.comment +FROM relExp +JOIN ippRelease USING(rel_id) +JOIN survey USING(surveyID) +JOIN rawExp using(exp_id) +JOIN chipRun using(chip_id, exp_id) +JOIN camRun using(cam_id, chip_id) +JOIN camProcessedExp using(cam_id) +JOIN fakeRun using(cam_id) +JOIN warpRun using(fake_id) Index: branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listsurvey.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listsurvey.sql (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/share/releasetool_listsurvey.sql (revision 35096) @@ -0,0 +1,1 @@ +SELECT * from survey Index: branches/eam_branches/ipp-20121219/ippTools/share/stacktool_tobkg.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/stacktool_tobkg.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/stacktool_tobkg.sql (revision 35096) @@ -25,5 +25,6 @@ -- WHERE hook %s GROUP BY stack_id -HAVING (SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id) AND +HAVING ((SUM(IF(warpRun.state = 'cleaned', 1, 0)) = COUNT(stackInputSkyfile.warp_id) OR + SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id)) AND SUM(IF(warpSkyfile.background_model = 1, 1, 0)) = COUNT(stackInputSkyfile.warp_id)) Index: branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_defineskycalrun.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_defineskycalrun.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_defineskycalrun.sql (revision 35096) @@ -1,5 +1,5 @@ SELECT - sky_id, - stack_id, + staticskyRun.sky_id, + stackRun.stack_id, skycell_id, filter, @@ -14,5 +14,5 @@ JOIN stackSumSkyfile USING(stack_id) JOIN skycell USING(tess_id, skycell_id) - LEFT JOIN skycalRun USING(sky_id, stack_id) + LEFT JOIN skycalRun ON staticskyRun.sky_id = skycalRun.sky_id AND stackRun.stack_id = skycalRun.stack_id -- join hook %s WHERE staticskyRun.state = 'full' AND staticskyResult.quality = 0 Index: branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revert.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revert.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revert.sql (revision 35096) @@ -1,5 +1,9 @@ DELETE FROM staticskyResult -USING staticskyResult, staticskyRun +USING staticskyResult, staticskyRun, staticskyInput, stackRun, skycell WHERE staticskyRun.sky_id = staticskyResult.sky_id + AND staticskyRun.sky_id = staticskyInput.sky_id + AND staticskyInput.stack_id = stackRun.stack_id + AND stackRun.tess_id = skycell.tess_id + AND stackRun.skycell_id = skycell.skycell_id AND staticskyRun.state = 'new' AND staticskyResult.fault != 0 Index: branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revertskycal.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revertskycal.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revertskycal.sql (revision 35096) @@ -1,6 +1,8 @@ DELETE FROM skycalResult -USING skycalResult, skycalRun, stackRun +USING skycalResult, skycalRun, stackRun, skycell WHERE skycalRun.skycal_id = skycalResult.skycal_id AND stackRun.stack_id = skycalRun.stack_id AND skycalRun.state = 'new' AND skycalResult.fault != 0 + AND stackRun.tess_id = skycell.tess_id + AND stackRun.skycell_id = skycell.skycell_id Index: branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_skycalresult.sql =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_skycalresult.sql (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_skycalresult.sql (revision 35096) @@ -2,13 +2,12 @@ skycalResult.*, skycalRun.stack_id, - stackRun.tess_id, - stackRun.skycell_id, stackRun.filter, skycalRun.state, skycalRun.label, skycalRun.data_group, - skycalRun.sky_id + skycalRun.sky_id, + skycell.* FROM skycalRun JOIN skycalResult USING(skycal_id) -JOIN stackRun USING(stack_id) - +JOIN stackRun USING(stack_id) +JOIN skycell USING(tess_id, skycell_id) Index: branches/eam_branches/ipp-20121219/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/Makefile.am (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/src/Makefile.am (revision 35096) @@ -32,5 +32,6 @@ laptool \ vptool \ - sctool + sctool \ + releasetool pkginclude_HEADERS = \ @@ -85,5 +86,6 @@ laptool.h \ vptool.h \ - sctool.h + sctool.h \ + releasetool.h lib_LTLIBRARIES = libpxtools.la @@ -324,4 +326,10 @@ sctoolConfig.c +releasetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +releasetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +releasetool_SOURCES = \ + releasetool.c \ + releasetoolConfig.c + clean-local: -rm -f TAGS Index: branches/eam_branches/ipp-20121219/ippTools/src/bgtool.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/bgtool.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/src/bgtool.c (revision 35096) @@ -693,4 +693,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipBackgroundRun.chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "LIKE"); Index: branches/eam_branches/ipp-20121219/ippTools/src/bgtoolConfig.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/bgtoolConfig.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/src/bgtoolConfig.c (revision 35096) @@ -162,4 +162,5 @@ psMetadata *chipArgs = psMetadataAlloc(); psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); psMetadataAddS16(chipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); Index: branches/eam_branches/ipp-20121219/ippTools/src/pxspace.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/pxspace.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/src/pxspace.c (revision 35096) @@ -67,4 +67,6 @@ psMetadataAddF32(md, PS_LIST_TAIL, "-glong_max", 0, "search by galactic longitude (degrees)", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-glat_max", 0, "search by galactic latitude (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-abs_glat_min", 0, "search by absolute value(galactic latitude) (degrees)", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-abs_glat_max", 0, "search by absolute value(galactic latitude) (degrees)", NAN); } @@ -79,4 +81,6 @@ PXOPT_COPY_F32(config->args, where, "-glong_max", "skycell.glong", "<"); PXOPT_COPY_F32(config->args, where, "-glat_max", "skycell.glat", "<"); + PXOPT_COPY_F32(config->args, where, "-abs_glat_min", "abs(skycell.glat)", ">="); + PXOPT_COPY_F32(config->args, where, "-abs_glat_max", "abs(skycell.glat)", "<"); return true; } Index: branches/eam_branches/ipp-20121219/ippTools/src/releasetool.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/releasetool.c (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/src/releasetool.c (revision 35096) @@ -0,0 +1,535 @@ +/* + * releasetool.c + * + * Copyright (C) 2013 IfA University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pxchip.h" +#include "pxspace.h" + +#include "releasetool.h" + +static bool definesurveyMode(pxConfig *config); +static bool listsurveyMode(pxConfig *config); +static bool definereleaseMode(pxConfig *config); +static bool updatereleaseMode(pxConfig *config); +static bool listreleaseMode(pxConfig *config); +static bool definerelexpMode(pxConfig *config); +static bool updaterelexpMode(pxConfig *config); +static bool listrelexpMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = releasetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(RELEASETOOL_MODE_DEFINESURVEY, definesurveyMode); + MODECASE(RELEASETOOL_MODE_LISTSURVEY, listsurveyMode); + + MODECASE(RELEASETOOL_MODE_DEFINERELEASE, definereleaseMode); + MODECASE(RELEASETOOL_MODE_UPDATERELEASE, updatereleaseMode); + MODECASE(RELEASETOOL_MODE_LISTRELEASE, listreleaseMode); + + MODECASE(RELEASETOOL_MODE_DEFINERELEXP, definerelexpMode); + MODECASE(RELEASETOOL_MODE_UPDATERELEXP, updaterelexpMode); + MODECASE(RELEASETOOL_MODE_LISTRELEXP, listrelexpMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool definesurveyMode(pxConfig *config) +{ + psError(PS_ERR_UNKNOWN, true, "not yet implemented"); + return false; +} + +static bool listsurveyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "surveyName", "LIKE"); + + psString query = pxDataGet("releasetool_listsurvey.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + // carry on The list of surveys is short there is no need to require parameters + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "survey", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definereleaseMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S32(surveyID, config->args, "-set_surveyID", true, false); + PXOPT_LOOKUP_STR(releaseName, config->args, "-set_releaseName", true, false); + PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state", true, false); + PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false); + PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", false, false); + + if (!ippReleaseInsert(config->dbh, + 0, // rel_id (auto_increment) + surveyID, + releaseName, + release_state, + dataRelease, + priority + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert ippRelease"); + return false; + } + + return true; +} + +static bool updatereleaseMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(releaseName, config->args, "-releaseName", false, false); + PXOPT_LOOKUP_S32(rel_id, config->args, "-rel_id", false, false); + + if (!releaseName && !rel_id) { + psError(PXTOOLS_ERR_CONFIG, true, "either -releaseName or -rel_id is required\n"); + return false; + } + PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S32(config->args, where, "-rel_id", "rel_id", "=="); + PXOPT_COPY_STR(config->args, where, "-releaseName", "releaseName", "=="); + + psString query = NULL; + psStringAppend(&query, "UPDATE ippRelease SET release_state = '%s'", release_state); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected); + + + psFree(query); + + return true; +} + +static bool listreleaseMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "surveyName", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-releaseName", "releaseName", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-release_state", "release_state", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "rel_id", "=="); + + psString query = pxDataGet("releasetool_listrelease.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + // The list of releases is short there is no need to require parameters + #ifdef notyet + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + #endif + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "ippRelease", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; + } + +static bool definerelexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "camRun.data_group", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + + // Insure that at least one of the camRun selectors is supplied + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "at least one of -label, -data_group, or -cam_id is required"); + return false; + } + + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + // insure that at least one of these is supplied to select the release + PXOPT_LOOKUP_S64(rel_id, config->args, "-rel_id", false, false); + PXOPT_LOOKUP_STR(releaseName, config->args, "-releaseName", false, false); + if (!rel_id && !releaseName) { + psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -releaseName is required"); + return false; + } + + // limit query to the target release. (Note: this will select nothing of releaseName and rel_id don't match) + // note: we add these after above to insure that search args are supplied + PXOPT_COPY_STR(config->args, where, "-releaseName", "ippRelease.releaseName", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "=="); + + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_U32(flags, config->args, "-set_flags", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the parameters of all the exposures that we want to add to the release + psString query = pxDataGet("releasetool_definerelexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + // loop over our list of exposures + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status = true; + psS32 rel_id = psMetadataLookupS32(&status, md, "rel_id"); + psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id"); + psS64 chip_id = psMetadataLookupS64(&status, md, "chip_id"); + psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); + psF32 zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs"); + psF32 zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev"); + + if (!relExpInsert(config->dbh, + 0, // relexp_id (auto increment) + rel_id, + exp_id, + chip_id, + cam_id, + state, + flags, + zpt_obs, + zpt_stdev, + path_base, + now, // registered + now // time_stamp + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to insert relExp for cam_id %" PRId64, cam_id); + psFree(output); + psFree(now); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updaterelexpMode(pxConfig *config) +{ + psError(PS_ERR_UNKNOWN, true, "not yet implemented"); + return false; +} + +static bool listrelexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-releaseName", "ippRelease.releaseName", "LIKE"); + pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "relExp.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-fwhm_min", "camProcessedExp.fwhm_major", ">="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "camProcessedExp.fwhm_major", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "relExp.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + + PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("releasetool_listrelexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString where2 = NULL; + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, "\nWHERE "); + } else { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + if (where2) { + psStringAppend(&query, "\n%s", where2); + psFree(where2); + } + + if (priority_order) { + psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority"); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + Index: branches/eam_branches/ipp-20121219/ippTools/src/releasetool.h =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/releasetool.h (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/src/releasetool.h (revision 35096) @@ -0,0 +1,39 @@ +/* + * releasetool.h + * + * Copyright (C) 2013 University of Hawaii IFA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef RELEASETOOL_H +#define RELEASETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + RELEASETOOL_MODE_NONE = 0x0, + RELEASETOOL_MODE_DEFINESURVEY, + RELEASETOOL_MODE_LISTSURVEY, + RELEASETOOL_MODE_DEFINERELEASE, + RELEASETOOL_MODE_UPDATERELEASE, + RELEASETOOL_MODE_LISTRELEASE, + RELEASETOOL_MODE_DEFINERELEXP, + RELEASETOOL_MODE_UPDATERELEXP, + RELEASETOOL_MODE_LISTRELEXP, +} releasetoolMode; + +pxConfig *releasetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // RELEASETOOL_H Index: branches/eam_branches/ipp-20121219/ippTools/src/releasetoolConfig.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/releasetoolConfig.c (revision 35096) +++ branches/eam_branches/ipp-20121219/ippTools/src/releasetoolConfig.c (revision 35096) @@ -0,0 +1,189 @@ +/* + * releasetoolConfig.c + * + * Copyright (C) 2013 IfA University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "pxspace.h" +#include "releasetool.h" + +pxConfig *releasetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definesurvey + psMetadata *definesurveyArgs = psMetadataAlloc(); + psMetadataAddS32(definesurveyArgs, PS_LIST_TAIL, "-set_surveyID", 0, "define survey ID (required)", 0); + psMetadataAddStr(definesurveyArgs, PS_LIST_TAIL, "-set_surveyName", 0, "define survey name (required)", NULL); + psMetadataAddStr(definesurveyArgs, PS_LIST_TAIL, "-set_description", 0, "define survey description", NULL); + + // -listsurvey + psMetadata *listsurveyArgs = psMetadataAlloc(); + psMetadataAddStr(listsurveyArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); + psMetadataAddU64(listsurveyArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listsurveyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerelease + psMetadata *definereleaseArgs = psMetadataAlloc(); + psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_surveyID", 0, "define survey ID (required)", 0); + psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_releaseName", 0, "define release name (required)", NULL); + psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0, "define dataRelease", -1); + psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define release state", NULL); + psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define release priority", -1); + + // -uptdaterelease + psMetadata *updatereleaseArgs = psMetadataAlloc(); + + // releaseName and/or rel_id is required. This is handled in updatereleaseMode + psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name", NULL); + psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + + // Only support changing state and priority for now + psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define new release state(required)", NULL); + psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define new release priority", 0); + +// psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_surveyID", 0, "define surveyID (required)", 0); +// psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_releaseName", 0, "define release name (required)", NULL); +// psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0, "define dataRelease", -1); + + + // -listrelease + psMetadata *listreleaseArgs = psMetadataAlloc(); + psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); + psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-release_state", 0, "select by release state", NULL); + psMetadataAddS32(listreleaseArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + + psMetadataAddU64(listreleaseArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listreleaseArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerelexp + psMetadata *definerelexpArgs = psMetadataAlloc(); + + // set the target release + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-releaseName", 0, "define release name", NULL); + psMetadataAddS32(definerelexpArgs, PS_LIST_TAIL, "-rel_id", 0, "define release ID", 0); + + // select the processing + psMetadataAddS64(definerelexpArgs, PS_LIST_TAIL, "-cam_id", 0, "select by cam ID", 0); + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-label", 0, "select by camRun.label", NULL); + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-data_group", 0, "select by camRun.data_group", NULL); + + // select a specific exposure from the "processing" + psMetadataAddS64(definerelexpArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID", 0); + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-exp_name", 0, "select by exposure name", NULL); + + // parameters of the relExp + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-set_path_base", 0, "define state", NULL); + psMetadataAddU32(definerelexpArgs, PS_LIST_TAIL, "-set_flags", 0, "define flags", 0); + + psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-pretend", 0, "do not actual modify the database", false); + psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simpel output format", false); + + + + // -updaterelexp + psMetadata *updaterelexpArgs = psMetadataAlloc(); + psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL); + + // -listrelexp + psMetadata *listrelexpArgs = psMetadataAlloc(); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-state", 0, "select by released exposure state", NULL); + + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-filter", 0, "select by exposure name (LIKE comparison)", NULL); + psMetadataAddTime(listrelexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0,"search for exposures by time (>=)", NULL); + psMetadataAddTime(listrelexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + + pxspaceAddArguments(listrelexpArgs); + + psMetadataAddF32(listrelexpArgs, PS_LIST_TAIL, "-fwhm_min", 0, "search by measured seeing (>=)", NAN); + psMetadataAddF32(listrelexpArgs, PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN); + + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-exp_name", 0, "select by exposure name", NULL); + + psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID", 0); + psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL, "-chip_id", 0, "select by chip ID", 0); + psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL, "-cam_id", 0, "select by cam ID", 0); + psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL, "-warp_id", 0, "select by warp ID", 0); + + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-chip_data_group", 0, "chipRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-cam_data_group", 0, "camRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-warp_data_group", 0, "warpRun.data_group (LIKE comparison)", NULL); + + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); + psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + + psMetadataAddBool(listrelexpArgs, PS_LIST_TAIL, "-priority_order", 0, "order by release priority", false); + + psMetadataAddU64(listrelexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrelexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definesurvey", "define a survey", RELEASETOOL_MODE_DEFINESURVEY, definesurveyArgs); + PXOPT_ADD_MODE("-listsurvey", "list skycell parameters", RELEASETOOL_MODE_LISTSURVEY, listsurveyArgs); + PXOPT_ADD_MODE("-definerelease", "define a release", RELEASETOOL_MODE_DEFINERELEASE, definereleaseArgs); + PXOPT_ADD_MODE("-updaterelease", "update a release", RELEASETOOL_MODE_UPDATERELEASE, updatereleaseArgs); + PXOPT_ADD_MODE("-listrelease", "list releases", RELEASETOOL_MODE_LISTRELEASE, listreleaseArgs); + PXOPT_ADD_MODE("-definerelexp", "define a released exposure", RELEASETOOL_MODE_DEFINERELEXP, definerelexpArgs); + PXOPT_ADD_MODE("-updaterelexp", "update a released exposure", RELEASETOOL_MODE_UPDATERELEXP, updaterelexpArgs); + PXOPT_ADD_MODE("-listrelexp", "list released exposures", RELEASETOOL_MODE_LISTRELEXP, listrelexpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/ipp-20121219/ippTools/src/staticskytool.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/staticskytool.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/src/staticskytool.c (revision 35096) @@ -503,4 +503,5 @@ PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "=="); pxAddLabelSearchArgs (config, whereMD, "-label", "staticskyRun.label", "=="); + pxskycellAddWhere(config, whereMD); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -736,4 +737,9 @@ PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); @@ -1066,16 +1072,20 @@ psFree(whereMD); + psString labelHook = psStringCopy(""); if (!rerun) { if (label) { - psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label); + psStringAppend(&labelHook, "\nAND skycalRun.label = '%s'", label); } psStringAppend(&query, "\nAND skycal_id IS NULL"); } - if (!p_psDBRunQuery(config->dbh, query)) { + + if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(labelHook); psFree(query); return false; } + psFree(labelHook); psFree(query); @@ -1392,5 +1402,11 @@ PXOPT_COPY_STR(config->args, where, "-label", "skycalRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-quality", "skycalResult.quality", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "skycalResult.fault", "=="); + + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1468,4 +1484,8 @@ pxAddLabelSearchArgs(config, where, "-filter", "stackRun.filter", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "skycalResult.fault", "=="); + if (!pxskycellAddWhere(config, where)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { Index: branches/eam_branches/ipp-20121219/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/eam_branches/ipp-20121219/ippTools/src/staticskytoolConfig.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ippTools/src/staticskytoolConfig.c (revision 35096) @@ -51,5 +51,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); @@ -103,4 +103,5 @@ psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + pxskycellAddArguments(todoArgs); psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -138,4 +139,5 @@ psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(revertArgs); psMetadataAddBool(revertArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); @@ -162,5 +164,5 @@ psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); - psMetadataAddF32(defineskycalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddF32(defineskycalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", NAN); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); @@ -238,4 +240,5 @@ psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0); psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0); + pxskycellAddArguments(revertskycalArgs); psMetadataAddS16(revertskycalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddBool(revertskycalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); @@ -257,5 +260,7 @@ psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-quality", 0, "search by quality", 0); psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxskycellAddArguments(skycalresultArgs); psMetadataAddU64(skycalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit skycalresult set to N items", 0); psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: branches/eam_branches/ipp-20121219/ippconfig/gpc1/psastro.config =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/gpc1/psastro.config (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/gpc1/psastro.config (revision 35096) @@ -382,4 +382,5 @@ PSASTRO.USE.MODEL BOOL FALSE PSASTRO.MOSAIC.MODE BOOL FALSE + DVO.GETSTAR.FIXED.ZEROPT F32 25.0 PSASTRO.CATDIR STR PS1.REF.20120524 @@ -394,25 +395,98 @@ END -# initial setup for MD@100 inputs, should work for 50-500 input +STATICSKY_CAL METADATA + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + DVO.GETSTAR.FIXED.ZEROPT F32 25.0 + + PSASTRO.CATDIR STR PS1.REF.20120524 + ZERO.POINT.USE.MEAN BOOL TRUE + + ## for skycell, radius shouldn't need to be as large as 90 pixel default. deeper stacks, crowded fields can be problems + PSASTRO.MATCH.RADIUS.N0 F32 15 + PSASTRO.MATCH.RADIUS.N1 F32 10 + PSASTRO.MATCH.RADIUS.N2 F32 5 + PSASTRO.MATCH.RADIUS.N3 F32 1 + PSASTRO.MATCH.RADIUS.N4 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 5 + + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, if larger than 0.2" on skycell then a problem.. + +END + + +# revised setup for MD nightly stack inputs, should work for case of N~2-30 +# - similar to DEEPCAL except for MIN.NSTAR 5 to attemp more edge skycells if poorer +STATICSKY_NIGHTCAL METADATA + PSASTRO.CATDIR STR PS1.REF.20120524 # reset here to ensure same catalog is used when reprocessing set uniformly + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels + PSASTRO.MAX.NRAW S32 1000 # include more than MAX.NREF to account for extras like false-positives + PSASTRO.MAX.NREF S32 500 # if edge skycell ~5% then want >50 to match (hit ~40% in normal field matches) + PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # + PSASTRO.MAX.INST.MAG.RAW F32 -5.0 + PSASTRO.GRID.SEARCH BOOL FALSE # skycell/stack and astrom well known, when used could cause problems and not help + #PSASTRO.GRID.NRAW.MAX S32 2000 + #PSASTRO.GRID.NREF.MAX S32 2000 + #PSASTRO.GRID.MIN.INST.MAG.RAW F32 -21.0 + #PSASTRO.GRID.MAX.INST.MAG.RAW F32 -14.0 + DVO.GETSTAR.MAX.RHO F32 6000.0 # slightly larger than MAX.NREF + DVO.GETSTAR.MIN.MAG.INST F32 -25.0 # set absurd range to cover large range of exptime, then clamp... + DVO.GETSTAR.MIN.MAG F32 15.0 # only used if keyword problem + REFSTAR_CLAMP_MAG_MIN F32 15.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list + DVO.GETSTAR.FIXED.ZEROPT F32 0.0 # not really needed, exptime overly dominates + + ## for skycell/stack astrom well known, radius shouldn't need to be as large as 90 pixel default. in deeper stacks can be problems + PSASTRO.MATCH.RADIUS.N0 F32 5 + PSASTRO.MATCH.RADIUS.N1 F32 3 + PSASTRO.MATCH.RADIUS.N2 F32 1 + PSASTRO.MATCH.RADIUS.N3 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 4 + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, still liberal limit for well known astrometry for skycells + PSASTRO.MIN.NSTAR S32 5 # edges may not have many and require special treatment +END + + +# revised setup for DEEP/MDS stacks with range of inputs, should work for N~2(edges)--500 inputs with large MIN/MAX.INST.MAG.RAW STATICSKY_DEEPCAL METADATA - REFSTAR_MASK BOOL FALSE - PSASTRO.SAVE.REFMATCH BOOL FALSE - PSASTRO.FIX.CHIPS BOOL FALSE - PSASTRO.USE.MODEL BOOL FALSE - PSASTRO.MOSAIC.MODE BOOL FALSE - - PSASTRO.CATDIR STR PS1.REF.20120524 - ZERO.POINT.USE.MEAN BOOL TRUE - PSASTRO.MAX.NRAW S32 0 - PSASTRO.MAX.NREF S32 10000 - PSASTRO.GRID.NRAW.MAX S32 2000 - PSASTRO.GRID.NREF.MAX S32 2000 - PSASTRO.MIN.INST.MAG.RAW F32 -19.0 - PSASTRO.MAX.INST.MAG.RAW F32 -14.0 - PSASTRO.GRID.MIN.INST.MAG.RAW F32 -19.0 - PSASTRO.GRID.MAX.INST.MAG.RAW F32 -15.0 - DVO.GETSTAR.MIN.MAG.INST F32 -19.0 - DVO.GETSTAR.MIN.MAG F32 16.0 - -END - + PSASTRO.CATDIR STR PS1.REF.20120524 # reset here to ensure same catalog is used when reprocessing set uniformly + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE + ZERO.POINT.USE.MEAN BOOL TRUE + PSASTRO.FIELD.PADDING F32 0.01 # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels + PSASTRO.MAX.NRAW S32 1000 # include more than MAX.NREF to account for extras like false-positives + PSASTRO.MAX.NREF S32 500 # if edge skycell ~10% then try ~50 to match + PSASTRO.MIN.INST.MAG.RAW F32 -25.0 # + PSASTRO.MAX.INST.MAG.RAW F32 -5.0 + PSASTRO.GRID.SEARCH BOOL FALSE # skycell/stack and astrom well known, when used could cause problems and not help + #PSASTRO.GRID.NRAW.MAX S32 2000 + #PSASTRO.GRID.NREF.MAX S32 2000 + #PSASTRO.GRID.MIN.INST.MAG.RAW F32 -21.0 + #PSASTRO.GRID.MAX.INST.MAG.RAW F32 -14.0 + DVO.GETSTAR.MAX.RHO F32 6000.0 # N/sqdeg, slightly larger than MAX.NREF + DVO.GETSTAR.MIN.MAG.INST F32 -25.0 # set absurd range to cover large range of exptime, then clamp... + DVO.GETSTAR.MIN.MAG F32 15.0 # only used if keyword problem so also using CLAMP_MAG_MIN + REFSTAR_CLAMP_MAG_MIN F32 15.0 # saturation in MD ~14 even in y-band, so toss out brighter in the list + DVO.GETSTAR.FIXED.ZEROPT F32 0.0 # not really needed, exptime overly dominates + + ## for skycell/stack astrom well known, radius shouldn't need to be as large as 90 pixel default. in deeper stacks can be problems + PSASTRO.MATCH.RADIUS.N0 F32 5 + PSASTRO.MATCH.RADIUS.N1 F32 3 + PSASTRO.MATCH.RADIUS.N2 F32 1 + PSASTRO.MATCH.RADIUS.N3 F32 0.5 + PSASTRO.MATCH.FIT.NITER S32 4 + PSASTRO.MAX.ERROR F32 0.2 # in arcsec units, still liberal limit for well known astrometry for skycells + PSASTRO.MIN.NSTAR S32 10 # edges may not have many and require special treatment +END + Index: branches/eam_branches/ipp-20121219/ippconfig/gpc1/psphot.config =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/gpc1/psphot.config (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/gpc1/psphot.config (revision 35096) @@ -53,4 +53,7 @@ PSPHOT.EXT.DIFF.ALTERNATE BOOL TRUE PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.25 + + PSPHOT.EXT.FIT.ALL.SOURCES BOOL TRUE + PSPHOT.EXT.FIT.ALL.THRESH F32 1000 END @@ -59,4 +62,7 @@ PSPHOT.EXT.DIFF.ALTERNATE BOOL TRUE PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.25 + + PSPHOT.EXT.FIT.ALL.SOURCES BOOL TRUE + PSPHOT.EXT.FIT.ALL.THRESH F32 1000 END Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-mef.mdc =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-mef.mdc (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-mef.mdc (revision 35096) @@ -214,5 +214,5 @@ PPIMAGE.PATTERN OUTPUT {OUTPUT}.ptn PATTERN NONE CHIP TRUE MEF PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE MEF - +PPIMAGE.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE NONE CHIP TRUE NONE ## note: these use the same output naming convention since they are used for different ppMerge runs Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-simple.mdc =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-simple.mdc (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-simple.mdc (revision 35096) @@ -178,5 +178,5 @@ PPIMAGE.PATTERN OUTPUT {OUTPUT}.ptn PATTERN NONE FPA TRUE NONE PPIMAGE.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE - +PPIMAGE.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE NONE CHIP TRUE NONE ## note: these use the same output naming convention since they are used for different ppMerge runs Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-split.mdc =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-split.mdc (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-split.mdc (revision 35096) @@ -198,4 +198,5 @@ PPIMAGE.PATTERN OUTPUT {OUTPUT}.{CHIP.NAME}.ptn PATTERN NONE CHIP TRUE NONE PPIMAGE.STATS OUTPUT {OUTPUT}.{CHIP.NAME}.stats STATS NONE CHIP TRUE NONE +PPIMAGE.BACKMDL OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits IMAGE NONE CHIP TRUE NONE ## note: these use the same output naming convention since they are used for different ppMerge runs Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/ppImage.config =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/ppImage.config (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/ppImage.config (revision 35096) @@ -42,5 +42,5 @@ MASK.STATS BOOL FALSE # calculate mask statistics. GAIN.OVERRIDE BOOL TRUE # Override a non-finite gain? - +BACKGROUND.CONTINUITY BOOL FALSE # Apply continuity correction to background models for an FPA. TILTYSTREAK.BY.CLASS METADATA # apply the 'tiltystreak' tool END @@ -1147,4 +1147,38 @@ MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels +END + +# Calculate new background models with continuity correction applied +PPIMAGE_BKGCONT METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + MASK.BUILD BOOL FALSE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PHOTOM BOOL FALSE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BASE.FITS BOOL FALSE # Save base image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? + BIN1.FITS BOOL FALSE # Save 1st binned chip image? + BIN2.FITS BOOL FALSE # Save 2nd binned chip image? + BIN1.JPEG BOOL FALSE # Save 1st binned jpeg? + BIN2.JPEG BOOL FALSE # Save 2nd binned jpeg? + BIN1.XBIN S32 1 # Image is already binned + BIN1.YBIN S32 1 # Image is already binned + BIN2.XBIN S32 1 # Image is already binned + BIN2.YBIN S32 1 # Image is already binned + MASK.SATURATED BOOL FALSE # DO NOT Mask the saturated pixels + MASK.LOW BOOL FALSE # DO NOT Mask the saturated pixels + BACKGROUND.CONTINUITY BOOL TRUE + PATTERN.CONTINUITY.WIDTH S32 2 END Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/psastro.config =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/psastro.config (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/psastro.config (revision 35096) @@ -190,4 +190,8 @@ DVO.GETSTAR.MIN.MAG.INST F32 -15.0 +# if non-zero use as the zero point for computing magnitude limits for getstar +# by default use the ZEROPT in the PHOTCODE.DATA +DVO.GETSTAR.FIXED.ZEROPT F32 0.0 + ZERO.POINT.BY.EXPOSURE BOOL TRUE ZERO.POINT.USE.MEAN BOOL TRUE @@ -268,9 +272,4 @@ STATICSKY_CAL METADATA - REFSTAR_MASK BOOL FALSE - PSASTRO.SAVE.REFMATCH BOOL FALSE - PSASTRO.FIX.CHIPS BOOL FALSE - PSASTRO.USE.MODEL BOOL FALSE - PSASTRO.MOSAIC.MODE BOOL FALSE END @@ -278,4 +277,7 @@ END +STATICSKY_NIGHTCAL METADATA +END + STATICSKY_DEEPCAL METADATA END Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/psphot.config =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/psphot.config (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/psphot.config (revision 35096) @@ -347,4 +347,6 @@ PSPHOT.EXT.DIFF.ALTERNATE BOOL FALSE # use alternate extended source calculation; designed for diff trails PSPHOT.EXT.DIFF.ALTERNATE.THRESH F32 1.5 # threshold for alternate extended source calculations +PSPHOT.EXT.FIT.ALL.SOURCES BOOL FALSE # Apply extended object fits to all sources +PSPHOT.EXT.FIT.ALL.THRESH F32 1000 # Set density threshold above which to not fit all sources PSPHOT.CR.GROW S32 1 # Number of pixels to grow CR mask PSPHOT.CR.NSIGMA.SOFTEN F32 0.0025 # Softening parameter for weights Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/pswarp.config =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/pswarp.config (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/pswarp.config (revision 35096) @@ -34,5 +34,6 @@ # Background restoration BACKGROUND METADATA - SOURCES BOOL FALSE # Write source list for warped image? - PSF BOOL FALSE # Measure PSF for warped image? + SOURCES BOOL FALSE # Write source list for warped image? + PSF BOOL FALSE # Measure PSF for warped image? + BACKGROUND.MODEL BOOL FALSE # do not model background END Index: branches/eam_branches/ipp-20121219/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/ipp-20121219/ippconfig/recipes/reductionClasses.mdc (revision 35095) +++ branches/eam_branches/ipp-20121219/ippconfig/recipes/reductionClasses.mdc (revision 35096) @@ -518,4 +518,6 @@ STACK_PPSUB STR STACK STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STATICSKY_CALIBRATION STR STATICSKY_NIGHTCAL END Index: branches/eam_branches/ipp-20121219/ppImage/src/Makefile.am =================================================================== --- branches/eam_branches/ipp-20121219/ppImage/src/Makefile.am (revision 35095) +++ branches/eam_branches/ipp-20121219/ppImage/src/Makefile.am (revision 35096) @@ -44,4 +44,5 @@ ppImageRebinReadout.c \ ppImageMosaic.c \ + ppImageMosaicBackground.c \ ppImageMaskStats.c \ ppImagePhotom.c \ Index: branches/eam_branches/ipp-20121219/ppImage/src/ppImage.h =================================================================== --- branches/eam_branches/ipp-20121219/ppImage/src/ppImage.h (revision 35095) +++ branches/eam_branches/ipp-20121219/ppImage/src/ppImage.h (revision 35096) @@ -41,4 +41,5 @@ bool doPatternCell; // Cell pattern correction bool doPatternContinuity; // Cell continuity correction + bool doBackgroundContinuity; // Do mosaic continuity correction bool doFringe; // Fringe subtraction bool doPhotom; // Source identification and photometry @@ -50,6 +51,5 @@ bool checkNoise; // measure cell-level variance bool applyParity; // Apply Cell parities - - bool doMaskStats; // Calculate mask statistics + bool doMaskStats; // Calculate mask statistics bool doCrosstalkMeasure; // measure crosstalk signal @@ -183,4 +183,7 @@ bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options); +// Do background continuity step +bool ppImageMosaicBackground(pmConfig *config, const ppImageOptions *options); + // Record which detrend file was used for the detrending bool ppImageDetrendRecord( Index: branches/eam_branches/ipp-20121219/ppImage/src/ppImageLoop.c =================================================================== --- branches/eam_branches/ipp-20121219/ppImage/src/ppImageLoop.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ppImage/src/ppImageLoop.c (revision 35096) @@ -41,5 +41,5 @@ ESCAPE("load failure for FPA"); } - + pmChip *chip; // Chip from FPA while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { @@ -233,5 +233,8 @@ ESCAPE("Unable to bin chip (level 2)."); } - + if (options->doBackgroundContinuity) { + pmFPAfile *out = psMetadataLookupPtr(NULL, config->files, "PPIMAGE.BACKMDL"); + pmFPAfileCopyView(out->fpa,out->src,view); + } // Close cells (XXX shouldn't pmFPAfileClose iterate down as needed?) view->cell = -1; @@ -251,4 +254,9 @@ } + // Do background model continuity updates + if (!ppImageMosaicBackground(config, options )) { + ESCAPE("failure in Background Mosaic"); + } + // generate the full-scale FPA mosaic if (!ppImageMosaicFPA(config, options, "PPIMAGE.OUTPUT.FPA1", "PPIMAGE.BIN1")) { Index: branches/eam_branches/ipp-20121219/ppImage/src/ppImageMosaicBackground.c =================================================================== --- branches/eam_branches/ipp-20121219/ppImage/src/ppImageMosaicBackground.c (revision 35096) +++ branches/eam_branches/ipp-20121219/ppImage/src/ppImageMosaicBackground.c (revision 35096) @@ -0,0 +1,77 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + + +bool ppImageMosaicBackground(pmConfig *config, const ppImageOptions *options) { + assert(config); + assert(options); + + if (options->doBackgroundContinuity) { + bool status; + pmFPAfile *in = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT"); + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + pmFPAfile *out = psMetadataLookupPtr(&status, config->files, "PPIMAGE.BACKMDL"); + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + //pmReadout *model = pmFPAGenerateReadout(config, view, psphotGetFilerule("PSPHOT.BACKMDL"), inFPA, binning, index); + + pmFPAview *view = pmFPAviewAlloc(0); + // pmFPAAddSourceFromView(out->fpa, view, out->format); + // psFree(view); + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + // ESCAPE("load failure for Chip"); + } + + pmChip *chip; + + if (!pmPatternContinuityBackground(in,out,options->patternCellBG,options->patternCellMean, + options->maskValue, options->darkMask, options->patternContinuityEdgeWidth)) { + // Free things? + psFree(view); + return(false); + } + + // Write out output models + psFree(view); + view = pmFPAviewAlloc(0); + while ((chip = pmFPAviewNextChip(view, out->fpa, 1)) != NULL) { + pmCell *cell; + while ((cell = pmFPAviewNextCell(view, out->fpa, 1)) != NULL) { + pmReadout *readout; + while ((readout = pmFPAviewNextReadout(view, out->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + psFree(view); + } + + return(true); +} Index: branches/eam_branches/ipp-20121219/ppImage/src/ppImageOptions.c =================================================================== --- branches/eam_branches/ipp-20121219/ppImage/src/ppImageOptions.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ppImage/src/ppImageOptions.c (revision 35096) @@ -35,4 +35,5 @@ options->doPatternCell = false; // Cell pattern correction options->doPatternContinuity = false; // Cell continuity correction + options->doBackgroundContinuity = false; // Chip level background continuity correction options->doFringe = false; // Fringe subtraction options->doPhotom = false; // Source identification and photometry @@ -429,4 +430,7 @@ } + // Option to enable the background continuity + options->doBackgroundContinuity = psMetadataLookupBool(NULL, recipe, "BACKGROUND.CONTINUITY"); + // Remnance options Index: branches/eam_branches/ipp-20121219/ppImage/src/ppImageParseCamera.c =================================================================== --- branches/eam_branches/ipp-20121219/ppImage/src/ppImageParseCamera.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ppImage/src/ppImageParseCamera.c (revision 35096) @@ -303,4 +303,5 @@ } + // chipImage -> psphotInput (pmFPAfileDefineFromFile) : fpa is constructed // psphotInput -> psphotOutput (pmFPAfileDefineOutputFromFile) : fpa is equated @@ -387,4 +388,25 @@ // the input data is the same as the outImage data : force the free levels to match input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel); + + // define the continuity corrected background model files + if (options->doBackgroundContinuity) { + pmFPAfile *bkgMosaicModel = pmFPAfileDefineFromFPA(config,input->fpa, 1, 1, "PPIMAGE.BACKMDL"); + if (!bkgMosaicModel) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BACKMDL")); + psFree(options); + return NULL; + } + if (bkgMosaicModel->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.BACKMDL is not of type IMAGE"); + psFree(options); + return NULL; + + } + bkgMosaicModel->save = options->doBackgroundContinuity; + // bkgMosaicModel->freeLevel = PS_MIN(bkgMosaicModel->freeLevel, PM_FPA_LEVEL_FPA); + bkgMosaicModel->freeLevel = PM_FPA_LEVEL_FPA; + bkgMosaicModel->dataLevel = bkgMosaicModel->dataLevel; + bkgMosaicModel->fileLevel = PS_MIN(bkgMosaicModel->fileLevel, bkgMosaicModel->dataLevel); + } // define the binned target files (which may just be carriers for some camera configurations) @@ -497,5 +519,5 @@ chipVariance->save = false; } - + if (psTraceGetLevel("ppImage.config") > 0) { // Get a look inside all the files. Index: branches/eam_branches/ipp-20121219/ppStack/src/ppStackCamera.c =================================================================== --- branches/eam_branches/ipp-20121219/ppStack/src/ppStackCamera.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ppStack/src/ppStackCamera.c (revision 35096) @@ -47,5 +47,6 @@ } bool convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images before stack? - + bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); + psArray *runImages = pmFPAfileDefineMultipleFromRun(&status, NULL, config, "PPSTACK.INPUT"); // Input images from previous run @@ -217,16 +218,25 @@ // Grab bkgmodel information here - if (!bkgmodel || strlen(bkgmodel) == 0) { - // We have no background models. + if (doBackground) { + if (!bkgmodel || strlen(bkgmodel) == 0) { + // We have no background models. + } + else { + pmFPAfile *inputBKG = defineFile(config,NULL,"PPSTACK.INPUT.BKGMODEL",bkgmodel, + PM_FPA_FILE_IMAGE); + if (!inputBKG) { + // We failed to generate an pmFPAfile, so disable the background construction and continue. + doBackground = false; + psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE, + "Do photometry on stacked image?", false); + +#if (0) + psError(psErrorCodeLast(), false, + "Unable to define file from bkgmodel %d (%s)",i,bkgmodel); + return(false); +#endif + } + }// End bkgmodel } - else { - pmFPAfile *inputBKG = defineFile(config,NULL,"PPSTACK.INPUT.BKGMODEL",bkgmodel, - PM_FPA_FILE_IMAGE); - if (!inputBKG) { - psError(psErrorCodeLast(), false, - "Unable to define file from bkgmodel %d (%s)",i,bkgmodel); - return(false); - } - }// End bkgmodel Index: branches/eam_branches/ipp-20121219/ppStack/src/ppStackCombineAlternate.c =================================================================== --- branches/eam_branches/ipp-20121219/ppStack/src/ppStackCombineAlternate.c (revision 35095) +++ branches/eam_branches/ipp-20121219/ppStack/src/ppStackCombineAlternate.c (revision 35096) @@ -198,5 +198,5 @@ offset = (Sy * Sxx - Sx * Sxy) / D; scale = (S * Sxy - Sx * Sy) / D; - fprintf(stderr,"Scales: %d %g %g\n",i,offset,scale); + fprintf(stderr,"Scales: %d %g %g %g %g %g %g %g %g\n",i,offset,scale,D,Sx,Sy,Sxx,Syy,Sxy); // Apply scaling factors for (int y = minInputRows; y < maxInputRows; y++) { Index: branches/eam_branches/ipp-20121219/psLib/src/imageops/psImageInterpolate.c =================================================================== --- branches/eam_branches/ipp-20121219/psLib/src/imageops/psImageInterpolate.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psLib/src/imageops/psImageInterpolate.c (revision 35096) @@ -31,5 +31,5 @@ #include "psImageConvolve.h" -# define IS_BILIN_SEPARABLE 1 +# define IS_BILIN_SEPARABLE 0 #include "psImageInterpolate.h" Index: branches/eam_branches/ipp-20121219/psModules/src/camera/pmFPAMaskWeight.c =================================================================== --- branches/eam_branches/ipp-20121219/psModules/src/camera/pmFPAMaskWeight.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psModules/src/camera/pmFPAMaskWeight.c (revision 35096) @@ -538,8 +538,9 @@ psF32 **imageData = readout->image->data.F32;// Dereference image psF32 **varianceData = readout->variance ? readout->variance->data.F32 : NULL; // Dereference variance map - + float maskFrac = 0.0; for (int y = 0; y < numRows; y++) { for (int x = 0; x < numCols; x++) { if (maskData[y][x] & maskVal) { + maskFrac += 1; imageData[y][x] = NAN; if (varianceData) { @@ -549,5 +550,7 @@ } } - + maskFrac = maskFrac / (1.0 * numRows * numCols); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "READOUT.MASK.FRAC", PS_META_REPLACE, + "fraction of pixels masked by pmReadoutMaskApply",maskFrac); return true; } Index: branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.c =================================================================== --- branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.c (revision 35096) @@ -938,3 +938,366 @@ } - +// This should reuse code more efficiently +bool pmPatternContinuityBackground(pmFPAfile *in, pmFPAfile *out, psStatsOptions bgStat, psStatsOptions cellStat, + psImageMaskType maskVal, psImageMaskType maskBad, int edgeWidth) +{ + PS_ASSERT_PTR_NON_NULL(in, false); + PS_ASSERT_PTR_NON_NULL(out, false); + + int numChips = out->fpa->chips->n; // Number of cells + + psVector *meanMask = psVectorAlloc(numChips, PS_TYPE_VECTOR_MASK); // Mask for means + psVectorInit(meanMask, 0); + + // Mask bits + enum { + PM_PATTERN_IGNORE = 0x01, // Ignore this cell + PM_PATTERN_TWEAK = 0x02, // Tweak this cell + PM_PATTERN_ERROR = 0x04, // Error in calculating background + PM_PATTERN_ALL = 0xFF, // All causes + }; + + // Measure mean of each cell edge, and use that to determine the cell offsets. + psStatsOptions stat = cellStat; // Define which statistic to use. + + psStats *bgStats = psStatsAlloc(stat); // Statistics on background + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + + psRegion region = {0,0,0,0}; + + /* These images hold the edge data for the OTA structure. */ + psImage *A = psImageAlloc(8,8,PS_TYPE_F64); // Top edge + psImage *B = psImageAlloc(8,8,PS_TYPE_F64); // Bottom edge + psImage *C = psImageAlloc(8,8,PS_TYPE_F64); // Right edge + psImage *D = psImageAlloc(8,8,PS_TYPE_F64); // Left edge + psImageInit(A,0.0); + psImageInit(B,0.0); + psImageInit(C,0.0); + psImageInit(D,0.0); + + // Corners don't exist. + A->data.F64[0][0] = NAN; + B->data.F64[0][0] = NAN; + C->data.F64[0][0] = NAN; + D->data.F64[0][0] = NAN; + A->data.F64[7][0] = NAN; + B->data.F64[7][0] = NAN; + C->data.F64[7][0] = NAN; + D->data.F64[7][0] = NAN; + A->data.F64[0][7] = NAN; + B->data.F64[0][7] = NAN; + C->data.F64[0][7] = NAN; + D->data.F64[0][7] = NAN; + A->data.F64[7][7] = NAN; + B->data.F64[7][7] = NAN; + C->data.F64[7][7] = NAN; + D->data.F64[7][7] = NAN; + + for (int i = 0; i < numChips; i++) { + pmChip *chip = out->fpa->chips->data[i]; + pmCell *cell = chip->cells->data[0]; // Cell of interest + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + + psStatsInit(bgStats); + + // Convert cell iterator i into an xy coordinate on the grid of cells + // This is wrong for chips + const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME"); + int xParity = psMetadataLookupS16(NULL,chip->concepts, "CHIP.XPARITY"); + int yParity = psMetadataLookupS16(NULL,chip->concepts, "CHIP.YPARITY"); + int x = chipName[2] - '0'; + int y = chipName[3] - '0'; + + + for (int j = 0; j < 4; j++) { + if (j == 0) { // Region B + region = psRegionSet(0,ro->image->numCols, + 0,edgeWidth); + } + else if (j == 1) { // Region A + region = psRegionSet(0,ro->image->numCols, + ro->image->numRows - edgeWidth,ro->image->numRows); + } + else if (j == 2) { // Region D + region = psRegionSet(0,edgeWidth, + 0,ro->image->numRows); + } + else if (j == 3) { // Region C + region = psRegionSet(ro->image->numCols - edgeWidth,ro->image->numCols, + 0,ro->image->numRows); + } + psImage *subset = psImageSubset(ro->image,region); + + if (!psImageBackground(bgStats, NULL, subset, NULL, maskVal, rng)) { + psWarning("Unable to measure background for cell %d on edge %d\n", i, j); + psErrorClear(); + meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PM_PATTERN_ERROR; + if (j == 0) { B->data.F64[y][x] = NAN; } + else if (j == 1) { A->data.F64[y][x] = NAN; } + else if (j == 2) { C->data.F64[y][x] = NAN; } + else if (j == 3) { D->data.F64[y][x] = NAN; } + psFree(subset); + continue; // Move on to next edge, as only part of this cell may be a problem + } + + // If the returned value is zero, assume something is wrong. Do I still need this? + if (psStatsGetValue(bgStats,stat) < 1e-6) { + if (j == 0) { B->data.F64[y][x] = NAN; } + else if (j == 1) { A->data.F64[y][x] = NAN; } + else if (j == 2) { C->data.F64[y][x] = NAN; } + else if (j == 3) { D->data.F64[y][x] = NAN; } + } + // If we have an error for this cell/edge, make sure we mask the value + if (meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PM_PATTERN_ERROR) { + if (j == 0) { B->data.F64[y][x] = NAN; } + else if (j == 1) { A->data.F64[y][x] = NAN; } + else if (j == 2) { C->data.F64[y][x] = NAN; } + else if (j == 3) { D->data.F64[y][x] = NAN; } + } + else { // Set the value to match what we got from the edge box. + if (xParity == -1) { + if (j == 2) { C->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + else if (j == 3) { D->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + } + if (xParity == 1) { + if (j == 3) { C->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + else if (j == 2) { D->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + } + if (yParity == 1) { + if (j == 0) { B->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + else if (j == 1) { A->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + } + if (yParity == -1) { + if (j == 1) { B->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + else if (j == 0) { A->data.F64[y][x] = psStatsGetValue(bgStats,stat); } + } + + } +#if (0) + for (int u = 0; u < subset->numCols; u++) { + for (int v = 0; v < subset->numRows; v++) { + psTrace("psModules.detrend.cont",10,"BOX: %d %d (%d %d) (%d %d) %f %d", + i,j,x,y,u,v,subset->data.F32[v][u],0); + } + } +#endif + psFree(subset); + + } + psTrace("psModules.detrend.cont",5, "OTA: %d (%d %d) (%d %d) A: %f B: %f C: %f D: %f", + i,x,y,xParity,yParity, + A->data.F64[y][x],B->data.F64[y][x],C->data.F64[y][x],D->data.F64[y][x]); + } + psFree(bgStats); + psFree(rng); + + // We've now allocated all the edge values, so we can now minimize the offsets. + // This involves solving the equation A x = b, where + // A is the (64x64 for GPC1) matrix containing the edges that match for each cell + // x is the solution vector + // b is the combination of offsets across each cell boundary for each cell. + // Below "XX" is used as the matrix A, and "solution" is used as both b and x + // (due to the way psMatrixLUSolve operates). + psVector *solution = psVectorAlloc(64,PS_TYPE_F64); + psImage *XX = psImageAlloc(64,64,PS_TYPE_F64); + psVectorInit(solution,0.0); + psImageInit(XX,0.0); + + + for (int i = 0; i < numChips; i++) { + // Accumulate all the possible edge differences we can for this cell. + // As we do so, make a note of the correlations by incrementing the element of the matrix. + pmChip *chip = out->fpa->chips->data[i]; + const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME"); + int x = chipName[2] - '0'; + int y = chipName[3] - '0'; + + int j; + int k = 8 * x + y; + double critical_value = 0.0; + + if (x + 1 < 8) { // We have a neighbor adjacent in the +x direction + j = 8 * (x + 1) + y; // Determine that neighbor's index + + if (fabs(C->data.F64[y][x]) > fabs(D->data.F64[y][x+1])) { + critical_value = 2.0 * fabs(D->data.F64[y][x+1]); + } + else { + critical_value = 2.0 * fabs(C->data.F64[y][x]); + } + if (critical_value < 25) { critical_value = 25; } + psTrace("psModules.detrend.cont",5,"CmD %d %d %d %d %g %g %g", // diagnostic + i,x,y,j, + C->data.F64[y][x], + D->data.F64[y][x+1], + critical_value + ); + if (// If there are no errors with the neighbor, + (isfinite(C->data.F64[y][x]))&&(isfinite(D->data.F64[y][x+1]))&& // and all edges have valid values, + (fabs(C->data.F64[y][x] - D->data.F64[y][x+1]) < critical_value) // and there are no large discontinuities, + ) { + solution->data.F64[k] += C->data.F64[y][x] - D->data.F64[y][x+1]; // Take the difference + XX->data.F64[k][k] += 1; // increment our relation with ourself + XX->data.F64[k][j] += -1; // decrement our relation with the neighbor + } + } + if (x - 1 > -1) { // etc. + j = 8 * (x - 1) + y; + if (fabs(C->data.F64[y][x-1]) > fabs(D->data.F64[y][x])) { + critical_value = 2.0 * fabs(D->data.F64[y][x]); + } + else { + critical_value = 2.0 * fabs(C->data.F64[y][x-1]); + } + if (critical_value < 25) { critical_value = 25; } + psTrace("psModules.detrend.cont",5,"DmC %d %d %d %d %g %g %g", + i,x,y,j, + D->data.F64[y][x], + C->data.F64[y][x-1], + critical_value + ); + + if ( + (isfinite(D->data.F64[y][x]))&&(isfinite(C->data.F64[y][x-1]))&& + (fabs(D->data.F64[y][x] - C->data.F64[y][x-1]) < critical_value) + ) { + solution->data.F64[k] += D->data.F64[y][x] - C->data.F64[y][x-1]; + XX->data.F64[k][k] += 1; + XX->data.F64[k][j] += -1; + } + } + if (y + 1 < 8) { + j = 8 * x + (y + 1); + psTrace("psModules.detrend.cont",5,"AmB %d %d %d %d %g %g", + i,x,y,j, + A->data.F64[y][x], + B->data.F64[y+1][x] + ); + if (fabs(A->data.F64[y][x]) > fabs(B->data.F64[y+1][x])) { + critical_value = 2.0 * fabs(B->data.F64[y+1][x]); + } + else { + critical_value = 2.0 * fabs(A->data.F64[y][x]); + } + if (critical_value < 25) { critical_value = 25; } + if ( + (isfinite(A->data.F64[y][x]))&&(isfinite(B->data.F64[y+1][x]))&& + (fabs(A->data.F64[y][x] - B->data.F64[y+1][x]) < critical_value) + ) { + solution->data.F64[k] += A->data.F64[y][x] - B->data.F64[y+1][x]; + XX->data.F64[k][k] += 1; + XX->data.F64[k][j] += -1; + } + } + if (y - 1 > -1) { + j = 8 * x + (y - 1); + psTrace("psModules.detrend.cont",5,"BmA %d %d %d %d %g %g", + i,x,y,j, + B->data.F64[y][x], + A->data.F64[y-1][x] + ); + if (fabs(A->data.F64[y-1][x]) > fabs(B->data.F64[y][x])) { + critical_value = 2.0 * fabs(B->data.F64[y][x]); + } + else { + critical_value = 2.0 * fabs(A->data.F64[y-1][x]); + } + if (critical_value < 25) { critical_value = 25; } + if ( + (isfinite(B->data.F64[y][x]))&&(isfinite(A->data.F64[y-1][x]))&& + (fabs(B->data.F64[y][x] - A->data.F64[y-1][x]) < critical_value) + ) { + solution->data.F64[k] += B->data.F64[y][x] - A->data.F64[y-1][x]; + XX->data.F64[k][k] += 1; + XX->data.F64[k][j] += -1; + } + } + } + double max_XX = 0; + double solution_V = 0; + int i_peak = -1; + for (int i = 0; i < numChips + 4; i++) { // If any cells have no value of themself, set the matrix to 1.0. + if (XX->data.F64[i][i] == 0.0) { + XX->data.F64[i][i] = 1.0; + } + if (XX->data.F64[i][i] > max_XX) { + max_XX = XX->data.F64[i][i]; + solution_V = solution->data.F64[i]; + i_peak = i; + } + } + psTrace("psModules.detrend.cont",5,"fixed point: %d %g\n", + i_peak,solution_V); + +#if (1) + for (int i = 0; i < numChips + 4; i++) { // print matrix A + psTrace("psModules.detrend.cont",5,"A: %3d % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f", + i, + XX->data.F64[i][0], XX->data.F64[i][1], XX->data.F64[i][2], XX->data.F64[i][3], + XX->data.F64[i][4], XX->data.F64[i][5], XX->data.F64[i][6], XX->data.F64[i][7], + XX->data.F64[i][8], XX->data.F64[i][9], XX->data.F64[i][10], XX->data.F64[i][11], + XX->data.F64[i][12], XX->data.F64[i][13], XX->data.F64[i][14], XX->data.F64[i][15], + XX->data.F64[i][16], XX->data.F64[i][17], XX->data.F64[i][18], XX->data.F64[i][19], + XX->data.F64[i][20], XX->data.F64[i][21], XX->data.F64[i][22], XX->data.F64[i][23], + XX->data.F64[i][24], XX->data.F64[i][25], XX->data.F64[i][26], XX->data.F64[i][27], + XX->data.F64[i][28], XX->data.F64[i][29], XX->data.F64[i][30], XX->data.F64[i][31], + XX->data.F64[i][32], XX->data.F64[i][33], XX->data.F64[i][34], XX->data.F64[i][35], + XX->data.F64[i][36], XX->data.F64[i][37], XX->data.F64[i][38], XX->data.F64[i][39], + XX->data.F64[i][40], XX->data.F64[i][41], XX->data.F64[i][42], XX->data.F64[i][43], + XX->data.F64[i][44], XX->data.F64[i][45], XX->data.F64[i][46], XX->data.F64[i][47], + XX->data.F64[i][48], XX->data.F64[i][49], XX->data.F64[i][50], XX->data.F64[i][51], + XX->data.F64[i][52], XX->data.F64[i][53], XX->data.F64[i][54], XX->data.F64[i][55], + XX->data.F64[i][56], XX->data.F64[i][57], XX->data.F64[i][58], XX->data.F64[i][59], + XX->data.F64[i][60], XX->data.F64[i][61], XX->data.F64[i][62], XX->data.F64[i][63] + ); + } + + for (int i = 0; i < numChips + 4; i++) { // print vector b + psTrace("psModules.detrend.cont",5,"b: %d %f", + i, + solution->data.F64[i] + ); + } +#endif + + // Solve the Ax=b equation + // psMatrixLUSolve(XX,solution); + psMatrixGJSolve(XX,solution); +#if (1) + for (int i = 0; i < numChips + 4; i++) { // print vector b + psTrace("psModules.detrend.cont",5,"x: %d %f", + i, + solution->data.F64[i] + ); + } +#endif + + // Cleanup + psFree(XX); + psFree(A); + psFree(B); + psFree(C); + psFree(D); + + // Correct cells based on the offsets calculated, and store the result in the analysis metadata. + for (int i = 0; i < numChips; i++) { + pmChip *chip = out->fpa->chips->data[i]; + pmCell *cell = chip->cells->data[0]; // Cell of interest + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME"); + int x = chipName[2] - '0'; + int y = chipName[3] - '0'; + int k = 8 * x + y; + float correction = solution->data.F64[k]; + psLogMsg("psModules.detrend", PS_LOG_DETAIL, "Correcting background of chip %s by %f", + chipName, correction); + psBinaryOp(ro->image, ro->image, "-", psScalarAlloc(correction, PS_TYPE_F32)); + psMetadataAddF32(ro->analysis, PS_LIST_TAIL, PM_PATTERN_CELL_CORRECTION, PS_META_REPLACE, + "Pattern chip correction solution", correction); + } + + psFree(solution); + psFree(meanMask); + + return true; +} Index: branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.h =================================================================== --- branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.h (revision 35095) +++ branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.h (revision 35096) @@ -14,4 +14,10 @@ #include #include +#include +#include +#include +#include +#include +#include /// @addtogroup detrend Detrend Creation and Application @@ -65,4 +71,12 @@ ); +bool pmPatternContinuityBackground(pmFPAfile *in, + pmFPAfile *out, + psStatsOptions bgStat, + psStatsOptions cellStat, + psImageMaskType maskVal, + psImageMaskType maskBad, + int edgeWidth); + /// Apply previously measured cell pattern correction bool pmPatternContinuityApply(pmReadout *ro, ///< Readout to correct Index: branches/eam_branches/ipp-20121219/psModules/src/imcombine/pmStack.c =================================================================== --- branches/eam_branches/ipp-20121219/psModules/src/imcombine/pmStack.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psModules/src/imcombine/pmStack.c (revision 35096) @@ -1385,4 +1385,5 @@ psVector *pixelData = psVectorAlloc(input->n,PS_TYPE_F32); + psVector *pixelMask = psVectorAlloc(input->n,PS_TYPE_VECTOR_MASK); psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); @@ -1392,10 +1393,25 @@ pmReadout *ro = stack->data[i]; psImage *image = ro->image; + pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0; pixelData->data.F32[i] = image->data.F32[y][x]; + if (isfinite(image->data.F32[y][x])&& + (fabs(image->data.F32[y][x]) < 1e5)) { + pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0; + } + else { + pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 1; + } +#if (0) + if ((x == 59)&&(y > 40)&&(y < 50)) { + fprintf(stderr,"%d %d %d %d %g\n", + x,y,i,pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i],pixelData->data.F32[i]); + } +#endif } - if (!psVectorStats(stats,pixelData,NULL,NULL,0)) { + if (!psVectorStats(stats,pixelData,NULL,pixelMask,1)) { psError(PS_ERR_UNKNOWN, false, "Unable to calculate median"); psFree(stats); psFree(pixelData); + psFree(pixelMask); psFree(stack); return(false); @@ -1403,4 +1419,11 @@ combined->image->data.F32[y][x] = stats->robustMedian; combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0; +#if (0) + if ((x == 59)&&(y > 40)&&(y < 50)) { + fprintf(stderr,"%d %d %d %d %g\n", + x,y,-1,combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x], + combined->image->data.F32[y][x]); + } +#endif } } @@ -1408,4 +1431,5 @@ psFree(stats); psFree(pixelData); + psFree(pixelMask); psFree(stack); return (true); Index: branches/eam_branches/ipp-20121219/psModules/src/objects/pmSourceIO_CMF.c.in =================================================================== --- branches/eam_branches/ipp-20121219/psModules/src/objects/pmSourceIO_CMF.c.in (revision 35095) +++ branches/eam_branches/ipp-20121219/psModules/src/objects/pmSourceIO_CMF.c.in (revision 35096) @@ -830,5 +830,5 @@ psMetadataAddF32 (row, PS_LIST_TAIL, "KRON_MAG", 0, "Kron Mag", kronMag); - @PS1_DV1,PS1_DV2@ psMetadataAddF32 (row, PS_LIST_TAIL, "NPARAMS", 0, "number of model parameters", model->params->n); + @PS1_DV0,PS1_DV1@ psMetadataAddF32 (row, PS_LIST_TAIL, "NPARAMS", 0, "number of model parameters", model->params->n); @ALL,!PS1_DV0,!PS1_DV1@ psMetadataAddS32 (row, PS_LIST_TAIL, "NPARAMS", 0, "number of model parameters", model->params->n); psMetadataAddStr (row, PS_LIST_TAIL, "MODEL_TYPE", 0, "name of model", pmModelClassGetName (model->type)); Index: branches/eam_branches/ipp-20121219/psastro/src/psastroChooseRefstars.c =================================================================== --- branches/eam_branches/ipp-20121219/psastro/src/psastroChooseRefstars.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psastro/src/psastroChooseRefstars.c (revision 35096) @@ -62,6 +62,6 @@ int nMax = psMetadataLookupS32 (&status, recipe, "PSASTRO.MAX.NREF"); - psF32 clampMagMin = psMetadataLookupS32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN"); - if (!status) clampMagMin = -5; + psF32 clampMagMin = psMetadataLookupF32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN"); + if (!status) clampMagMin = -5.0; // de-activate all files except PSASTRO.REFSTARS Index: branches/eam_branches/ipp-20121219/psastro/src/psastroLoadRefstars.c =================================================================== --- branches/eam_branches/ipp-20121219/psastro/src/psastroLoadRefstars.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psastro/src/psastroLoadRefstars.c (revision 35096) @@ -363,4 +363,7 @@ if (!status) ESCAPE ("missing DVO.GETSTAR.MIN.MAG.INST"); + // if non zero override the zero point in the PHOTCODE.DATA with this value + float fixedzeropt = psMetadataLookupF32(&status, recipe, "DVO.GETSTAR.FIXED.ZEROPT"); + // PHOTCODE.DATA is a multi of metadata items psListIterator *iter = psListIteratorAlloc(item->data.list, PS_LIST_HEAD, false); @@ -391,4 +394,8 @@ continue; } + if (fixedzeropt != 0.0) { + // override the recipe's zero point with the fixed value (used for stacks) + zeropt = fixedzeropt; + } // convert the minInst to a calibrated minimum magnitude Index: branches/eam_branches/ipp-20121219/psconfig/psbuild =================================================================== --- branches/eam_branches/ipp-20121219/psconfig/psbuild (revision 35095) +++ branches/eam_branches/ipp-20121219/psconfig/psbuild (revision 35096) @@ -426,13 +426,13 @@ print STDOUT "** if you use sh, bash or equivalent as your shell, add the following to your .bashrc\n"; - print STDOUT " if [ -e $psconfdir/psconfig.bash ] ; then"; - print STDOUT " psconfig () {"; - print STDOUT " source $psconfdir/psconfig.bash"; - print STDOUT " }"; - print STDOUT " else"; - print STDOUT " psconfig () {"; - print STDOUT " echo psconfig not available"; - print STDOUT " }"; - print STDOUT " fi"; + print STDOUT " if [ -e $psconfdir/psconfig.bash ] ; then\n"; + print STDOUT " psconfig () {\n"; + print STDOUT " source $psconfdir/psconfig.bash\n"; + print STDOUT " }\n"; + print STDOUT " else\n"; + print STDOUT " psconfig () {\n"; + print STDOUT " echo psconfig not available\n"; + print STDOUT " }\n"; + print STDOUT " fi\n"; print STDOUT " psconfig default\n"; print STDOUT "\n"; Index: branches/eam_branches/ipp-20121219/psphot/src/psphotSourceSize.c =================================================================== --- branches/eam_branches/ipp-20121219/psphot/src/psphotSourceSize.c (revision 35095) +++ branches/eam_branches/ipp-20121219/psphot/src/psphotSourceSize.c (revision 35096) @@ -13,4 +13,7 @@ bool altDiffExt; float altDiffExtThresh; + bool extFitAll; + bool extFitAllReadout; + float extFitAllThresh; float soft; int grow; @@ -127,4 +130,10 @@ // Threshold for this alternate method options.altDiffExtThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE.THRESH"); + assert (status); + // Option to enable fitting of all objects with extended model. + options.extFitAll = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.FIT.ALL.SOURCES"); + assert (status); + // Fitting everything is fine, but if the source density is high, we probably shouldn't. + options.extFitAllThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.FIT.ALL.THRESH"); assert (status); @@ -413,5 +422,18 @@ psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4s %4s %4s %4s %4s", "Npsf", "Next", "Nsat", "Ncr", "Nskip"); - + // Determine if this readout is above the threshold to ext fit all sources + if (options->extFitAll) { + float maskFrac = psMetadataLookupF32(&status,readout->analysis,"READOUT.MASK.FRAC"); + if (status) { + maskFrac = 0.0; + } + if (sources->n * (1.0 - maskFrac) > options->extFitAllThresh) { + options->extFitAllReadout = false; + } + else { + options->extFitAllReadout = true; + } + } + if (!psphotSourceClassRegion (NULL, &psfClump, sources, recipe, psf, options)) { psLogMsg ("psphot", 4, "Failed to determine source classification for full image\n"); @@ -420,5 +442,5 @@ } return true; - + int nRegions = psMetadataLookupS32 (&status, readout->analysis, "PSF.CLUMP.NREGIONS"); for (int i = 0; i < nRegions; i ++) { @@ -426,8 +448,8 @@ psMetadata *regionMD = psMetadataLookupPtr (&status, readout->analysis, regionName); psAssert (regionMD, "regions must be defined by earlier call to psphotRoughClassRegion"); - + psRegion *region = psMetadataLookupPtr (&status, regionMD, "REGION"); psAssert (region, "regions must be defined by earlier call to psphotRoughClassRegion"); - + // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); psAssert (status, "missing PSF.CLUMP.X"); @@ -559,4 +581,9 @@ // * SAT stars should not be faint, but defects may? + // If the recipe requests we do extended source fits to everything, set + // the EXT_LIMIT flag + if (options->extFitAllReadout) { + source->mode |= PM_SOURCE_MODE_EXT_LIMIT; + } // Defects may not always match CRs from peak curvature analysis // Defects may also be marked as SATSTAR -- XXX deactivate this flag? Index: branches/eam_branches/ipp-20121219/tools/queuessky.lap =================================================================== --- branches/eam_branches/ipp-20121219/tools/queuessky.lap (revision 35095) +++ branches/eam_branches/ipp-20121219/tools/queuessky.lap (revision 35096) @@ -18,5 +18,5 @@ -my ($ramin, $ramax, $decmin, $decmax, $absglatmin, $go); +my ($ramin, $ramax, $decmin, $decmax, $absglatmin, $glatmin, $glatmax, $go); my $combos = 5; my $verbose = 0; @@ -27,8 +27,10 @@ 'dec_min=s' => \$decmin, 'dec_max=s' => \$decmax, -# 'absglatmin=s' => \$absglatmin, # not yet - 'nfilters=s' => \$combos, - 'go' => \$go, - 'verbose|v' => \$verbose, + 'glat_min=s' => \$glatmin, + 'glat_max=s' => \$glatmax, + 'abs_glat_min=s' => \$absglatmin, + 'nfilters=s' => \$combos, + 'go' => \$go, + 'verbose|v' => \$verbose, ) or pod2usage(2); @@ -57,5 +59,8 @@ $command .= " -dec_max $decmax" if defined $decmax; -# XXX TODO: implement absglatmin +$command .= " -glat_min $glatmin" if defined $glatmin; +$command .= " -glat_max $glatmax" if defined $glatmax; + +$command .= " -abs_glat_min $absglatmin" if defined $absglatmin; foreach my $filter_combo (@$filter_combos) { @@ -72,6 +77,4 @@ } } - - exit 0;