IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Feb 6, 2013, 2:34:49 PM (13 years ago)
Author:
eugene
Message:

merging changes from trunk

Location:
branches/eam_branches/ipp-20121219
Files:
85 edited
9 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20121219

  • branches/eam_branches/ipp-20121219/Nebulous

  • branches/eam_branches/ipp-20121219/Nebulous-Server

  • branches/eam_branches/ipp-20121219/Nebulous-Server/bin/nebdiskd

    r28525 r35096  
    1111use Config::YAML;
    1212use DBI;
    13 use File::Mountpoint qw( is_mountpoint );
    1413use File::Spec;
    15 use Filesys::Df;
    1614use Log::Log4perl;
    1715use Nebulous::Server::SQL;
    … …  
    2220
    2321my (
    24     $debug, 
    25     $db, 
    26     $dbhost, 
    27     $dbuser, 
    28     $dbpass, 
    29     $pidfile, 
    30     $stop, 
    31     $restart, 
     22    $debug,
     23    $db,
     24    $dbhost,
     25    $dbuser,
     26    $dbpass,
     27    $pidfile,
     28    $stop,
     29    $restart,
    3230    $verbose,
    3331    $user,
    … …  
    111109    log4perl.appender.SERVERLOGFILE.layout    = Log::Log4perl::Layout::PatternLayout
    112110    log4perl.appender.SERVERLOGFILE.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
     111    log4perl.appender.SERVERLOGFILE.size      = 10485760
    113112
    114113    log4perl.filter.MatchWarn  = Log::Log4perl::Filter::LevelMatch
    … …  
    143142$SIG{TERM} = sub { unlink_pid_file($pidfile); exit(); };
    144143$SIG{INT}  = $SIG{TERM};
    145 $SIG{HUP}  = sub { $c = read_rcfile($rcfile) }; 
     144$SIG{HUP}  = sub { $c = read_rcfile($rcfile) };
    146145
    147146my $date = localtime();
    … …  
    192191    eval {
    193192        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ?,note FROM volume WHERE mountpoint = ?");
    194 #       my $d_query = $dbh->prepare_cached("UPDATE mountedvol SET allocate = ?, available = ? WHERE mountpoint = ?");
    195193        my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?");
    196194
    … …  
    199197        {
    200198            # there may be multiple vol_ids per mountpoint but we only need to
    201             # check each mointpont once
    202             my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume");
     199            # check each mountpoint once
     200            my $query = $dbh->prepare_cached("SELECT DISTINCT host, mountpoint FROM volume");
    203201            $query->execute;
    204202            while (my $row = $query->fetchrow_hashref) {
    … …  
    207205            $query->finish;
    208206        }
    209        
     207
    210208        # determine valid mountpoints
    211209        foreach my $mnt (@$mounts) {
    212210            my $mountpoint = $mnt->{'mountpoint'};
    213             $log->debug("checking $mountpoint");
    214             # this /SHOULD/ fail if the mount point is handled by the
    215             # automounter and it fails to mount
    216             my $tries = 0;
    217             my $valid_mountpoint = 1;
    218             unless (exists($host_failure_counts{$mountpoint})) {
    219                 $host_failure_counts{$mountpoint} = 0;
    220             }
    221             TEST: eval {
    222                 $tries++;
    223                 unless (is_mountpoint($mountpoint)) {
    224                     unless(defined($host_removed{$mountpoint})) {
    225                         $log->warn("$mountpoint is not a valid mountpoint ($tries $host_failure_counts{$mountpoint})");
    226                     }
    227                     $valid_mountpoint = 0;
    228                 }
    229             };
    230 
    231             # fetch stats on the mounted device.  this has to be done AFTER
    232             # we determine if it's a valid mountpoint incase
    233             # is_mountpoint() invokes the automounter
    234             my $dev_info = df($mountpoint, 1024);
     211            my $alt_mountpoint = $mountpoint;
     212            $alt_mountpoint =~ s/data/export/;
     213            my $hostname = $mnt->{'host'};
     214            my $valid_mountpoint = 1;
     215
     216            # fetch stats on the device using ssh and df
     217            my $dev_info = df_through_ssh($hostname, $alt_mountpoint, 1024);
    235218            unless (defined $dev_info) {
    236                 $valid_mountpoint = 0;
    237             }
    238 
    239             if (!$valid_mountpoint) {
    240                 # try is_mountpoint() again if $retry > 1
    241                 if ($tries < $retry) {
    242                     $log->warn("retrying test of $mountpoint");
    243                     goto TEST;
    244                 }
    245                 $host_failure_counts{$mountpoint}++;
    246                
    247                 if (!(defined($host_removed{$mountpoint})) || !($host_removed{$mountpoint})) {
    248                     if (($host_failure_counts{$mountpoint} > $failure_limit)) {
    249                         $host_removed{$mountpoint} = 1;
    250                         $log->warn("Removing $mountpoint from the mountedvol table ($host_failure_counts{$mountpoint} > $failure_limit) No further warnings unless state changes.");
    251                         $d_query->execute($mountpoint);
    252                     }
    253                     else {
    254                         $log->warn("Mountpoint $mountpoint has had $host_failure_counts{$mountpoint} failures. Will remove after $failure_limit");
    255                     }
    256                 }
    257                 next;
    258                
    259             }
    260             if ($host_failure_counts{$mountpoint} != 0) {
    261                 $host_removed{$mountpoint} = 0;
    262                 $log->warn("Mountpoint $mountpoint failures cleared ($host_failure_counts{$mountpoint})");
    263                 $host_failure_counts{$mountpoint} = 0;
    264             }
    265 
    266             # fetch stats on the mounted device.  this has to be done AFTER
    267             # we determine if it's a valid mountpoint incase
    268             # is_mountpoint() invokes the automounter
    269             $dev_info = df($mountpoint, 1024);
    270             unless (defined $dev_info) {
    271                 $log->error("can't find device info for $mountpoint");
     219                $valid_mountpoint = 0;
     220                $log->error("can't find device info for $mountpoint / $alt_mountpoint");
     221                $d_query->execute($mountpoint);
    272222                next;
    273223            }
    … …  
    278228
    279229        }
    280 
    281 #        $dbh->do("call getmountedvol()");
    282 
    283 #        $dbh->commit;
    284 #        $log->debug("commited to database");
    285230    };
    286231    if ($@) {
    287 #        $dbh->rollback;
    288 #        $log->debug("rolledback transaction");
    289232        $log->logdie($@);
    290233    }
    … …  
    336279#        $dbh->commit;
    337280    };
    338     if ($@) { 
     281    if ($@) {
    339282#        $dbh->rollback;
    340283        $log->logdie($@);
    … …  
    391334  # this follows BSD syntax ps (BSD's and linux)
    392335  # this will fail on Unix98 syntax ps (Solaris, etc)
    393   }elsif( `ps h o pid p $$` =~ /^\s*$$\s*$/ ){ # can I play ps on myself ?
    394     $exists = `ps h o pid p $current_pid`;
    395 
     336  } elsif( `ps h o pid p $$` =~ /^\s*$$\s*$/ ){ # can I play ps on myself ?
     337      $exists = `ps h o pid p $current_pid`;
    396338  }
    397339
    … …  
    412354}
    413355
     356sub df_through_ssh {
     357    my $hostname = shift;
     358    my $mountpoint = shift;
     359    my $units = shift;
     360    $log->debug("Getting info from mountpoint = [$mountpoint] / unit = [$units] / hostname = [$hostname]");
     361    my $df_data = qx/ssh $hostname 'df $mountpoint' | grep -v Filesystem/;
     362    $df_data =~ s/\s+/:/g;
     363    $log->debug("Received from ssh = [$df_data]");
     364    my %df_info = ();
     365    $df_info{"blocks"} = 0;
     366    $df_info{"used"} = 0;
     367    unless ($df_data) {
     368        # We cannot connect to the host for some reason However, the
     369        # partition can be mounted on some other hosts (in the case of
     370        # an original host failure or upgrade for instance) Let's look
     371        # for an entry in /etc/autofs/auto.data starting with
     372        # <hostname>.[012] If it's in there, return (blocks => 1, used
     373        # => 1). The partition is mounted but not available for
     374        # replication (since blocks-used = 0 and used/blocks=1=100%).
     375        my $partition_name = $mountpoint;
     376        $partition_name =~ s/\/export\///;
     377        $log->debug("Looking for $partition_name in /etc/autofs/auto.data");
     378        open(AUTODATA, "</etc/autofs/auto.data") or die("Can't open /etc/autofs/auto.data");
     379        foreach my $line (<AUTODATA>) {
     380            if ($line =~ /^$partition_name/) {
     381                $log->debug("$partition_name found in /etc/autofs/auto.data");
     382                close(AUTODATA);
     383                $df_info{"blocks"} = 1;
     384                $df_info{"used"} = 1;
     385                return \%df_info;
     386            }
     387        }
     388        $log->debug("$partition_name NOT found in /etc/autofs/auto.data");
     389        close(AUTODATA);
     390        return undef;
     391    }
     392    my @data = split(/:/, $df_data);
     393    $df_info{"blocks"} = $data[1];
     394    $df_info{"used"} = $data[2];
     395    $log->debug("blocks = " . $df_info{"blocks"} . " / used = " . $df_info{"used"});
     396    return \%df_info;
     397}
    414398
    415399__END__
    … …  
    525509=item * C<NEB_PASS>
    526510
    527 Equivalent to --dbpass|-p 
     511Equivalent to --dbpass|-p
    528512
    529513=back

  • branches/eam_branches/ipp-20121219/Nebulous/bin/neb-df

    r31413 r35096  
    5353    my $path        = $vol{host};
    5454    if ($names) {
    55         $path = $vol{name};
     55        $path = $vol{name};
    5656    }
    5757
    … …  
    6161    $total_used     += $used;
    6262    my $available   = $total - $used;
    63     my $usedper     = ($used / $total) * 100;
     63    my $usedper     = ($used / ($total+1)) * 100;
    6464    my $mountpoint  = $vol{mountpoint};
    6565
    6666    if ($vol{allocate} == 1) {
    67         $allocated_total += $total;
    68         $allocated_used += $used;
     67        $allocated_total += $total;
     68        $allocated_used += $used;
    6969    }
    7070    # indicated the part of the nebulous volume name that matches up with a
    71     # local volume (mountpoint) 
     71    # local volume (mountpoint)
    7272    #$path =~ s/$mountpoint/[$mountpoint]/;
    7373
    … …  
    9595
    9696my $total_available   = $total_total - $total_used;
    97 my $total_usedper     = ($total_used / $total_total) * 100; 
     97my $total_usedper     = ($total_used / $total_total) * 100;
    9898
    9999# summary
    … …  
    112112
    113113my $allocated_available   = $allocated_total - $allocated_used;
    114 my $allocated_usedper     = ($allocated_used / $allocated_total) * 100; 
     114my $allocated_usedper     = ($allocated_used / $allocated_total) * 100;
    115115
    116116# summary
    … …  
    188188=item * C<NEB_PASS>
    189189
    190 Equivalent to --pass|-p 
     190Equivalent to --pass|-p
    191191
    192192=back

  • branches/eam_branches/ipp-20121219/Ohana

  • branches/eam_branches/ipp-20121219/Ohana/src/getstar/src/select_by_region.c

    r34459 r35096  
    5353    gfits_scan (&catalog[0].header, "DEC1", "%lf", 1, &Dmax);
    5454
    55     if (VERBOSE) fprintf (stderr, "extracting from catalog covering region %f,%f to %f,%f\n", Rmin, Dmin, Rmax, Dmax);
    56 
    5755    AREA = fabs(Dmax - Dmin) * fabs(Rmax - Rmin) * cos (0.5*RAD_DEG*(Dmax + Dmin));
     56    if (VERBOSE) fprintf (stderr, "extracting from catalog covering region %f,%f to %f,%f for area %f\n", Rmin, Dmin, Rmax, Dmax, AREA);
    5857    assert (AREA > 0);
    5958
    … …  
    8382      if (isnan(mag)) continue;
    8483      if (mag < MinMagValue) continue;
    85       if (mag > MagLimitValue) continue;
    8684
    8785      bin = (mag - MagMin) / dMag;
    … …  
    10199    }
    102100    MagLimitValue = i*dMag + MagMin;
    103     if (VERBOSE) fprintf (stderr, "using %d stars in mag range %f - %f\n", Nsum, MinMagValue, MagLimitValue);
     101    if (VERBOSE) fprintf (stderr, "using %d (Nmax %d) stars in mag range %f - %f\n", Nsum, Nmax, MinMagValue, MagLimitValue);
    104102  }
    105103

  • branches/eam_branches/ipp-20121219/Ohana/src/libohana/src

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/Ohana/src/opihi

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/Ohana/src/opihi/cmd.astro

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/Ohana/src/opihi/cmd.data

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/dbconfig/changes.txt

    r34789 r35096  
    22652265ALTER TABLE skycalResult ADD column n_forced INT after n_extended;
    22662266UPDATE dbversion set schema_version = '1.1.73', updated= CURRENT_TIMESTAMP();
     2267
     2268
     2269-- new tables for managing the "release" of ipp data
     2270
     2271CREATE TABLE survey (
     2272    surveyID    INT(8),
     2273    surveyName  VARCHAR(16) NOT NULL,
     2274    description VARCHAR(255),
     2275    PRIMARY KEY(surveyID),
     2276    UNIQUE  KEY(surveyName)
     2277) ENGINE=InnoDB CHARSET=latin1;
     2278
     2279
     2280CREATE TABLE ippRelease (
     2281    rel_id      INT(8) AUTO_INCREMENT,
     2282    surveyID    INT(8),
     2283    releaseName VARCHAR(64),
     2284    release_state VARCHAR(16),    -- active, pending archive drop
     2285    dataRelease INT(8),         -- PSPS dataRelease
     2286    PRIMARY KEY(rel_id),
     2287    KEY(releaseName),
     2288    KEY(state),
     2289    FOREIGN KEY(surveyID) REFERENCES survey(surveyID)
     2290) ENGINE=Innodb DEFAULT CHARSET=latin1;
     2291
     2292CREATE TABLE relExp (
     2293    relexp_id   BIGINT AUTO_INCREMENT,
     2294    rel_id      INT(8),
     2295    exp_id      BIGINT,
     2296    chip_id     BIGINT,         -- links to the runs that supplied the data
     2297    cam_id      BIGINT,         -- for this release
     2298    state       VARCHAR(16),    -- released, pending, archive, drop 
     2299    flags       INT UNSIGNED,   -- flags for relphot, relastro, ??
     2300    zpt_obs     FLOAT,          -- calibrated zero point for this release of
     2301    zpt_stdev   FLOAT,          -- this exposure
     2302    path_base   VARCHAR(255),   -- path_base of any supporting files for this
     2303                                -- release of this exposure.
     2304    registered  DATETIME,       -- insertion time for this row
     2305    time_stamp  DATETIME,       -- time of last update for this row
     2306    PRIMARY KEY (relexp_id),
     2307    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     2308    KEY (state),
     2309    FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id),
     2310    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
     2311    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
     2312) ENGINE=innodb DEFAULT CHARSET=latin1;
     2313
     2314CREATE TABLE relStack (
     2315    relstack_id BIGINT AUTO_INCREMENT,
     2316    rel_id      INT(8),
     2317    skycal_id   BIGINT,         -- id of the sky calibration run that supplied the calibration
     2318                                -- for this release. We get back to stack from here
     2319    skycell_id  varchar(64),
     2320    tess_id     varchar(64),
     2321    state       VARCHAR(16),    -- released, pending, archive, drop 
     2322    flags       INT UNSIGNED,
     2323    stack_type  VARCHAR(16),    -- nightly, deep, reference
     2324    zpt_obs     FLOAT,          -- calibrated zero point for this release of this skycell
     2325    zpt_stdev   FLOAT,
     2326    fwhm_major  FLOAT,          -- seeing as measured by staticsky
     2327    path_base   VARCHAR(255),   -- path_base of any supporting files,
     2328    registered  DATETIME,
     2329    time_stamp  DATETIME,
     2330    PRIMARY KEY (relstack_id),
     2331    KEY (tess_id, skycell_id),
     2332    KEY (rel_id, tess_id, skycell_id),
     2333    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     2334    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
     2335) ENGINE=innodb DEFAULT CHARSET=latin1;
     2336
     2337UPDATE dbversion set schema_version = '1.1.74', updated= CURRENT_TIMESTAMP();

  • branches/eam_branches/ipp-20121219/dbconfig/ipp.m4

    r33720 r35096  
    3939include(vp.md)
    4040include(skycell.md)
     41include(release.md)

  • branches/eam_branches/ipp-20121219/ippMonitor

  • branches/eam_branches/ipp-20121219/ippMonitor/Makefile.in

    r34903 r35096  
    215215$(DESTWWW)/simplePlotStackImage.php \
    216216$(DESTWWW)/simplePlotStaticskyImage.php \
    217 $(DESTWWW)/newsimplePlotStaticskyImage.php \
    218217$(DESTWWW)/simplePlotSkycalImage.php \
    219218$(DESTWWW)/histogramBackgroundImage.php \

  • branches/eam_branches/ipp-20121219/ippMonitor/czartool/czarDbSchema.sql

    r33434 r35096  
    7878  `total` float default NULL,
    7979  `available` float default NULL,
     80  `usable` float default NULL,
    8081  `used` float default NULL,
    8182  `hostsOver98` smallint(6) default NULL,

  • branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/CzarDb.pm

    r34096 r35096  
    99
    1010my @stages = (
    11         "chip", 
    12         "cam", 
    13         "fake", 
    14         "warp", 
    15         "stack", 
    16         "diff", 
    17         "dist", 
     11        "chip",
     12        "cam",
     13        "fake",
     14        "warp",
     15        "stack",
     16        "diff",
     17        "dist",
    1818        "pub",
    1919        "chipbackground"); # TODO put elsewhere
    … …  
    6464    SELECT DISTINCT label
    6565        FROM processed
    66         WHERE timestamp >= '$begin' 
    67         AND timestamp <= '$end' 
     66        WHERE timestamp >= '$begin'
     67        AND timestamp <= '$end'
    6868SQL
    6969
    … …  
    109109
    110110    SELECT DISTINCT label
    111         FROM chip 
    112         WHERE timestamp >= '$begin' 
    113         AND timestamp <= '$end' 
     111        FROM chip
     112        WHERE timestamp >= '$begin'
     113        AND timestamp <= '$end'
    114114        AND label NOT LIKE '%_ud%'
    115115        AND label NOT LIKE 'update%'
    … …  
    153153
    154154    my $query = $self->{_db}->prepare(<<SQL);
    155     SELECT reverting 
     155    SELECT reverting
    156156        FROM reverts
    157          WHERE stage LIKE '$stage'; 
     157         WHERE stage LIKE '$stage';
    158158SQL
    159159
    … …  
    171171
    172172    my $query = $self->{_db}->prepare(<<SQL);
    173      UPDATE reverts 
    174          SET reverting = $reverting 
     173     UPDATE reverts
     174         SET reverting = $reverting
    175175         WHERE stage LIKE '$stage';
    176176SQL
    … …  
    188188
    189189    my $query = $self->{_db}->prepare(<<SQL);
    190      UPDATE current_labels 
    191          SET priority = $priority 
    192          WHERE label LIKE '$label' 
     190     UPDATE current_labels
     191         SET priority = $priority
     192         WHERE label LIKE '$label'
    193193         AND server LIKE 'stdscience';
    194194SQL
    … …  
    323323###########################################################################
    324324sub insertNewClusterSpace {
    325     my ($self, $total, $available, $used, $hostsOver98) = @_;
     325    my ($self, $total, $available, $usable, $used, $hostsOver98) = @_;
    326326
    327327    my $query = $self->{_db}->prepare(<<SQL);
    328328    INSERT INTO cluster_space
    329         (total, available, used, hostsOver98)
     329        (total, available, usable, used, hostsOver98)
    330330        VALUES
    331         ( $total, $available, $used, $hostsOver98);
     331        ( $total, $available, $usable, $used, $hostsOver98);
    332332SQL
    333333
    … …  
    362362
    363363    my $query = $self->{_db}->prepare(<<SQL);
    364     SELECT 
    365         DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
     364    SELECT
     365        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'),
    366366        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
    367367        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
    368             FROM $stage 
     368            FROM $stage
    369369            WHERE label LIKE '$label'
    370             AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
     370            AND timestamp >= '$fromTime' AND timestamp <= '$toTime';
    371371SQL
    372372
    … …  
    413413###########################################################################
    414414#
    415 # Analysis this stage between these times 
     415# Analysis this stage between these times
    416416#
    417417###########################################################################
    … …  
    429429
    430430    my $query = $self->{_db}->prepare(<<SQL);
    431     SELECT 
    432         timestamp, pending, faults, processed 
    433         FROM $stage 
    434         WHERE label LIKE '$label' 
    435         AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
     431    SELECT
     432        timestamp, pending, faults, processed
     433        FROM $stage
     434        WHERE label LIKE '$label'
     435        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
    436436        ORDER BY timestamp;
    437437SQL
    … …  
    452452        # get linear values
    453453        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
    454            
     454
    455455            $totalProcessed = $totalProcessed + ($thisProcessed - $lastProcessed);
    456456        }
    … …  
    480480    my $started = undef;
    481481    my $finished = undef;
    482    
     482
    483483    my ($have25, $have50, $have75, $have90, $have95, $have100) = 0;
    484484
    … …  
    492492        # get linear values
    493493        if($lastProcessed != -1 && $thisProcessed > $lastProcessed) {
    494            
     494
    495495            $linearProcessed = $linearProcessed + ($thisProcessed - $lastProcessed);
    496496        }
    … …  
    508508
    509509            my $timeDiff = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp) + 0.001;
    510                
     510
    511511            if ($thisProcessed >= $lastProcessed){
    512                 $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff; 
     512                $derivProcessed = abs($thisProcessed - $lastProcessed)/$timeDiff;
    513513            }
    514514            else {$derivProcessed = 0;}
    515             $derivPending = abs($thisPending - $lastPending)/$timeDiff;   
    516             $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;   
     515            $derivPending = abs($thisPending - $lastPending)/$timeDiff;
     516            $derivFaults = abs($thisFaults - $lastFaults)/$timeDiff;
    517517        }
    518518        else {
    519        
     519
    520520            $derivProcessed = 0;
    521521            $derivPending = 0;
    … …  
    539539            else {
    540540                if (defined $notProcessing) {
    541                    
    542                     $notProcessing = undef; 
    543                     $stuck = undef; 
     541
     542                    $notProcessing = undef;
     543                    $stuck = undef;
    544544                }
    545545                if (!defined $started) {
    546546
    547547                    $started = $lastTimestamp;
    548                     $finished = undef; 
     548                    $finished = undef;
    549549                }
    550550            }
    … …  
    564564            # how long not processing?
    565565            if (defined $notProcessing) {
    566                
     566
    567567                $howLongNotProcessing = $self->diffTimes($lastTimestamp, $notProcessing);
    568568                $notProcessingLongerThanInterval = $self->isBefore($interval, $howLongNotProcessing);
    569569            }
    570570            else {
    571            
     571
    572572                $howLongNotProcessing = 0;
    573573                $notProcessingLongerThanInterval = 0;
    … …  
    611611
    612612    if (!defined $started){
    613    
     613
    614614        $stageMetrics->setStarted(undef);
    615615    }
    616616    else {
    617    
     617
    618618        my $rate = ($stageMetrics->getProcessed()/$self->getTimeInSecs($stageMetrics->getTotalTime) ) * 3600.0;
    619619        $stageMetrics->setRate($rate);
    … …  
    625625    }
    626626    else {
    627    
     627
    628628        $stageMetrics->setStuck(undef);
    629629    }
    … …  
    642642    my $query = $self->{_db}->prepare(<<SQL);
    643643    SELECT  COUNT(*)
    644         FROM $stage 
     644        FROM $stage
    645645        WHERE label LIKE '$label'
    646         AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
     646        AND timestamp >= '$fromTime' AND timestamp <= '$toTime';
    647647SQL
    648648
    … …  
    657657    # grab all the data for the provided time range
    658658    $query = $self->{_db}->prepare(<<SQL);
    659     SELECT 
    660         timestamp, pending, faults, processed 
    661         FROM $stage 
    662         WHERE label LIKE '$label' 
    663         AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
     659    SELECT
     660        timestamp, pending, faults, processed
     661        FROM $stage
     662        WHERE label LIKE '$label'
     663        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
    664664        ORDER BY timestamp;
    665665SQL
    … …  
    724724            $diffFaults = $thisFaults - $lastFaults;
    725725
    726             # First, look for large positive jumps in processed, most likely a label 
     726            # First, look for large positive jumps in processed, most likely a label
    727727            # added, and store value to subtract from linear value below. If > 1000
    728728            # processed per hour, then it must be wrong
    … …  
    741741            }
    742742            else {
    743            
     743
    744744                $diffProcessed = 0;
    745745                $linearProcessed = $linearProcessed;
    … …  
    747747
    748748            # calculate first derivative in units of images per hour
    749             $derivPending = $diffPending/($timeSep/3600); 
    750             $derivFaults = $diffFaults/($timeSep/3600); 
    751             $derivProcessed = $diffProcessed/($timeSep/3600); 
     749            $derivPending = $diffPending/($timeSep/3600);
     750            $derivFaults = $diffFaults/($timeSep/3600);
     751            $derivProcessed = $diffProcessed/($timeSep/3600);
    752752
    753753        }
    754754        # first time in
    755755        else {
    756        
     756
    757757            $derivPending = 0;
    758758            $derivFaults = 0;
    … …  
    777777            my $totalPending = 0;
    778778            my $totalFaults = 0;
    779             for (my $i=$size-1; $i>=($size - $range); $i--) { 
    780                
     779            for (my $i=$size-1; $i>=($size - $range); $i--) {
     780
    781781                $totalPending = $totalPending + $ratePending[$i];
    782782                $totalFaults = $totalFaults + $rateFaults[$i];
    … …  
    833833    # get earliest time for this label
    834834    my $query = $self->{_db}->prepare(<<SQL);
    835     SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval) 
     835    SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval)
    836836        FROM $stage
    837         WHERE label LIKE '$label'; 
    838 SQL
    839     $query->execute;
    840 
    841 my $timestamp = $query->fetchrow_array(); 
     837        WHERE label LIKE '$label';
     838SQL
     839    $query->execute;
     840
     841my $timestamp = $query->fetchrow_array();
    842842
    843843    $query = $self->{_db}->prepare(<<SQL);
    844     SELECT faults 
     844    SELECT faults
    845845        FROM $stage
    846         WHERE label LIKE '$label' 
     846        WHERE label LIKE '$label'
    847847        AND timestamp <= '$timestamp' LIMIT 1;
    848848SQL
    … …  
    861861
    862862    my $query = $self->{_db}->prepare(<<SQL);
    863     SELECT pending - faults 
    864         FROM $stage 
    865         WHERE label LIKE '$label' 
     863    SELECT pending - faults
     864        FROM $stage
     865        WHERE label LIKE '$label'
    866866        ORDER BY timestamp DESC LIMIT 1;
    867867SQL
    … …  
    880880
    881881    my $query = $self->{_db}->prepare(<<SQL);
    882     SELECT 
    883         pending, faults 
     882    SELECT
     883        pending, faults
    884884        FROM $stage
    885885        WHERE label LIKE '$label'
    … …  
    890890    if (!$query->execute) {return 0;}
    891891
    892     (${$pending}, ${$faults}) = $query->fetchrow_array();   
     892    (${$pending}, ${$faults}) = $query->fetchrow_array();
    893893
    894894    ${$processed} = $self->countProcessed($label, $stage, $fromTime, $toTime);
    … …  
    912912
    913913    my $query = $self->{_db}->prepare(<<SQL);
    914     SELECT 
    915         host, used, available, writable, readable 
     914    SELECT
     915        host, used, available, writable, readable
    916916        FROM hosts
    917917        ORDER BY host;
    … …  
    926926
    927927        my ($host, $used, $available, $writable, $readable) = @row;
    928    
    929         if (($used+$available) == 0) {
    930           $underLimit = 1;
    931         } elsif (($used/($used+$available))*100 > $limit) {
    932           $underLimit = 0;
    933         } else {
    934           $underLimit = 1;
    935         }
     928
     929        if (($used+$available) == 0) {
     930          $underLimit = 1;
     931        } elsif (($used/($used+$available))*100 > $limit) {
     932          $underLimit = 0;
     933        } else {
     934          $underLimit = 1;
     935        }
    936936
    937937        # Col 1: host
    938938        print GNUDAT "$host";
    939        
     939
    940940        # Col 2: available, readable used space under limit
    941941        if ($readable && $underLimit) {print GNUDAT " $used";}
    … …  
    973973
    974974    my $query = $self->{_db}->prepare(<<SQL);
    975     SELECT used/total*100 
    976         FROM cluster_space 
     975    SELECT used/total*100
     976        FROM cluster_space
    977977        ORDER BY timestamp DESC LIMIT 1;
    978978SQL
    … …  
    983983###########################################################################
    984984#
    985 # Gets time series data for cluster storage and saves it to file 
     985# Gets time series data for cluster storage and saves it to file
    986986#
    987987###########################################################################
    988988sub createStorageTimeSeriesData {
    989989    my ($self, $tmpFile, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
     990    my $maxAvailable;
     991    my $maxUsable;
     992    my $minAvailable;
     993    my $minUsable;
    990994
    991995    open (GNUDAT, ">".$tmpFile->filename);
    992996
    993997    my $query = $self->{_db}->prepare(<<SQL);
    994     SELECT 
    995         MAX(available), MIN(available),  
    996         DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
     998    SELECT
     999        MAX(available), MIN(available), MAX(usable), MIN(usable),
     1000        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'),
    9971001        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
    9981002        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
    999             FROM cluster_space 
    1000             WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
     1003            FROM cluster_space
     1004            WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime';
    10011005SQL
    10021006
    10031007    if (!$query->execute) {return undef;}
    10041008
    1005     (${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
    1006 
     1009    ($maxAvailable, $minAvailable, $maxUsable, $minUsable, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
     1010    ${$maxY} = ($maxAvailable > $maxUsable ? $maxAvailable : $maxUsable);
     1011    ${$minY} = ($minAvailable > $minUsable ? $minAvailable : $minUsable);
    10071012
    10081013    if (!${$maxY} || !${$minY} || !${$maxX} || !${$minX} || !${$timeDiff}) {return 0;}
    10091014
    10101015    $query = $self->{_db}->prepare(<<SQL);
    1011     SELECT 
    1012         DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available 
    1013         FROM cluster_space 
    1014         WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime' 
     1016    SELECT
     1017        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), available, usable
     1018        FROM cluster_space
     1019        WHERE timestamp >= '$fromTime' AND timestamp <= '$toTime'
    10151020        ORDER BY timestamp;
    10161021SQL
    … …  
    10211026    while (my @row = $query->fetchrow_array()) {
    10221027
    1023         my ($timestamp, $available) = @row;
    1024         print GNUDAT "$timestamp $available\n";
     1028        my ($timestamp, $available, $usable) = @row;
     1029        print GNUDAT "$timestamp $available $usable\n";
    10251030    }
    10261031
    … …  
    10741079###########################################################################
    10751080#
    1076 # Gets count of processed stuff in a given time period 
     1081# Gets count of processed stuff in a given time period
    10771082#
    10781083###########################################################################
    … …  
    10811086
    10821087    my $query = $self->{_db}->prepare(<<SQL);
    1083     SELECT 
    1084         processed 
     1088    SELECT
     1089        processed
    10851090        FROM $stage
    1086         WHERE label LIKE '$label' 
    1087         AND timestamp >= '$begin' 
    1088         AND timestamp < '$end' 
     1091        WHERE label LIKE '$label'
     1092        AND timestamp >= '$begin'
     1093        AND timestamp < '$end'
    10891094SQL
    10901095        $query->execute;
    … …  
    11681173
    11691174                $query = $self->{_db}->prepare(<<SQL);
    1170                 SELECT COUNT(*) 
    1171                     FROM $stage 
     1175                SELECT COUNT(*)
     1176                    FROM $stage
    11721177                    WHERE timestamp > '$fromTime'
    11731178                    AND timestamp <= '$toTime'
    1174                     AND label = '$label' 
     1179                    AND label = '$label'
    11751180SQL
    11761181
    … …  
    11811186
    11821187                $query = $self->{_db}->prepare(<<SQL);
    1183                 DELETE FROM $stage 
    1184                     WHERE timestamp > '$fromTime' 
    1185                     AND timestamp <= '$toTime' 
     1188                DELETE FROM $stage
     1189                    WHERE timestamp > '$fromTime'
     1190                    AND timestamp <= '$toTime'
    11861191                    AND label = '$label' ORDER BY timestamp DESC LIMIT $toDelete
    11871192SQL
    … …  
    12031208
    12041209                $query = $self->{_db}->prepare(<<SQL);
    1205                 SELECT COUNT(*) 
    1206                     FROM servers 
     1210                SELECT COUNT(*)
     1211                    FROM servers
    12071212                    WHERE timestamp > '$fromTime'
    12081213                    AND timestamp <= '$toTime'
    1209                     AND server = '$server' 
     1214                    AND server = '$server'
    12101215SQL
    12111216                    $query->execute;
    … …  
    12151220
    12161221                    $query = $self->{_db}->prepare(<<SQL);
    1217                     DELETE FROM servers 
    1218                         WHERE timestamp > '$fromTime' 
    1219                         AND timestamp <= '$toTime' 
     1222                    DELETE FROM servers
     1223                        WHERE timestamp > '$fromTime'
     1224                        AND timestamp <= '$toTime'
    12201225                        AND server = '$server' ORDER BY timestamp DESC LIMIT $toDelete
    12211226SQL
    … …  
    12291234        # now deal with cluster_space table
    12301235        $query = $self->{_db}->prepare(<<SQL);
    1231         SELECT COUNT(*) 
    1232             FROM cluster_space 
     1236        SELECT COUNT(*)
     1237            FROM cluster_space
    12331238            WHERE timestamp > '$fromTime'
    12341239            AND timestamp <= '$toTime'
    … …  
    12401245
    12411246            $query = $self->{_db}->prepare(<<SQL);
    1242             DELETE FROM cluster_space 
    1243                 WHERE timestamp > '$fromTime' 
    1244                 AND timestamp <= '$toTime' 
     1247            DELETE FROM cluster_space
     1248                WHERE timestamp > '$fromTime'
     1249                AND timestamp <= '$toTime'
    12451250                ORDER BY timestamp DESC LIMIT $toDelete
    12461251SQL
    … …  
    12571262###########################################################################
    12581263#
    1259 # Optimizes all tables that need to be optimized 
     1264# Optimizes all tables that need to be optimized
    12601265#
    12611266###########################################################################
    … …  
    12751280###########################################################################
    12761281#
    1277 # Returns if a particular server has been stopped for a certain interval 
     1282# Returns if a particular server has been stopped for a certain interval
    12781283#
    12791284###########################################################################
    … …  
    12861291###########################################################################
    12871292#
    1288 # Returns if a particular server has been down for a certain interval 
     1293# Returns if a particular server has been down for a certain interval
    12891294#
    12901295###########################################################################
    … …  
    12971302###########################################################################
    12981303#
    1299 # Returns if a particular server has been down for a certain interval 
     1304# Returns if a particular server has been down for a certain interval
    13001305#
    13011306###########################################################################
    … …  
    13041309
    13051310    my $query = $self->{_db}->prepare(<<SQL);
    1306     SELECT COUNT(*) 
     1311    SELECT COUNT(*)
    13071312        FROM servers
    1308         WHERE server = '$server' 
     1313        WHERE server = '$server'
    13091314        AND timestamp > now() - INTERVAL $interval
    13101315SQL
    … …  
    13161321
    13171322    $query = $self->{_db}->prepare(<<SQL);
    1318     SELECT COUNT(*) 
     1323    SELECT COUNT(*)
    13191324        FROM servers
    1320         WHERE server = '$server' 
     1325        WHERE server = '$server'
    13211326        AND timestamp > now() - INTERVAL $interval
    13221327        AND !$mode
    … …  
    13291334
    13301335    $query = $self->{_db}->prepare(<<SQL);
    1331     SELECT 
     1336    SELECT
    13321337        timestamp
    13331338        FROM servers
    13341339        WHERE server = '$server'
    1335         AND $mode 
    1336         ORDER BY timestamp DESC 
     1340        AND $mode
     1341        ORDER BY timestamp DESC
    13371342        LIMIT 1
    13381343
    … …  
    13461351###########################################################################
    13471352#
    1348 # Returns an array of servers 
     1353# Returns an array of servers
    13491354#
    13501355###########################################################################
    … …  
    13691374###########################################################################
    13701375#
    1371 # Returns an array of labels present during the provided time frame 
     1376# Returns an array of labels present during the provided time frame
    13721377#
    13731378###########################################################################
    … …  
    13761381
    13771382    my $query = $self->{_db}->prepare(<<SQL);
    1378     SELECT DISTINCT label 
    1379         FROM $stage 
    1380         WHERE timestamp > '$fromTime' 
     1383    SELECT DISTINCT label
     1384        FROM $stage
     1385        WHERE timestamp > '$fromTime'
    13811386        AND timestamp <= '$toTime';
    13821387SQL

  • branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Nebulous.pm

    r34096 r35096  
    2828###########################################################################
    2929#
    30 # Converts Kb to Tb 
    31 # 
     30# Converts Kb to Tb
     31#
    3232###########################################################################
    3333sub convertKbToTb {
    … …  
    5050###########################################################################
    5151#
    52 # Returns total cluster usage in T 
     52# Returns total cluster usage in T
    5353#
    5454###########################################################################
    … …  
    6161###########################################################################
    6262#
    63 # Returns total cluster space in T 
     63# Returns total cluster space in T
    6464#
    6565###########################################################################
    … …  
    7272###########################################################################
    7373#
    74 # Returns total cluster available in T 
     74# Returns total cluster available in T
    7575#
    7676###########################################################################
    … …  
    8383###########################################################################
    8484#
    85 # Runs neb-df and puts output in gnuplot ready format 
     85# Runs neb-df and puts output in gnuplot ready format
    8686#
    8787###########################################################################
    … …  
    9393    $self->{_totalHosts} = 0;
    9494    $self->{_hostsOverNinetyEightPC} = 0;
     95    $self->{_totalUsable} = 0;
    9596    my $readable;
    9697    my $writable;
    … …  
    100101        if ($line =~ m/.*Filesystem.*/i) {next;}
    101102        elsif ($line =~ m/.*summmary.*/i) {next;}
    102         elsif ($line =~ m/.*total.*/i) {next;}
    103         elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) {
     103        elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*total.*/i) {
     104            $self->{_totalUsable} += $self->convertKbToTb($2);
     105        } elsif ($line =~ m/\s+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+.*allocated.*/i) {
    104106
    105107            $self->{_totalUsed} = $self->convertKbToTb($1);
    … …  
    109111
    110112            next;
    111         }
    112         elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
    113                ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) ||
    114                ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
    115        
     113        } elsif (($line =~ m/(ipp[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)||
     114               ($line =~ m/(ippb[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) ||
     115               ($line =~ m/(stsci0[0-9]+\.[0-9]{1})\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) {
     116
    116117            $self->{_totalHosts}++;
    117118            if ($4 >= 98) {$self->{_hostsOverNinetyEightPC}++;}
    … …  
    119120            $self->getReadableWritableStatus($1, \$readable, \$writable);
    120121
    121             $self->{_czarDb}->updateHost($1, 
    122                     $self->convertKbToTb($2+$3), 
    123                     $self->convertKbToTb($3), 
    124                     $self->convertKbToTb($2), 
     122            $self->{_czarDb}->updateHost($1,
     123                    $self->convertKbToTb($2+$3),
     124                    $self->convertKbToTb($3),
     125                    $self->convertKbToTb($2),
    125126                    $readable, $writable);
    126127        }
    … …  
    128129
    129130    $self->{_czarDb}->insertNewClusterSpace(
    130             $self->{_totalCluster},
    131             $self->{_totalAvail}, 
    132             $self->{_totalUsed},
     131            $self->{_totalCluster},
     132            $self->{_totalAvail},
     133            $self->{_totalUsable},
     134            $self->{_totalUsed},
    133135            $self->{_hostsOverNinetyEightPC});
    134136}
    … …  
    136138###########################################################################
    137139#
    138 # Runs neb-host and readable/writable status for this host 
     140# Runs neb-host and readable/writable status for this host
    139141#
    140142###########################################################################

  • branches/eam_branches/ipp-20121219/ippMonitor/czartool/czartool/Plotter.pm

    r34249 r35096  
    630630        set xlabel "Time"
    631631        set ylabel "Available (TB)"
    632         plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2
     632        plot "$datFile" using 1:2 title "Available" with lines lt 2 lw 2, "$datFile" using 1:3 title "Theoretical" with lines lt 3 lw 2
    633633PLOT
    634634

  • branches/eam_branches/ipp-20121219/ippScripts/scripts/camera_exp.pl

    r33576 r35096  
    4141
    4242my ( $exp_tag, $cam_id, $camera, $outroot, $dbname, $reduction, $dvodb, $verbose, $no_update,
    43      $no_op, $redirect, $save_temps, $run_state, $skip_binned, $skip_masks);
     43     $no_op, $redirect, $save_temps, $run_state, $skip_binned, $skip_masks, $bkg_only);
    4444GetOptions(
    4545    'exp_tag=s'         => \$exp_tag, # Exposure identifier
    … …  
    5353    'skip-binned'       => \$skip_binned, # override recipe - don't create binned images
    5454    'skip-refmask'      => \$skip_masks, # override recipe - don't create refmask
     55    'bkg-only'          => \$bkg_only,  # override recipe - only do background continuity
    5556    'verbose'           => \$verbose,   # Print to stdout
    5657    'no-update'         => \$no_update, # Update the database?
    … …  
    111112&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
    112113
     114my $bkg_recipe     = 'PPIMAGE_BKGCONT'; # Add to reduction?
    113115my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    114116
    … …  
    215217my ($list4File, $list4Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b4.list.XXXX", UNLINK => !$save_temps ); # For astrometry
    216218
     219### Create temp file for background models
     220my ($list5File, $list5Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b5.list.XXXX", UNLINK => !$save_temps ); # For background models
     221
    217222# XXX we perform astrometry iff photometry output exists
    218223my @outMasks;                   # Names of output masks
     224my @bkg_models;                 # Names of output background models
    219225foreach my $file (@$files) {
    220226    next if $file->{quality} != 0;
    … …  
    226232    my $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id);
    227233    my $chipMask   = $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id);
    228 
     234   
    229235    print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n");
    230236    print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n");
    231237    print $list3File ($chipObjects . "\n");
    232238    print $list4File ($chipMask . "\n");
     239    print $list5File ($ipprc->filename("PSPHOT.BACKMDL", $file->{path_base}, $class_id) . "\n");
    233240
    234241    push @outMasks, prepare_output("PSASTRO.OUTPUT.MASK", $outroot, $class_id, 1) if $produceMasks;
     242    push @bkg_models, prepare_output("PPIMAGE.BACKMDL", $outroot, $class_id, 1);
    235243}
    236244close $list1File;
    … …  
    238246close $list3File;
    239247close $list4File;
    240 
     248close $list5File;
    241249# Output products
    242250$ipprc->outroot_prepare($outroot);
    … …  
    291299    }
    292300
    293     {
     301    if (!$bkg_only) {
    294302        # run psastro on the chipObjects, producing fpaObjects
    295303        my $command;
    … …  
    353361            }
    354362        }
     363    }
     364    # Construct FPA continuity corrected background images
     365    {
     366        my $command;
     367        $command = "$ppImage";
     368        $command .= " -list $list5Name $outroot";
     369        $command .= " -recipe PPIMAGE $bkg_recipe";
     370        $command .= " -dbname $dbname" if defined $dbname;
     371
     372        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     373            run(command => $command, verbose => $verbose);
     374        unless ($success) {
     375            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     376            &my_die("Unable to perform ppImage to fix background: $error_code", $cam_id, $error_code);
     377        }
     378        foreach my $bkgModel (@bkg_models) {
     379            check_output($bkgModel, $replicateOutputs);
     380        }
    355381    }
    356382}

  • branches/eam_branches/ipp-20121219/ippScripts/scripts/destreak_restore_camera.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippScripts/scripts/ipp_apply_burntool_single.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippScripts/scripts/ipp_cleanup.pl

    r34903 r35096  
    6565        &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
    6666
     67
     68my $hostname = `hostname`;
     69chomp $hostname;
     70print STDERR "starting cleanup for $stage $stage_id on $hostname\n" ;
     71
     72my $nebulous_server;
    6773
    6874my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n';
    … …  
    22802286    if (my $resolved = $ipprc->file_resolve($filename)) {
    22812287        my $bzip2_filename = $filename . '.bz2';
    2282         if ($ipprc->file_exists($bzip2_filename)) {
    2283             $ipprc->kill_file($bzip2_filename);
    2284         }
     2288        if (file_scheme($bzip2_filename) eq 'neb') {
     2289            # XXX: GRRRR $ipprc->file_exists() returns false if storage object exists but instance has
     2290            # size of zero
     2291            $nebulous_server = $ipprc->nebulous() if !$nebulous_server;
     2292            if ($nebulous_server->storage_object_exists($bzip2_filename)) {
     2293                print STDERR "$bzip2_filename exists, killing\n";
     2294                $ipprc->kill_file($bzip2_filename);
     2295            } else {
     2296                print STDERR "$bzip2_filename does not exist\n" if $very_verbose;
     2297            }
     2298        } else {
     2299            if ($ipprc->file_exists($bzip2_filename)) {
     2300                print STDERR "$bzip2_filename exists, killing\n";
     2301                $ipprc->kill_file($bzip2_filename);
     2302            } else {
     2303                print STDERR "$bzip2_filename does not exist\n" if $very_verbose;
     2304            }
     2305        }
    22852306        my $bzip2_file = $ipprc->file_create($bzip2_filename);
    2286         my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
     2307        my_die("Unable to create $bzip2_filename", $stage_id, $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
    22872308
    22882309        my $command = "$bzip2 < $resolved > $bzip2_file";

  • branches/eam_branches/ipp-20121219/ippScripts/scripts/magic_destreak.pl

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippTasks

  • branches/eam_branches/ipp-20121219/ippTasks/survey.pro

    r34828 r35096  
    15291529    # note: currently skycal uses the staticskyRun workdir
    15301530
    1531     $run = staticskytool -defineskycalrun -select_label $label -set_dist_group $dist_group
     1531    $run = staticskytool -defineskycalrun -set_label $label -select_label $label -set_dist_group $dist_group
    15321532
    15331533    if ($DB:n == 0)

  • branches/eam_branches/ipp-20121219/ippToPsps

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippTools/share/Makefile.am

    r34800 r35096  
    461461        vptool_processedcell.sql \
    462462        vptool_revertrun.sql \
    463         sctool_list.sql
     463        sctool_list.sql \
     464        releasetool_listsurvey.sql \
     465        releasetool_listrelease.sql \
     466        releasetool_listrelexp.sql \
     467        releasetool_definerelexp.sql
    464468

  • branches/eam_branches/ipp-20121219/ippTools/share/bgtool_definechip.sql

    r32680 r35096  
    33FROM chipRun
    44JOIN rawExp USING(exp_id)
     5JOIN camRun USING(chip_id)
     6JOIN camProcessedExp USING(cam_id)
    57LEFT JOIN chipBackgroundRun USING(chip_id)
    68WHERE chipRun.state = 'full'
     9AND camRun.state = 'full'
     10AND camProcessedExp.quality = 0

  • branches/eam_branches/ipp-20121219/ippTools/share/camtool_find_pendingimfile.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippTools/share/chiptool_setimfiletoupdate.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_create_tables.sql

    r34903 r35096  
    21482148) ENGINE=InnoDB DEFAULT CHARSET=latin1;
    21492149
    2150 
    2151 -- This comment line is here to avoid empty query error.
     2150CREATE TABLE survey (
     2151    surveyID    INT(8),
     2152    surveyName  VARCHAR(16) NOT NULL,
     2153    description VARCHAR(255),
     2154    PRIMARY KEY(surveyID),
     2155    UNIQUE  KEY(name)
     2156) ENGINE=InnoDB CHARSET=latin1;
     2157
     2158
     2159CREATE TABLE ippRelease (
     2160    rel_id      INT(8) AUTO_INCREMENT,
     2161    surveyID    INT(8),
     2162    releaseName VARCHAR(64),
     2163    relase_state VARCHAR(16),   -- active, pending, archive, drop
     2164    dataRelease INT(8),         -- PSPS dataRelease
     2165    priority    INT(8),
     2166    PRIMARY KEY(rel_id),
     2167    KEY(releaseName),
     2168    KEY(state),
     2169    KEY(priority),
     2170    FOREIGN KEY(surveyID) REFERENCES survey(surveyID)
     2171) ENGINE=Innodb DEFAULT CHARSET=latin1;
     2172
     2173CREATE TABLE relExp (
     2174    relexp_id   BIGINT AUTO_INCREMENT,
     2175    rel_id      INT(8),
     2176    exp_id      BIGINT,
     2177    chip_id     BIGINT,         -- links to the runs that supplied the data
     2178    cam_id      BIGINT,         -- for this release
     2179    state       VARCHAR(16),    -- released, pending, archive, drop 
     2180    flags       INT UNSIGNED,   -- flags for relphot, relastro, ??
     2181    zpt_obs     FLOAT,          -- calibrated zero point for this release of
     2182    zpt_stdev   FLOAT,          -- this exposure
     2183    path_base   VARCHAR(255),   -- path_base of any supporting files for this
     2184                                -- release of this exposure.
     2185    registered  DATETIME,       -- insertion time for this row
     2186    time_stamp  DATETIME,       -- time of last update for this row
     2187    PRIMARY KEY (relexp_id),
     2188    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     2189    KEY (state),
     2190    FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id),
     2191    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
     2192    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
     2193) ENGINE=innodb DEFAULT CHARSET=latin1;
     2194
     2195CREATE TABLE relStack (
     2196    relstack_id BIGINT AUTO_INCREMENT,
     2197    rel_id      INT(8),
     2198    skycal_id   BIGINT,         -- id of the sky calibration run that supplied the calibration
     2199                                -- for this release. We get back to stack from here
     2200    skycell_id  varchar(64),
     2201    tess_id     varchar(64),
     2202    state       VARCHAR(16),    -- released, pending, archive, drop 
     2203    flags       INT UNSIGNED,
     2204    stack_type  VARCHAR(16),    -- nightly, deep, reference
     2205    zpt_obs     FLOAT,          -- calibrated zero point for this release of this skycell
     2206    zpt_stdev   FLOAT,
     2207    fwhm_major  FLOAT,          -- seeing as measured by staticsky
     2208    path_base   VARCHAR(255),   -- path_base of any supporting files,
     2209    registered  DATETIME,
     2210    time_stamp  DATETIME,
     2211    PRIMARY KEY (relstack_id),
     2212    KEY (tess_id, skycell_id),
     2213    KEY (rel_id, tess_id, skycell_id),
     2214    FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id),
     2215    FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id)
     2216) ENGINE=innodb DEFAULT CHARSET=latin1;
     2217
     2218
     2219
     2220-- These comment lines are here to avoid an empty query error.
    21522221-- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.

  • branches/eam_branches/ipp-20121219/ippTools/share/pxadmin_drop_tables.sql

    r33720 r35096  
    109109DROP TABLE IF EXISTS skycalResult;
    110110DROP TABLE IF EXISTS skycell;
     111DROP TABLE IF EXISTS relExp;
     112DROP TABLE IF EXISTS ippRelease;
     113DROP TABLE IF EXISTS survey;
    111114
    112115SET FOREIGN_KEY_CHECKS=1

  • branches/eam_branches/ipp-20121219/ippTools/share/stacktool_tobkg.sql

    r34800 r35096  
    2525    -- WHERE hook %s
    2626GROUP BY stack_id
    27 HAVING (SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id) AND
     27HAVING ((SUM(IF(warpRun.state = 'cleaned', 1, 0)) = COUNT(stackInputSkyfile.warp_id) OR
     28         SUM(IF(warpRun.state = 'full', 1, 0)) = COUNT(stackInputSkyfile.warp_id)) AND
    2829        SUM(IF(warpSkyfile.background_model = 1, 1, 0)) = COUNT(stackInputSkyfile.warp_id))
    2930

  • branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_defineskycalrun.sql

    r34761 r35096  
    11SELECT
    2     sky_id,
    3     stack_id,
     2    staticskyRun.sky_id,
     3    stackRun.stack_id,
    44    skycell_id,
    55    filter,
    … …  
    1414    JOIN stackSumSkyfile USING(stack_id)
    1515    JOIN skycell USING(tess_id, skycell_id)
    16     LEFT JOIN skycalRun USING(sky_id, stack_id)
     16    LEFT JOIN skycalRun ON staticskyRun.sky_id = skycalRun.sky_id AND stackRun.stack_id = skycalRun.stack_id -- join hook %s
    1717WHERE staticskyRun.state = 'full'
    1818    AND staticskyResult.quality = 0

  • branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revert.sql

    r28154 r35096  
    11DELETE FROM staticskyResult
    2 USING staticskyResult, staticskyRun
     2USING staticskyResult, staticskyRun, staticskyInput, stackRun, skycell
    33WHERE staticskyRun.sky_id = staticskyResult.sky_id
     4    AND staticskyRun.sky_id = staticskyInput.sky_id
     5    AND staticskyInput.stack_id = stackRun.stack_id
     6    AND stackRun.tess_id = skycell.tess_id
     7    AND stackRun.skycell_id = skycell.skycell_id
    48    AND staticskyRun.state = 'new'
    59    AND staticskyResult.fault != 0

  • branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_revertskycal.sql

    r32962 r35096  
    11DELETE FROM skycalResult
    2 USING skycalResult, skycalRun, stackRun
     2USING skycalResult, skycalRun, stackRun, skycell
    33WHERE skycalRun.skycal_id = skycalResult.skycal_id
    44    AND stackRun.stack_id = skycalRun.stack_id
    55    AND skycalRun.state = 'new'
    66    AND skycalResult.fault != 0
     7    AND stackRun.tess_id = skycell.tess_id
     8    AND stackRun.skycell_id = skycell.skycell_id

  • branches/eam_branches/ipp-20121219/ippTools/share/staticskytool_skycalresult.sql

    r32962 r35096  
    22    skycalResult.*,
    33    skycalRun.stack_id,
    4     stackRun.tess_id,
    5     stackRun.skycell_id,
    64    stackRun.filter,
    75    skycalRun.state,
    86    skycalRun.label,
    97    skycalRun.data_group,
    10     skycalRun.sky_id
     8    skycalRun.sky_id,
     9    skycell.*
    1110FROM skycalRun
    1211JOIN skycalResult USING(skycal_id)
    13 JOIN stackRun USING(stack_id)
    14 
     12JOIN stackRun USING(stack_id) 
     13JOIN skycell USING(tess_id, skycell_id)

  • branches/eam_branches/ipp-20121219/ippTools/share/warptool_towarped.sql

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippTools/src

  • branches/eam_branches/ipp-20121219/ippTools/src/Makefile.am

    r33720 r35096  
    3232        laptool \
    3333        vptool \
    34         sctool
     34        sctool \
     35        releasetool
    3536
    3637pkginclude_HEADERS = \
    … …  
    8586        laptool.h \
    8687        vptool.h \
    87         sctool.h
     88        sctool.h \
     89        releasetool.h
    8890
    8991lib_LTLIBRARIES = libpxtools.la
    … …  
    324326    sctoolConfig.c
    325327
     328releasetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
     329releasetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
     330releasetool_SOURCES = \
     331    releasetool.c \
     332    releasetoolConfig.c
     333
    326334clean-local:
    327335        -rm -f TAGS

  • branches/eam_branches/ipp-20121219/ippTools/src/bgtool.c

    r34081 r35096  
    693693    psMetadata *where = psMetadataAlloc();
    694694    PXOPT_COPY_S64(config->args, where, "-chip_bg_id",    "chipBackgroundRun.chip_bg_id", "==");
     695    PXOPT_COPY_S64(config->args, where, "-chip_id",    "chipBackgroundRun.chip_id", "==");
    695696    PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "==");
    696697    pxAddLabelSearchArgs(config, where, "-label",   "chipBackgroundRun.label", "LIKE");

  • branches/eam_branches/ipp-20121219/ippTools/src/bgtoolConfig.c

    r32199 r35096  
    162162    psMetadata *chipArgs = psMetadataAlloc();
    163163    psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
     164    psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0);
    164165    psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL);
    165166    psMetadataAddS16(chipArgs, PS_LIST_TAIL, "-fault",  0, "search by fault code", 0);

  • branches/eam_branches/ipp-20121219/ippTools/src/magictool.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippTools/src/pxspace.c

    r33921 r35096  
    6767    psMetadataAddF32(md,  PS_LIST_TAIL, "-glong_max",         0, "search by galactic longitude (degrees)", NAN);
    6868    psMetadataAddF32(md,  PS_LIST_TAIL, "-glat_max",          0, "search by galactic latitude (degrees)", NAN);
     69    psMetadataAddF32(md,  PS_LIST_TAIL, "-abs_glat_min",       0, "search by absolute value(galactic latitude) (degrees)", NAN);
     70    psMetadataAddF32(md,  PS_LIST_TAIL, "-abs_glat_max",       0, "search by absolute value(galactic latitude) (degrees)", NAN);
    6971}
    7072
    … …  
    7981    PXOPT_COPY_F32(config->args, where, "-glong_max", "skycell.glong", "<");
    8082    PXOPT_COPY_F32(config->args, where, "-glat_max", "skycell.glat", "<");
     83    PXOPT_COPY_F32(config->args, where, "-abs_glat_min", "abs(skycell.glat)", ">=");
     84    PXOPT_COPY_F32(config->args, where, "-abs_glat_max", "abs(skycell.glat)", "<");
    8185    return true;
    8286}

  • branches/eam_branches/ipp-20121219/ippTools/src/staticskytool.c

    r34815 r35096  
    503503    PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "==");
    504504    pxAddLabelSearchArgs (config, whereMD, "-label", "staticskyRun.label", "==");
     505    pxskycellAddWhere(config, whereMD);
    505506
    506507    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    … …  
    736737    PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "==");
    737738
     739    if (!pxskycellAddWhere(config, where)) {
     740        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
     741        return false;
     742    }
     743
    738744    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
    739745        psFree(where);
    … …  
    10661072    psFree(whereMD);
    10671073
     1074    psString labelHook = psStringCopy("");
    10681075    if (!rerun) {
    10691076        if (label)  {
    1070             psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label);
     1077            psStringAppend(&labelHook, "\nAND skycalRun.label = '%s'", label);
    10711078        }
    10721079        psStringAppend(&query, "\nAND skycal_id IS NULL");
    10731080    }
    10741081
    1075     if (!p_psDBRunQuery(config->dbh, query)) {
     1082
     1083    if (!p_psDBRunQueryF(config->dbh, query, labelHook)) {
    10761084        psError(PS_ERR_UNKNOWN, false, "database error");
     1085        psFree(labelHook);
    10771086        psFree(query);
    10781087        return false;
    10791088    }
     1089    psFree(labelHook);
    10801090    psFree(query);
    10811091
    … …  
    13921402    PXOPT_COPY_STR(config->args, where, "-label",      "skycalRun.label", "==");
    13931403    PXOPT_COPY_STR(config->args, where, "-data_group", "skycalRun.data_group", "LIKE");
     1404    PXOPT_COPY_S16(config->args, where, "-quality",     "skycalResult.quality", "==");
    13941405    PXOPT_COPY_S16(config->args, where, "-fault",      "skycalResult.fault", "==");
     1406
     1407    if (!pxskycellAddWhere(config, where)) {
     1408        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
     1409        return false;
     1410    }
    13951411
    13961412    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    … …  
    14681484    pxAddLabelSearchArgs(config, where, "-filter", "stackRun.filter", "==");
    14691485    PXOPT_COPY_S16(config->args, where, "-fault", "skycalResult.fault", "==");
     1486    if (!pxskycellAddWhere(config, where)) {
     1487        psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments");
     1488        return false;
     1489    }
    14701490
    14711491    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {

  • branches/eam_branches/ipp-20121219/ippTools/src/staticskytoolConfig.c

    r34789 r35096  
    5151    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL);
    5252    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL);
    53     psMetadataAddF32(definebyqueryArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0);
     53    psMetadataAddF32(definebyqueryArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", NAN);
    5454    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL);
    5555    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
    … …  
    103103    psMetadataAddS64(todoArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0);
    104104    psMetadataAddStr(todoArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     105    pxskycellAddArguments(todoArgs);
    105106    psMetadataAddU64(todoArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
    106107    psMetadataAddBool(todoArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
    … …  
    138139    psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0);
    139140    psMetadataAddS16(revertArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
     141    pxskycellAddArguments(revertArgs);
    140142    psMetadataAddBool(revertArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false);
    141143
    … …  
    162164    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL);
    163165    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL);
    164     psMetadataAddF32(defineskycalrunArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0);
     166    psMetadataAddF32(defineskycalrunArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", NAN);
    165167    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL);
    166168    psMetadataAddStr(defineskycalrunArgs,  PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL);
    … …  
    238240    psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0);
    239241    psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0);
     242    pxskycellAddArguments(revertskycalArgs);
    240243    psMetadataAddS16(revertskycalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
    241244    psMetadataAddBool(revertskycalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false);
    … …  
    257260    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL);
    258261    psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
     262    psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-quality",  0, "search by quality", 0);
    259263    psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
     264    pxskycellAddArguments(skycalresultArgs);
    260265    psMetadataAddU64(skycalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit skycalresult set to N items", 0);
    261266    psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);

  • branches/eam_branches/ipp-20121219/ippconfig

  • branches/eam_branches/ipp-20121219/ippconfig/gpc1/psastro.config

    r34512 r35096  
    382382   PSASTRO.USE.MODEL           BOOL FALSE
    383383   PSASTRO.MOSAIC.MODE         BOOL FALSE
     384   DVO.GETSTAR.FIXED.ZEROPT    F32   25.0
    384385
    385386   PSASTRO.CATDIR                       STR     PS1.REF.20120524
    … …  
    394395END
    395396
    396 # initial setup for MD@100 inputs, should work for 50-500 input
     397STATICSKY_CAL   METADATA
     398    REFSTAR_MASK                BOOL FALSE
     399    PSASTRO.SAVE.REFMATCH       BOOL FALSE
     400    PSASTRO.FIX.CHIPS           BOOL FALSE
     401    PSASTRO.USE.MODEL           BOOL FALSE
     402    PSASTRO.MOSAIC.MODE         BOOL FALSE
     403    DVO.GETSTAR.FIXED.ZEROPT    F32   25.0
     404
     405    PSASTRO.CATDIR                       STR     PS1.REF.20120524
     406    ZERO.POINT.USE.MEAN                  BOOL    TRUE
     407
     408    ## for skycell, radius shouldn't need to be as large as 90 pixel default. deeper stacks, crowded fields can be problems
     409    PSASTRO.MATCH.RADIUS.N0 F32    15
     410    PSASTRO.MATCH.RADIUS.N1 F32    10
     411    PSASTRO.MATCH.RADIUS.N2 F32    5
     412    PSASTRO.MATCH.RADIUS.N3 F32    1
     413    PSASTRO.MATCH.RADIUS.N4 F32    0.5
     414    PSASTRO.MATCH.FIT.NITER S32    5
     415
     416    PSASTRO.MAX.ERROR      F32      0.2  # in arcsec units, if larger than 0.2" on skycell then a problem..
     417
     418END
     419
     420
     421# revised setup for MD nightly stack inputs, should work for case of N~2-30
     422# - similar to DEEPCAL except for MIN.NSTAR 5 to attemp more edge skycells if poorer
     423STATICSKY_NIGHTCAL   METADATA
     424   PSASTRO.CATDIR                  STR   PS1.REF.20120524  # reset here to ensure same catalog is used when reprocessing set uniformly
     425   REFSTAR_MASK                    BOOL  FALSE
     426   PSASTRO.SAVE.REFMATCH           BOOL  FALSE
     427   PSASTRO.FIX.CHIPS               BOOL  FALSE
     428   PSASTRO.USE.MODEL               BOOL  FALSE
     429   PSASTRO.MOSAIC.MODE             BOOL  FALSE
     430   ZERO.POINT.USE.MEAN             BOOL  TRUE
     431   PSASTRO.FIELD.PADDING           F32   0.01   # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels
     432   PSASTRO.MAX.NRAW                S32   1000   # include more than MAX.NREF to account for extras like false-positives
     433   PSASTRO.MAX.NREF                S32   500   # if edge skycell ~5% then want >50 to match (hit ~40% in normal field matches)
     434   PSASTRO.MIN.INST.MAG.RAW        F32   -25.0  #
     435   PSASTRO.MAX.INST.MAG.RAW        F32   -5.0
     436   PSASTRO.GRID.SEARCH             BOOL  FALSE  # skycell/stack and astrom well known, when used could cause problems and not help
     437   #PSASTRO.GRID.NRAW.MAX           S32   2000
     438   #PSASTRO.GRID.NREF.MAX           S32   2000
     439   #PSASTRO.GRID.MIN.INST.MAG.RAW   F32   -21.0
     440   #PSASTRO.GRID.MAX.INST.MAG.RAW   F32   -14.0
     441   DVO.GETSTAR.MAX.RHO             F32   6000.0  # slightly larger than MAX.NREF
     442   DVO.GETSTAR.MIN.MAG.INST        F32   -25.0   # set absurd range to cover large range of exptime, then clamp...
     443   DVO.GETSTAR.MIN.MAG             F32   15.0    # only used if keyword problem
     444   REFSTAR_CLAMP_MAG_MIN           F32   15.0    # saturation in MD ~14 even in y-band, so toss out brighter in the list
     445   DVO.GETSTAR.FIXED.ZEROPT        F32   0.0     # not really needed, exptime overly dominates
     446 
     447   ## for skycell/stack astrom well known, radius shouldn't need to be as large as 90 pixel default. in deeper stacks can be problems
     448   PSASTRO.MATCH.RADIUS.N0         F32   5
     449   PSASTRO.MATCH.RADIUS.N1         F32   3
     450   PSASTRO.MATCH.RADIUS.N2         F32   1
     451   PSASTRO.MATCH.RADIUS.N3         F32   0.5
     452   PSASTRO.MATCH.FIT.NITER         S32   4
     453   PSASTRO.MAX.ERROR               F32   0.2  # in arcsec units, still liberal limit for well known astrometry for skycells
     454   PSASTRO.MIN.NSTAR               S32   5    # edges may not have many and require special treatment
     455END
     456
     457
     458# revised setup for DEEP/MDS stacks with range of inputs, should work for N~2(edges)--500 inputs with large MIN/MAX.INST.MAG.RAW
    397459STATICSKY_DEEPCAL   METADATA
    398    REFSTAR_MASK                BOOL FALSE
    399    PSASTRO.SAVE.REFMATCH       BOOL FALSE
    400    PSASTRO.FIX.CHIPS           BOOL FALSE
    401    PSASTRO.USE.MODEL           BOOL FALSE
    402    PSASTRO.MOSAIC.MODE         BOOL FALSE
    403 
    404    PSASTRO.CATDIR                       STR     PS1.REF.20120524
    405    ZERO.POINT.USE.MEAN                  BOOL    TRUE
    406    PSASTRO.MAX.NRAW                     S32     0
    407    PSASTRO.MAX.NREF                     S32     10000
    408    PSASTRO.GRID.NRAW.MAX                S32     2000
    409    PSASTRO.GRID.NREF.MAX                S32     2000
    410    PSASTRO.MIN.INST.MAG.RAW             F32     -19.0
    411    PSASTRO.MAX.INST.MAG.RAW             F32     -14.0
    412    PSASTRO.GRID.MIN.INST.MAG.RAW        F32     -19.0
    413    PSASTRO.GRID.MAX.INST.MAG.RAW        F32     -15.0
    414    DVO.GETSTAR.MIN.MAG.INST             F32     -19.0
    415    DVO.GETSTAR.MIN.MAG                  F32     16.0
    416 
    417 END
    418 
     460   PSASTRO.CATDIR                  STR   PS1.REF.20120524  # reset here to ensure same catalog is used when reprocessing set uniformly
     461   REFSTAR_MASK                    BOOL  FALSE
     462   PSASTRO.SAVE.REFMATCH           BOOL  FALSE
     463   PSASTRO.FIX.CHIPS               BOOL  FALSE
     464   PSASTRO.USE.MODEL               BOOL  FALSE
     465   PSASTRO.MOSAIC.MODE             BOOL  FALSE
     466   ZERO.POINT.USE.MEAN             BOOL  TRUE
     467   PSASTRO.FIELD.PADDING           F32   0.01   # skycells are well defined, don't need large padding. 1% of 6400 is 64 pixels
     468   PSASTRO.MAX.NRAW                S32   1000   # include more than MAX.NREF to account for extras like false-positives
     469   PSASTRO.MAX.NREF                S32    500   # if edge skycell ~10% then try ~50 to match
     470   PSASTRO.MIN.INST.MAG.RAW        F32   -25.0  #
     471   PSASTRO.MAX.INST.MAG.RAW        F32   -5.0
     472   PSASTRO.GRID.SEARCH             BOOL  FALSE  # skycell/stack and astrom well known, when used could cause problems and not help
     473   #PSASTRO.GRID.NRAW.MAX           S32   2000
     474   #PSASTRO.GRID.NREF.MAX           S32   2000
     475   #PSASTRO.GRID.MIN.INST.MAG.RAW   F32   -21.0
     476   #PSASTRO.GRID.MAX.INST.MAG.RAW   F32   -14.0
     477   DVO.GETSTAR.MAX.RHO             F32   6000.0  # N/sqdeg, slightly larger than MAX.NREF
     478   DVO.GETSTAR.MIN.MAG.INST        F32   -25.0   # set absurd range to cover large range of exptime, then clamp...
     479   DVO.GETSTAR.MIN.MAG             F32   15.0    # only used if keyword problem so also using CLAMP_MAG_MIN
     480   REFSTAR_CLAMP_MAG_MIN           F32   15.0    # saturation in MD ~14 even in y-band, so toss out brighter in the list
     481   DVO.GETSTAR.FIXED.ZEROPT        F32   0.0     # not really needed, exptime overly dominates
     482 
     483   ## for skycell/stack astrom well known, radius shouldn't need to be as large as 90 pixel default. in deeper stacks can be problems
     484   PSASTRO.MATCH.RADIUS.N0         F32   5
     485   PSASTRO.MATCH.RADIUS.N1         F32   3
     486   PSASTRO.MATCH.RADIUS.N2         F32   1
     487   PSASTRO.MATCH.RADIUS.N3         F32   0.5
     488   PSASTRO.MATCH.FIT.NITER         S32   4
     489   PSASTRO.MAX.ERROR               F32   0.2  # in arcsec units, still liberal limit for well known astrometry for skycells
     490   PSASTRO.MIN.NSTAR               S32   10   # edges may not have many and require special treatment
     491END
     492

  • branches/eam_branches/ipp-20121219/ippconfig/gpc1/psphot.config

    r34769 r35096  
    5353  PSPHOT.EXT.DIFF.ALTERNATE         BOOL TRUE
    5454  PSPHOT.EXT.DIFF.ALTERNATE.THRESH  F32  1.25
     55
     56  PSPHOT.EXT.FIT.ALL.SOURCES        BOOL TRUE
     57  PSPHOT.EXT.FIT.ALL.THRESH         F32  1000
    5558END
    5659
    … …  
    5962  PSPHOT.EXT.DIFF.ALTERNATE         BOOL TRUE
    6063  PSPHOT.EXT.DIFF.ALTERNATE.THRESH  F32  1.25
     64
     65  PSPHOT.EXT.FIT.ALL.SOURCES        BOOL TRUE
     66  PSPHOT.EXT.FIT.ALL.THRESH         F32  1000
    6167END
    6268

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-mef.mdc

    r34800 r35096  
    214214PPIMAGE.PATTERN         OUTPUT {OUTPUT}.ptn                      PATTERN   NONE       CHIP       TRUE      MEF
    215215PPIMAGE.STATS           OUTPUT {OUTPUT}.stats                    STATS     NONE       FPA        TRUE      MEF
    216 
     216PPIMAGE.BACKMDL              OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits     IMAGE           NONE       CHIP       TRUE      NONE
    217217
    218218## note: these use the same output naming convention since they are used for different ppMerge runs

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-simple.mdc

    r34800 r35096  
    178178PPIMAGE.PATTERN              OUTPUT {OUTPUT}.ptn                  PATTERN         NONE       FPA        TRUE      NONE
    179179PPIMAGE.STATS                OUTPUT {OUTPUT}.stats                STATS           NONE       FPA        TRUE      NONE
    180 
     180PPIMAGE.BACKMDL              OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits     IMAGE           NONE       CHIP       TRUE      NONE
    181181
    182182## note: these use the  same output naming convention since they are used for different ppMerge runs

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/filerules-split.mdc

    r34800 r35096  
    198198PPIMAGE.PATTERN              OUTPUT {OUTPUT}.{CHIP.NAME}.ptn          PATTERN         NONE       CHIP       TRUE      NONE
    199199PPIMAGE.STATS                OUTPUT {OUTPUT}.{CHIP.NAME}.stats        STATS           NONE       CHIP       TRUE      NONE
     200PPIMAGE.BACKMDL              OUTPUT {OUTPUT}.{CHIP.NAME}.mdl.fits     IMAGE           NONE       CHIP       TRUE      NONE
    200201
    201202## note: these use the  same output naming convention since they are used for different ppMerge runs

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/ppImage.config

    r34063 r35096  
    4242MASK.STATS         BOOL    FALSE           # calculate mask statistics.
    4343GAIN.OVERRIDE           BOOL    TRUE            # Override a non-finite gain?
    44 
     44BACKGROUND.CONTINUITY BOOL FALSE           # Apply continuity correction to background models for an FPA.
    4545TILTYSTREAK.BY.CLASS METADATA              # apply the 'tiltystreak' tool
    4646END
    … …  
    11471147  MASK.SATURATED   BOOL    FALSE           # DO NOT Mask the saturated pixels
    11481148  MASK.LOW         BOOL    FALSE           # DO NOT Mask the saturated pixels
     1149END
     1150
     1151# Calculate new background models with continuity correction applied
     1152PPIMAGE_BKGCONT    METADATA
     1153  OVERSCAN         BOOL    FALSE           # Overscan subtraction
     1154  NONLIN           BOOL    FALSE           # Non-linearity correction; not implemented
     1155  BIAS             BOOL    FALSE           # Bias subtraction
     1156  DARK             BOOL    FALSE           # Dark subtraction
     1157  SHUTTER          BOOL    FALSE           # Shutter correction
     1158  FLAT             BOOL    FALSE           # Flat-field normalisation
     1159  MASK             BOOL    FALSE           # Mask bad pixels
     1160  MASK.BUILD       BOOL    FALSE           # Build internal mask?
     1161  FRINGE           BOOL    FALSE           # Fringe subtraction
     1162  PHOTOM           BOOL    FALSE           # Source identification and photometry
     1163  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
     1164  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
     1165  BASE.FITS        BOOL    FALSE           # Save base image?
     1166  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
     1167  BASE.VARIANCE.FITS BOOL    FALSE           # Save base detrended image?
     1168  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image?
     1169  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image?
     1170  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image?
     1171  BIN1.FITS        BOOL    FALSE           # Save 1st binned chip image?
     1172  BIN2.FITS        BOOL    FALSE           # Save 2nd binned chip image?
     1173  BIN1.JPEG        BOOL    FALSE           # Save 1st binned jpeg?
     1174  BIN2.JPEG        BOOL    FALSE           # Save 2nd binned jpeg?
     1175  BIN1.XBIN        S32     1               # Image is already binned
     1176  BIN1.YBIN        S32     1               # Image is already binned
     1177  BIN2.XBIN        S32     1               # Image is already binned
     1178  BIN2.YBIN        S32     1               # Image is already binned
     1179  MASK.SATURATED   BOOL    FALSE           # DO NOT Mask the saturated pixels
     1180  MASK.LOW         BOOL    FALSE           # DO NOT Mask the saturated pixels
     1181  BACKGROUND.CONTINUITY BOOL TRUE
     1182  PATTERN.CONTINUITY.WIDTH S32 2
    11491183END
    11501184

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/ppSub.config

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/psastro.config

    r34301 r35096  
    190190DVO.GETSTAR.MIN.MAG.INST      F32     -15.0
    191191
     192# if non-zero use as the zero point for computing magnitude limits for getstar
     193# by default use the ZEROPT in the PHOTCODE.DATA
     194DVO.GETSTAR.FIXED.ZEROPT      F32       0.0     
     195
    192196ZERO.POINT.BY.EXPOSURE          BOOL     TRUE
    193197ZERO.POINT.USE.MEAN             BOOL     TRUE
    … …  
    268272
    269273STATICSKY_CAL   METADATA
    270     REFSTAR_MASK                BOOL FALSE
    271     PSASTRO.SAVE.REFMATCH       BOOL FALSE
    272     PSASTRO.FIX.CHIPS           BOOL FALSE
    273     PSASTRO.USE.MODEL           BOOL FALSE
    274     PSASTRO.MOSAIC.MODE         BOOL FALSE
    275274END
    276275
    … …  
    278277END
    279278
     279STATICSKY_NIGHTCAL   METADATA
     280END
     281
    280282STATICSKY_DEEPCAL   METADATA
    281283END

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/psphot.config

    r34903 r35096  
    347347PSPHOT.EXT.DIFF.ALTERNATE           BOOL  FALSE           # use alternate extended source calculation; designed for diff trails
    348348PSPHOT.EXT.DIFF.ALTERNATE.THRESH    F32   1.5             # threshold for alternate extended source calculations
     349PSPHOT.EXT.FIT.ALL.SOURCES          BOOL  FALSE           # Apply extended object fits to all sources
     350PSPHOT.EXT.FIT.ALL.THRESH           F32   1000            # Set density threshold above which to not fit all sources
    349351PSPHOT.CR.GROW                      S32   1               # Number of pixels to grow CR mask
    350352PSPHOT.CR.NSIGMA.SOFTEN             F32   0.0025          # Softening parameter for weights

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/pswarp.config

    r34800 r35096  
    3434# Background restoration
    3535BACKGROUND      METADATA
    36         SOURCES         BOOL    FALSE           # Write source list for warped image?
    37         PSF             BOOL    FALSE           # Measure PSF for warped image?
     36        SOURCES             BOOL        FALSE           # Write source list for warped image?
     37        PSF                 BOOL        FALSE           # Measure PSF for warped image?
     38        BACKGROUND.MODEL    BOOL        FALSE           # do not model background
    3839END

  • branches/eam_branches/ipp-20121219/ippconfig/recipes/reductionClasses.mdc

    r34903 r35096  
    518518      STACK_PPSUB     STR      STACK
    519519      STACK_PSPHOT    STR      STACK
     520      STACKPHOT_PSPHOT        STR     STACKPHOT
     521      STATICSKY_CALIBRATION   STR     STATICSKY_NIGHTCAL
    520522END
    521523

  • branches/eam_branches/ipp-20121219/ppImage/src

  • branches/eam_branches/ipp-20121219/ppImage/src/Makefile.am

    r33590 r35096  
    4444        ppImageRebinReadout.c \
    4545        ppImageMosaic.c \
     46        ppImageMosaicBackground.c \
    4647        ppImageMaskStats.c \
    4748        ppImagePhotom.c \

  • branches/eam_branches/ipp-20121219/ppImage/src/ppImage.h

    r33848 r35096  
    4141    bool doPatternCell;                 // Cell pattern correction
    4242    bool doPatternContinuity;           // Cell continuity correction
     43    bool doBackgroundContinuity;        // Do mosaic continuity correction
    4344    bool doFringe;                      // Fringe subtraction
    4445    bool doPhotom;                      // Source identification and photometry
    … …  
    5051    bool checkNoise;                    // measure cell-level variance
    5152    bool applyParity;                   // Apply Cell parities
    52 
    53   bool doMaskStats;                      // Calculate mask statistics
     53    bool doMaskStats;                   // Calculate mask statistics
    5454 
    5555    bool doCrosstalkMeasure;            // measure crosstalk signal
    … …  
    183183bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options);
    184184
     185// Do background continuity step
     186bool ppImageMosaicBackground(pmConfig *config, const ppImageOptions *options);
     187
    185188// Record which detrend file was used for the detrending
    186189bool ppImageDetrendRecord(

  • branches/eam_branches/ipp-20121219/ppImage/src/ppImageLoop.c

    r33590 r35096  
    4141        ESCAPE("load failure for FPA");
    4242    }
    43 
     43   
    4444    pmChip *chip;                       // Chip from FPA
    4545    while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) {
    … …  
    233233            ESCAPE("Unable to bin chip (level 2).");
    234234        }
    235 
     235        if (options->doBackgroundContinuity) {
     236          pmFPAfile *out = psMetadataLookupPtr(NULL, config->files, "PPIMAGE.BACKMDL");
     237          pmFPAfileCopyView(out->fpa,out->src,view);
     238        }
    236239        // Close cells (XXX shouldn't pmFPAfileClose iterate down as needed?)
    237240        view->cell = -1;
    … …  
    251254    }
    252255
     256    // Do background model continuity updates
     257    if (!ppImageMosaicBackground(config, options )) {
     258      ESCAPE("failure in Background Mosaic");
     259    }
     260   
    253261    // generate the full-scale FPA mosaic
    254262    if (!ppImageMosaicFPA(config, options, "PPIMAGE.OUTPUT.FPA1", "PPIMAGE.BIN1")) {

  • branches/eam_branches/ipp-20121219/ppImage/src/ppImageOptions.c

    r33848 r35096  
    3535    options->doPatternCell   = false;   // Cell pattern correction
    3636    options->doPatternContinuity = false; // Cell continuity correction
     37    options->doBackgroundContinuity = false; // Chip level background continuity correction
    3738    options->doFringe        = false;   // Fringe subtraction
    3839    options->doPhotom        = false;   // Source identification and photometry
    … …  
    429430    }
    430431
     432    // Option to enable the background continuity
     433    options->doBackgroundContinuity = psMetadataLookupBool(NULL, recipe, "BACKGROUND.CONTINUITY");
     434
    431435
    432436    // Remnance options

  • branches/eam_branches/ipp-20121219/ppImage/src/ppImageParseCamera.c

    r33848 r35096  
    303303    }
    304304
     305
    305306    // chipImage    -> psphotInput  (pmFPAfileDefineFromFile)       : fpa is constructed
    306307    // psphotInput  -> psphotOutput (pmFPAfileDefineOutputFromFile) : fpa is equated
    … …  
    387388    // the input data is the same as the outImage data : force the free levels to match
    388389    input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel);
     390
     391    // define the continuity corrected background model files
     392    if (options->doBackgroundContinuity) {
     393      pmFPAfile *bkgMosaicModel = pmFPAfileDefineFromFPA(config,input->fpa, 1, 1,  "PPIMAGE.BACKMDL");
     394      if (!bkgMosaicModel) {
     395        psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BACKMDL"));
     396        psFree(options);
     397        return NULL;
     398      }
     399      if (bkgMosaicModel->type != PM_FPA_FILE_IMAGE) {
     400        psError(PS_ERR_IO, true, "PPIMAGE.BACKMDL is not of type IMAGE");
     401        psFree(options);
     402        return NULL;
     403
     404      }
     405      bkgMosaicModel->save = options->doBackgroundContinuity;
     406      //      bkgMosaicModel->freeLevel = PS_MIN(bkgMosaicModel->freeLevel, PM_FPA_LEVEL_FPA);
     407      bkgMosaicModel->freeLevel = PM_FPA_LEVEL_FPA;
     408      bkgMosaicModel->dataLevel = bkgMosaicModel->dataLevel;
     409      bkgMosaicModel->fileLevel = PS_MIN(bkgMosaicModel->fileLevel, bkgMosaicModel->dataLevel);
     410    }
    389411
    390412    // define the binned target files (which may just be carriers for some camera configurations)
    … …  
    497519        chipVariance->save = false;
    498520    }
    499 
     521   
    500522    if (psTraceGetLevel("ppImage.config") > 0) {
    501523        // Get a look inside all the files.

  • branches/eam_branches/ipp-20121219/ppStack/src/ppStackCamera.c

    r34800 r35096  
    4747    }
    4848    bool convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images before stack?
    49 
     49    bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL");
     50   
    5051    psArray *runImages = pmFPAfileDefineMultipleFromRun(&status, NULL, config,
    5152                                                        "PPSTACK.INPUT"); // Input images from previous run
    … …  
    217218
    218219            // Grab bkgmodel information here
    219             if (!bkgmodel ||  strlen(bkgmodel) == 0) {
    220               // We have no background models.
     220            if (doBackground) {
     221              if (!bkgmodel ||  strlen(bkgmodel) == 0) {
     222                // We have no background models.
     223              }
     224              else {
     225                pmFPAfile *inputBKG = defineFile(config,NULL,"PPSTACK.INPUT.BKGMODEL",bkgmodel,
     226                                                 PM_FPA_FILE_IMAGE);
     227                if (!inputBKG) {
     228                  // We failed to generate an pmFPAfile, so disable the background construction and continue.
     229                  doBackground = false;
     230                  psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE,
     231                                    "Do photometry on stacked image?", false);
     232
     233#if (0)
     234                  psError(psErrorCodeLast(), false,
     235                          "Unable to define file from bkgmodel %d (%s)",i,bkgmodel);
     236                  return(false);
     237#endif
     238                }
     239              }// End bkgmodel
    221240            }
    222             else {
    223               pmFPAfile *inputBKG = defineFile(config,NULL,"PPSTACK.INPUT.BKGMODEL",bkgmodel,
    224                                                PM_FPA_FILE_IMAGE);
    225               if (!inputBKG) {
    226                 psError(psErrorCodeLast(), false,
    227                         "Unable to define file from bkgmodel %d (%s)",i,bkgmodel);
    228                 return(false);
    229               }
    230             }// End bkgmodel
    231241
    232242

  • branches/eam_branches/ipp-20121219/ppStack/src/ppStackCombineAlternate.c

    r34848 r35096  
    198198      offset = (Sy * Sxx - Sx * Sxy) / D;
    199199      scale  = (S * Sxy - Sx * Sy) / D;
    200       fprintf(stderr,"Scales: %d %g %g\n",i,offset,scale);
     200      fprintf(stderr,"Scales: %d %g %g %g %g %g %g %g %g\n",i,offset,scale,D,Sx,Sy,Sxx,Syy,Sxy);
    201201      // Apply scaling factors
    202202      for (int y = minInputRows; y < maxInputRows; y++) {

  • branches/eam_branches/ipp-20121219/psLib/src/imageops/psImageInterpolate.c

    r34800 r35096  
    3131#include "psImageConvolve.h"
    3232
    33 # define IS_BILIN_SEPARABLE 1
     33# define IS_BILIN_SEPARABLE 0
    3434
    3535#include "psImageInterpolate.h"

  • branches/eam_branches/ipp-20121219/psModules

  • branches/eam_branches/ipp-20121219/psModules/src/camera/pmFPAMaskWeight.c

    r31451 r35096  
    538538    psF32 **imageData = readout->image->data.F32;// Dereference image
    539539    psF32 **varianceData = readout->variance ? readout->variance->data.F32 : NULL; // Dereference variance map
    540 
     540    float maskFrac = 0.0;
    541541    for (int y = 0; y < numRows; y++) {
    542542        for (int x = 0; x < numCols; x++) {
    543543            if (maskData[y][x] & maskVal) {
     544                maskFrac += 1;
    544545                imageData[y][x] = NAN;
    545546                if (varianceData) {
    … …  
    549550        }
    550551    }
    551 
     552    maskFrac = maskFrac / (1.0 * numRows * numCols);
     553    psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "READOUT.MASK.FRAC", PS_META_REPLACE,
     554                     "fraction of pixels masked by pmReadoutMaskApply",maskFrac);
    552555    return true;
    553556}

  • branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.c

    r33340 r35096  
    938938}
    939939
    940 
     940// This should reuse code more efficiently
     941bool pmPatternContinuityBackground(pmFPAfile *in, pmFPAfile *out,  psStatsOptions bgStat, psStatsOptions cellStat,
     942                                   psImageMaskType maskVal, psImageMaskType maskBad, int edgeWidth)
     943{
     944    PS_ASSERT_PTR_NON_NULL(in, false);
     945    PS_ASSERT_PTR_NON_NULL(out, false);
     946
     947    int numChips = out->fpa->chips->n;            // Number of cells
     948
     949    psVector *meanMask = psVectorAlloc(numChips, PS_TYPE_VECTOR_MASK); // Mask for means
     950    psVectorInit(meanMask, 0);
     951
     952    // Mask bits
     953    enum {
     954        PM_PATTERN_IGNORE = 0x01,       // Ignore this cell
     955        PM_PATTERN_TWEAK  = 0x02,       // Tweak this cell
     956        PM_PATTERN_ERROR  = 0x04,       // Error in calculating background
     957        PM_PATTERN_ALL    = 0xFF,       // All causes
     958    };
     959
     960    // Measure mean of each cell edge, and use that to determine the cell offsets.
     961    psStatsOptions stat = cellStat;     // Define which statistic to use.
     962   
     963    psStats *bgStats = psStatsAlloc(stat); // Statistics on background
     964    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
     965
     966    psRegion region = {0,0,0,0};
     967
     968    /* These images hold the edge data for the OTA structure.  */
     969    psImage *A = psImageAlloc(8,8,PS_TYPE_F64); // Top edge
     970    psImage *B = psImageAlloc(8,8,PS_TYPE_F64); // Bottom edge
     971    psImage *C = psImageAlloc(8,8,PS_TYPE_F64); // Right edge
     972    psImage *D = psImageAlloc(8,8,PS_TYPE_F64); // Left edge
     973    psImageInit(A,0.0);
     974    psImageInit(B,0.0);
     975    psImageInit(C,0.0);
     976    psImageInit(D,0.0);
     977
     978    // Corners don't exist.
     979    A->data.F64[0][0] = NAN;
     980    B->data.F64[0][0] = NAN;
     981    C->data.F64[0][0] = NAN;
     982    D->data.F64[0][0] = NAN;
     983    A->data.F64[7][0] = NAN;
     984    B->data.F64[7][0] = NAN;
     985    C->data.F64[7][0] = NAN;
     986    D->data.F64[7][0] = NAN;
     987    A->data.F64[0][7] = NAN;
     988    B->data.F64[0][7] = NAN;
     989    C->data.F64[0][7] = NAN;
     990    D->data.F64[0][7] = NAN;
     991    A->data.F64[7][7] = NAN;
     992    B->data.F64[7][7] = NAN;
     993    C->data.F64[7][7] = NAN;
     994    D->data.F64[7][7] = NAN;
     995   
     996    for (int i = 0; i < numChips; i++) {
     997      pmChip *chip = out->fpa->chips->data[i];
     998      pmCell *cell = chip->cells->data[0]; // Cell of interest
     999      pmReadout *ro = cell->readouts->data[0]; // Readout of interest
     1000
     1001      psStatsInit(bgStats);
     1002
     1003      // Convert cell iterator i into an xy coordinate on the grid of cells
     1004      // This is wrong for chips
     1005      const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME");
     1006      int xParity          = psMetadataLookupS16(NULL,chip->concepts, "CHIP.XPARITY");
     1007      int yParity          = psMetadataLookupS16(NULL,chip->concepts, "CHIP.YPARITY");
     1008      int x = chipName[2] - '0';
     1009      int y = chipName[3] - '0';
     1010
     1011     
     1012      for (int j = 0; j < 4; j++) {
     1013        if (j == 0) {  // Region B
     1014          region = psRegionSet(0,ro->image->numCols,
     1015                               0,edgeWidth);
     1016        }
     1017        else if (j == 1) { // Region A
     1018          region = psRegionSet(0,ro->image->numCols,
     1019                               ro->image->numRows - edgeWidth,ro->image->numRows);
     1020        }
     1021        else if (j == 2) { // Region D
     1022          region = psRegionSet(0,edgeWidth,
     1023                               0,ro->image->numRows);
     1024        }
     1025        else if (j == 3) { // Region C
     1026          region = psRegionSet(ro->image->numCols - edgeWidth,ro->image->numCols,
     1027                               0,ro->image->numRows);
     1028        }
     1029        psImage *subset  = psImageSubset(ro->image,region);
     1030
     1031        if (!psImageBackground(bgStats, NULL, subset, NULL, maskVal, rng)) {
     1032          psWarning("Unable to measure background for cell %d on edge %d\n", i, j);
     1033          psErrorClear();
     1034          meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PM_PATTERN_ERROR;
     1035          if (j == 0)      { B->data.F64[y][x] = NAN; }
     1036          else if (j == 1) { A->data.F64[y][x] = NAN; }
     1037          else if (j == 2) { C->data.F64[y][x] = NAN; }
     1038          else if (j == 3) { D->data.F64[y][x] = NAN; }
     1039          psFree(subset);
     1040          continue; // Move on to next edge, as only part of this cell may be a problem
     1041        }
     1042       
     1043        // If the returned value is zero, assume something is wrong.  Do I still need this?
     1044        if (psStatsGetValue(bgStats,stat) < 1e-6) {
     1045          if (j == 0)      { B->data.F64[y][x] = NAN; }
     1046          else if (j == 1) { A->data.F64[y][x] = NAN; }
     1047          else if (j == 2) { C->data.F64[y][x] = NAN; }
     1048          else if (j == 3) { D->data.F64[y][x] = NAN; }
     1049        }
     1050        // If we have an error for this cell/edge, make sure we mask the value
     1051        if (meanMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PM_PATTERN_ERROR) {
     1052          if (j == 0)      { B->data.F64[y][x] = NAN; }
     1053          else if (j == 1) { A->data.F64[y][x] = NAN; }
     1054          else if (j == 2) { C->data.F64[y][x] = NAN; }
     1055          else if (j == 3) { D->data.F64[y][x] = NAN; }
     1056        }
     1057        else { // Set the value to match what we got from the edge box.
     1058          if (xParity == -1) {
     1059            if (j == 2)      { C->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1060            else if (j == 3) { D->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1061          }
     1062          if (xParity == 1) {
     1063            if (j == 3)      { C->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1064            else if (j == 2) { D->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1065          }
     1066          if (yParity == 1) {
     1067            if (j == 0)      { B->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1068            else if (j == 1) { A->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1069          }
     1070          if (yParity == -1) {
     1071            if (j == 1)      { B->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1072            else if (j == 0) { A->data.F64[y][x] = psStatsGetValue(bgStats,stat); }
     1073          }
     1074         
     1075        }
     1076#if (0)
     1077        for (int u = 0; u < subset->numCols; u++) {
     1078          for (int v = 0; v < subset->numRows; v++) {
     1079            psTrace("psModules.detrend.cont",10,"BOX: %d %d (%d %d) (%d %d) %f %d",
     1080                    i,j,x,y,u,v,subset->data.F32[v][u],0);
     1081          }
     1082        }
     1083#endif 
     1084        psFree(subset);
     1085       
     1086      }
     1087      psTrace("psModules.detrend.cont",5, "OTA: %d (%d %d) (%d %d) A: %f B: %f C: %f D: %f",
     1088              i,x,y,xParity,yParity,
     1089              A->data.F64[y][x],B->data.F64[y][x],C->data.F64[y][x],D->data.F64[y][x]);         
     1090    }
     1091    psFree(bgStats);
     1092    psFree(rng);
     1093   
     1094    // We've now allocated all the edge values, so we can now minimize the offsets.
     1095    // This involves solving the equation A x = b, where
     1096    // A is the (64x64 for GPC1) matrix containing the edges that match for each cell
     1097    // x is the solution vector
     1098    // b is the combination of offsets across each cell boundary for each cell.
     1099    // Below "XX" is used as the matrix A, and "solution" is used as both b and x
     1100    //   (due to the way psMatrixLUSolve operates).
     1101    psVector *solution = psVectorAlloc(64,PS_TYPE_F64);
     1102    psImage  *XX       = psImageAlloc(64,64,PS_TYPE_F64);
     1103    psVectorInit(solution,0.0);
     1104    psImageInit(XX,0.0);
     1105
     1106
     1107    for (int i = 0; i < numChips; i++) {
     1108      // Accumulate all the possible edge differences we can for this cell.
     1109      // As we do so, make a note of the correlations by incrementing the element of the matrix.
     1110      pmChip *chip = out->fpa->chips->data[i];
     1111      const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME");
     1112      int x = chipName[2] - '0';
     1113      int y = chipName[3] - '0';
     1114
     1115      int j;
     1116      int k = 8 * x + y;
     1117      double critical_value = 0.0;
     1118
     1119      if (x + 1 < 8) {  // We have a neighbor adjacent in the +x direction
     1120        j = 8 * (x + 1) + y; // Determine that neighbor's index
     1121       
     1122        if (fabs(C->data.F64[y][x]) > fabs(D->data.F64[y][x+1])) {
     1123          critical_value = 2.0 * fabs(D->data.F64[y][x+1]);
     1124        }
     1125        else {
     1126          critical_value = 2.0 * fabs(C->data.F64[y][x]);
     1127        }
     1128        if (critical_value < 25) { critical_value = 25; }
     1129        psTrace("psModules.detrend.cont",5,"CmD %d %d %d %d %g %g %g", // diagnostic
     1130                i,x,y,j,
     1131                C->data.F64[y][x],
     1132                D->data.F64[y][x+1],
     1133                critical_value
     1134                );
     1135        if (// If there are no errors with the neighbor,
     1136            (isfinite(C->data.F64[y][x]))&&(isfinite(D->data.F64[y][x+1]))&&     // and all edges have valid values,
     1137            (fabs(C->data.F64[y][x] - D->data.F64[y][x+1]) < critical_value)     // and there are no large discontinuities,
     1138            ) {   
     1139          solution->data.F64[k] += C->data.F64[y][x] - D->data.F64[y][x+1];     // Take the difference
     1140          XX->data.F64[k][k] += 1;                                              // increment our relation with ourself
     1141          XX->data.F64[k][j] += -1;                                             // decrement our relation with the neighbor
     1142        }
     1143      }
     1144      if (x - 1 > -1) { // etc.
     1145        j = 8 * (x - 1) + y;
     1146        if (fabs(C->data.F64[y][x-1]) > fabs(D->data.F64[y][x])) {
     1147          critical_value = 2.0 * fabs(D->data.F64[y][x]);
     1148        }
     1149        else {
     1150          critical_value = 2.0 * fabs(C->data.F64[y][x-1]);
     1151        }
     1152        if (critical_value < 25) { critical_value = 25; }
     1153        psTrace("psModules.detrend.cont",5,"DmC %d %d %d %d %g %g %g",
     1154                i,x,y,j,
     1155                D->data.F64[y][x],
     1156                C->data.F64[y][x-1],
     1157                critical_value
     1158                );
     1159
     1160        if (
     1161            (isfinite(D->data.F64[y][x]))&&(isfinite(C->data.F64[y][x-1]))&&
     1162            (fabs(D->data.F64[y][x] - C->data.F64[y][x-1]) < critical_value)
     1163            ) {
     1164          solution->data.F64[k] += D->data.F64[y][x] - C->data.F64[y][x-1];
     1165          XX->data.F64[k][k] += 1;
     1166          XX->data.F64[k][j] += -1;
     1167        }
     1168      }
     1169      if (y + 1 < 8) {
     1170        j = 8 * x + (y + 1);
     1171        psTrace("psModules.detrend.cont",5,"AmB %d %d %d %d %g %g",
     1172                i,x,y,j,
     1173                A->data.F64[y][x],
     1174                B->data.F64[y+1][x]
     1175                );
     1176        if (fabs(A->data.F64[y][x]) > fabs(B->data.F64[y+1][x])) {
     1177          critical_value = 2.0 * fabs(B->data.F64[y+1][x]);
     1178        }
     1179        else {
     1180          critical_value = 2.0 * fabs(A->data.F64[y][x]);
     1181        }
     1182        if (critical_value < 25) { critical_value = 25; }
     1183        if (
     1184            (isfinite(A->data.F64[y][x]))&&(isfinite(B->data.F64[y+1][x]))&&
     1185            (fabs(A->data.F64[y][x] - B->data.F64[y+1][x]) < critical_value)
     1186            ) {
     1187          solution->data.F64[k] += A->data.F64[y][x] - B->data.F64[y+1][x];
     1188          XX->data.F64[k][k] += 1;
     1189          XX->data.F64[k][j] += -1;
     1190        }
     1191      }
     1192      if (y - 1 > -1) {
     1193        j = 8 * x +  (y - 1);
     1194        psTrace("psModules.detrend.cont",5,"BmA %d %d %d %d %g %g",
     1195                i,x,y,j,
     1196                B->data.F64[y][x],
     1197                A->data.F64[y-1][x]
     1198                );
     1199        if (fabs(A->data.F64[y-1][x]) > fabs(B->data.F64[y][x])) {
     1200          critical_value = 2.0 * fabs(B->data.F64[y][x]);
     1201        }
     1202        else {
     1203          critical_value = 2.0 * fabs(A->data.F64[y-1][x]);
     1204        }
     1205        if (critical_value < 25) { critical_value = 25; }
     1206        if (
     1207            (isfinite(B->data.F64[y][x]))&&(isfinite(A->data.F64[y-1][x]))&&
     1208            (fabs(B->data.F64[y][x] - A->data.F64[y-1][x]) < critical_value)
     1209            ) {
     1210          solution->data.F64[k] += B->data.F64[y][x] - A->data.F64[y-1][x];
     1211          XX->data.F64[k][k] += 1;
     1212          XX->data.F64[k][j] += -1;
     1213        }
     1214      }
     1215    }
     1216    double max_XX = 0;
     1217    double solution_V = 0;
     1218    int i_peak = -1;
     1219    for (int i = 0; i < numChips + 4; i++) { // If any cells have no value of themself, set the matrix to 1.0.
     1220      if (XX->data.F64[i][i] == 0.0) {
     1221        XX->data.F64[i][i] = 1.0;
     1222      }
     1223      if (XX->data.F64[i][i] > max_XX) {
     1224        max_XX = XX->data.F64[i][i];
     1225        solution_V = solution->data.F64[i];
     1226        i_peak = i;
     1227      }
     1228    }
     1229    psTrace("psModules.detrend.cont",5,"fixed point: %d %g\n",
     1230            i_peak,solution_V);
     1231   
     1232#if (1)
     1233    for (int i = 0; i < numChips + 4; i++) { // print matrix A
     1234      psTrace("psModules.detrend.cont",5,"A: %3d % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f % 2.0f",
     1235              i,
     1236              XX->data.F64[i][0],             XX->data.F64[i][1],             XX->data.F64[i][2],             XX->data.F64[i][3],
     1237              XX->data.F64[i][4],             XX->data.F64[i][5],             XX->data.F64[i][6],             XX->data.F64[i][7],
     1238              XX->data.F64[i][8],             XX->data.F64[i][9],             XX->data.F64[i][10],            XX->data.F64[i][11],
     1239              XX->data.F64[i][12],            XX->data.F64[i][13],            XX->data.F64[i][14],            XX->data.F64[i][15],
     1240              XX->data.F64[i][16],            XX->data.F64[i][17],            XX->data.F64[i][18],            XX->data.F64[i][19],
     1241              XX->data.F64[i][20],            XX->data.F64[i][21],            XX->data.F64[i][22],            XX->data.F64[i][23],
     1242              XX->data.F64[i][24],            XX->data.F64[i][25],            XX->data.F64[i][26],            XX->data.F64[i][27],
     1243              XX->data.F64[i][28],            XX->data.F64[i][29],            XX->data.F64[i][30],            XX->data.F64[i][31],
     1244              XX->data.F64[i][32],            XX->data.F64[i][33],            XX->data.F64[i][34],            XX->data.F64[i][35],
     1245              XX->data.F64[i][36],            XX->data.F64[i][37],            XX->data.F64[i][38],            XX->data.F64[i][39],
     1246              XX->data.F64[i][40],            XX->data.F64[i][41],            XX->data.F64[i][42],            XX->data.F64[i][43],
     1247              XX->data.F64[i][44],            XX->data.F64[i][45],            XX->data.F64[i][46],            XX->data.F64[i][47],
     1248              XX->data.F64[i][48],            XX->data.F64[i][49],            XX->data.F64[i][50],            XX->data.F64[i][51],
     1249              XX->data.F64[i][52],            XX->data.F64[i][53],            XX->data.F64[i][54],            XX->data.F64[i][55],
     1250              XX->data.F64[i][56],            XX->data.F64[i][57],            XX->data.F64[i][58],            XX->data.F64[i][59],
     1251              XX->data.F64[i][60],            XX->data.F64[i][61],            XX->data.F64[i][62],            XX->data.F64[i][63]
     1252              );
     1253    }
     1254
     1255    for (int i = 0; i < numChips + 4; i++) { // print vector b
     1256      psTrace("psModules.detrend.cont",5,"b: %d %f",
     1257              i,
     1258              solution->data.F64[i]
     1259              );
     1260    }
     1261#endif   
     1262   
     1263    // Solve the Ax=b equation
     1264    //    psMatrixLUSolve(XX,solution);
     1265    psMatrixGJSolve(XX,solution);
     1266#if (1)
     1267    for (int i = 0; i < numChips + 4; i++) { // print vector b
     1268      psTrace("psModules.detrend.cont",5,"x: %d %f",
     1269              i,
     1270              solution->data.F64[i]
     1271              );
     1272    }
     1273#endif
     1274   
     1275    // Cleanup
     1276    psFree(XX);
     1277    psFree(A);
     1278    psFree(B);
     1279    psFree(C);
     1280    psFree(D);
     1281
     1282    // Correct cells based on the offsets calculated, and store the result in the analysis metadata.
     1283    for (int i = 0; i < numChips; i++) {
     1284        pmChip *chip = out->fpa->chips->data[i];
     1285        pmCell *cell = chip->cells->data[0]; // Cell of interest
     1286        pmReadout *ro = cell->readouts->data[0]; // Readout of interest
     1287        const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME");
     1288        int x = chipName[2] - '0';
     1289        int y = chipName[3] - '0';
     1290        int k = 8 * x + y;
     1291        float correction = solution->data.F64[k];
     1292        psLogMsg("psModules.detrend", PS_LOG_DETAIL, "Correcting background of chip %s by %f",
     1293                 chipName, correction);
     1294        psBinaryOp(ro->image, ro->image, "-", psScalarAlloc(correction, PS_TYPE_F32));
     1295        psMetadataAddF32(ro->analysis, PS_LIST_TAIL, PM_PATTERN_CELL_CORRECTION, PS_META_REPLACE,
     1296                         "Pattern chip correction solution", correction);
     1297    }
     1298
     1299    psFree(solution);
     1300    psFree(meanMask);
     1301
     1302    return true;
     1303}

  • branches/eam_branches/ipp-20121219/psModules/src/detrend/pmPattern.h

    r33243 r35096  
    1414#include <pslib.h>
    1515#include <pmFPA.h>
     16#include <pmFPAview.h>
     17#include <pmFPAfile.h>
     18#include <pmFPAfileFitsIO.h>
     19#include <pmFPAHeader.h>
     20#include <pmHDU.h>
     21#include <pmHDUUtils.h>
    1622
    1723/// @addtogroup detrend Detrend Creation and Application
    … …  
    6571    );
    6672
     73bool pmPatternContinuityBackground(pmFPAfile *in,
     74                                   pmFPAfile *out,
     75                                   psStatsOptions bgStat,
     76                                   psStatsOptions cellStat,
     77                                   psImageMaskType maskVal,
     78                                   psImageMaskType maskBad,
     79                                   int edgeWidth);
     80
    6781/// Apply previously measured cell pattern correction
    6882bool pmPatternContinuityApply(pmReadout *ro,          ///< Readout to correct

  • branches/eam_branches/ipp-20121219/psModules/src/imcombine/pmStack.c

    r34842 r35096  
    13851385
    13861386  psVector *pixelData = psVectorAlloc(input->n,PS_TYPE_F32);
     1387  psVector *pixelMask = psVectorAlloc(input->n,PS_TYPE_VECTOR_MASK);
    13871388  psStats  *stats     = psStatsAlloc(PS_STAT_ROBUST_MEDIAN);
    13881389
    … …  
    13921393        pmReadout *ro  = stack->data[i];
    13931394        psImage *image = ro->image;
     1395        pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0;
    13941396        pixelData->data.F32[i] = image->data.F32[y][x];
     1397        if (isfinite(image->data.F32[y][x])&&
     1398            (fabs(image->data.F32[y][x]) < 1e5)) {
     1399          pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0;
     1400        }
     1401        else {
     1402          pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 1;
     1403        }
     1404#if (0)
     1405      if ((x == 59)&&(y > 40)&&(y < 50)) {
     1406          fprintf(stderr,"%d %d %d %d %g\n",
     1407                  x,y,i,pixelMask->data.PS_TYPE_VECTOR_MASK_DATA[i],pixelData->data.F32[i]);
     1408        }
     1409#endif
    13951410      }
    1396       if (!psVectorStats(stats,pixelData,NULL,NULL,0)) {
     1411      if (!psVectorStats(stats,pixelData,NULL,pixelMask,1)) {
    13971412        psError(PS_ERR_UNKNOWN, false, "Unable to calculate median");
    13981413        psFree(stats);
    13991414        psFree(pixelData);
     1415        psFree(pixelMask);
    14001416        psFree(stack);
    14011417        return(false);
    … …  
    14031419      combined->image->data.F32[y][x] = stats->robustMedian;
    14041420      combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0;
     1421#if (0)
     1422      if ((x == 59)&&(y > 40)&&(y < 50)) {
     1423        fprintf(stderr,"%d %d %d %d %g\n",
     1424                x,y,-1,combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x],
     1425                combined->image->data.F32[y][x]);
     1426      }
     1427#endif
    14051428    }
    14061429  }
    … …  
    14081431  psFree(stats);
    14091432  psFree(pixelData);
     1433  psFree(pixelMask);
    14101434  psFree(stack);
    14111435  return (true);

  • branches/eam_branches/ipp-20121219/psModules/src/objects/pmSourceIO_CMF.c.in

    r34564 r35096  
    830830            psMetadataAddF32 (row, PS_LIST_TAIL, "KRON_MAG",         0, "Kron Mag",                                   kronMag);
    831831
    832             @PS1_DV1,PS1_DV2@ psMetadataAddF32 (row, PS_LIST_TAIL, "NPARAMS",          0, "number of model parameters",                 model->params->n);
     832            @PS1_DV0,PS1_DV1@ psMetadataAddF32 (row, PS_LIST_TAIL, "NPARAMS",          0, "number of model parameters",                 model->params->n);
    833833            @ALL,!PS1_DV0,!PS1_DV1@ psMetadataAddS32 (row, PS_LIST_TAIL, "NPARAMS",          0, "number of model parameters",                 model->params->n);
    834834            psMetadataAddStr (row, PS_LIST_TAIL, "MODEL_TYPE",       0, "name of model",                              pmModelClassGetName (model->type));

  • branches/eam_branches/ipp-20121219/psastro/src/psastroChooseRefstars.c

    r34301 r35096  
    6262
    6363    int nMax = psMetadataLookupS32 (&status, recipe, "PSASTRO.MAX.NREF");
    64     psF32 clampMagMin = psMetadataLookupS32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN");
    65     if (!status) clampMagMin = -5;
     64    psF32 clampMagMin = psMetadataLookupF32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN");
     65    if (!status) clampMagMin = -5.0;
    6666
    6767    // de-activate all files except PSASTRO.REFSTARS

  • branches/eam_branches/ipp-20121219/psastro/src/psastroLoadRefstars.c

    r33643 r35096  
    363363    if (!status) ESCAPE ("missing DVO.GETSTAR.MIN.MAG.INST");
    364364
     365    // if non zero override the zero point in the PHOTCODE.DATA with this value
     366    float fixedzeropt = psMetadataLookupF32(&status, recipe, "DVO.GETSTAR.FIXED.ZEROPT");
     367
    365368    // PHOTCODE.DATA is a multi of metadata items
    366369    psListIterator *iter = psListIteratorAlloc(item->data.list, PS_LIST_HEAD, false);
    … …  
    391394            continue;
    392395        }
     396        if (fixedzeropt != 0.0) {
     397            // override the recipe's zero point with the fixed value (used for stacks)
     398            zeropt = fixedzeropt;
     399        }
    393400
    394401        // convert the minInst to a calibrated minimum magnitude

  • branches/eam_branches/ipp-20121219/psconfig

  • branches/eam_branches/ipp-20121219/psconfig/psbuild

    r34087 r35096  
    426426
    427427    print STDOUT "** if you use sh, bash or equivalent as your shell, add the following to your .bashrc\n";
    428     print STDOUT "    if [ -e $psconfdir/psconfig.bash ] ; then";
    429     print STDOUT "        psconfig () {";
    430     print STDOUT "            source $psconfdir/psconfig.bash";
    431     print STDOUT "        }";
    432     print STDOUT "    else";
    433     print STDOUT "        psconfig () {";
    434     print STDOUT "            echo psconfig not available";
    435     print STDOUT "        }";
    436     print STDOUT "    fi";
     428    print STDOUT "    if [ -e $psconfdir/psconfig.bash ] ; then\n";
     429    print STDOUT "        psconfig () {\n";
     430    print STDOUT "            source $psconfdir/psconfig.bash\n";
     431    print STDOUT "        }\n";
     432    print STDOUT "    else\n";
     433    print STDOUT "        psconfig () {\n";
     434    print STDOUT "            echo psconfig not available\n";
     435    print STDOUT "        }\n";
     436    print STDOUT "    fi\n";
    437437    print STDOUT "    psconfig default\n";
    438438    print STDOUT "\n";

  • branches/eam_branches/ipp-20121219/psphot

  • branches/eam_branches/ipp-20121219/psphot/src

  • branches/eam_branches/ipp-20121219/psphot/src/psphotSourceSize.c

    r34769 r35096  
    1313    bool altDiffExt;
    1414    float altDiffExtThresh;
     15    bool extFitAll;
     16    bool extFitAllReadout;
     17    float extFitAllThresh;
    1518    float soft;
    1619    int grow;
    … …  
    127130    // Threshold for this alternate method
    128131    options.altDiffExtThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE.THRESH");
     132    assert (status);
     133    // Option to enable fitting of all objects with extended model.
     134    options.extFitAll = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.FIT.ALL.SOURCES");
     135    assert (status);
     136    // Fitting everything is fine, but if the source density is high, we probably shouldn't.
     137    options.extFitAllThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.FIT.ALL.THRESH");
    129138    assert (status);
    130139   
    … …  
    413422
    414423    psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4s %4s %4s %4s %4s", "Npsf", "Next", "Nsat", "Ncr", "Nskip");
    415 
     424    // Determine if this readout is above the threshold to ext fit all sources
     425    if (options->extFitAll) {
     426      float maskFrac = psMetadataLookupF32(&status,readout->analysis,"READOUT.MASK.FRAC");
     427      if (status) {
     428        maskFrac = 0.0;
     429      }
     430      if (sources->n * (1.0 - maskFrac) > options->extFitAllThresh) {
     431        options->extFitAllReadout = false;
     432      }
     433      else {
     434        options->extFitAllReadout = true;
     435      }
     436    }
     437   
    416438    if (!psphotSourceClassRegion (NULL, &psfClump, sources, recipe, psf, options)) {
    417439        psLogMsg ("psphot", 4, "Failed to determine source classification for full image\n");
    … …  
    420442    }
    421443    return true;
    422 
     444   
    423445    int nRegions = psMetadataLookupS32 (&status, readout->analysis, "PSF.CLUMP.NREGIONS");
    424446    for (int i = 0; i < nRegions; i ++) {
    … …  
    426448        psMetadata *regionMD = psMetadataLookupPtr (&status, readout->analysis, regionName);
    427449        psAssert (regionMD, "regions must be defined by earlier call to psphotRoughClassRegion");
    428 
     450       
    429451        psRegion *region = psMetadataLookupPtr (&status, regionMD, "REGION");
    430452        psAssert (region, "regions must be defined by earlier call to psphotRoughClassRegion");
    431 
     453       
    432454        // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y
    433455        psfClump.X  = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X");   psAssert (status, "missing PSF.CLUMP.X");
    … …  
    559581        // * SAT stars should not be faint, but defects may?
    560582
     583        // If the recipe requests we do extended source fits to everything, set
     584        // the EXT_LIMIT flag
     585        if (options->extFitAllReadout) {
     586          source->mode |= PM_SOURCE_MODE_EXT_LIMIT;
     587        }
    561588        // Defects may not always match CRs from peak curvature analysis
    562589        // Defects may also be marked as SATSTAR -- XXX deactivate this flag?

  • branches/eam_branches/ipp-20121219/psphot/src/psphotStackImageLoop.c

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/pstamp/scripts

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/psvideophot

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20121219/tools/queuessky.lap

    r34902 r35096  
    1818
    1919
    20 my ($ramin, $ramax, $decmin, $decmax, $absglatmin, $go);
     20my ($ramin, $ramax, $decmin, $decmax, $absglatmin, $glatmin, $glatmax, $go);
    2121my $combos = 5;
    2222my $verbose = 0;
    … …  
    2727    'dec_min=s'          =>  \$decmin,
    2828    'dec_max=s'          =>  \$decmax,
    29 #    'absglatmin=s'      =>  \$absglatmin,  # not yet
    30     'nfilters=s'        =>  \$combos,
    31     'go'                =>  \$go,
    32     'verbose|v'         =>  \$verbose,
     29    'glat_min=s'         =>  \$glatmin,
     30    'glat_max=s'         =>  \$glatmax,
     31    'abs_glat_min=s'     =>  \$absglatmin,
     32    'nfilters=s'         =>  \$combos,
     33    'go'                 =>  \$go,
     34    'verbose|v'          =>  \$verbose,
    3335) or pod2usage(2);
    3436
    … …  
    5759$command .= " -dec_max $decmax" if defined $decmax;
    5860
    59 # XXX TODO: implement absglatmin
     61$command .= " -glat_min $glatmin" if defined $glatmin;
     62$command .= " -glat_max $glatmax" if defined $glatmax;
     63
     64$command .= " -abs_glat_min $absglatmin" if defined $absglatmin;
    6065
    6166foreach my $filter_combo (@$filter_combos) {
    … …  
    7277    }
    7378}
    74 
    75 
    7679
    7780exit 0;
Note: See TracChangeset for help on using the changeset viewer.