IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jan 11, 2015, 2:05:35 PM (12 years ago)
Author:
eugene
Message:

merge changes from trunk

Location:
branches/eam_branches/ipp-20140904
Files:
91 edited
22 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20140904

  • branches/eam_branches/ipp-20140904/DataStoreServer/scripts/dsreg

  • branches/eam_branches/ipp-20140904/Nebulous/bin/neb-rm

    r26333 r37804  
    4444foreach my $key (@ARGV) {
    4545    unless (defined $move) {
    46         $neb->delete($key, $force,$invalid) or warn "failed to delete key: $key";
     46        eval {
     47            $neb->delete($key, $force,$invalid) or warn "failed to delete key: $key";
     48        }
    4749    } else {
    4850        $neb->move($key, "${key}.rm") or warn "failed to move key: $key";

  • branches/eam_branches/ipp-20140904/dvodist

    • Property svn:mergeinfo changed (with no actual effect on merging)

  • branches/eam_branches/ipp-20140904/extsrc/gpcsw/gpcsrc/fits/burntool

  • branches/eam_branches/ipp-20140904/extsrc/gpcsw/gpcsrc/fits/burntool/persistio.c

    r34220 r37804  
    115115#define DIFFERENT_STREAK 20     /* Proximity for union versus intersection */
    116116
     117# define myMAXSIZE 20000
     118
     119
    117120/****************************************************************/
    118121/* persist_merge(): Disentangle overlapping persistence streaks */
    … …  
    123126   int lapmax;
    124127   OBJBOX *box;
    125    int xusage[MAXSIZE], yctr[MAXSIZE], boxid[MAXSIZE];
     128   int xusage[myMAXSIZE], yctr[myMAXSIZE], boxid[myMAXSIZE];
    126129   double zk, zj;
    127130
    … …  
    131134
    132135/* Assess the clustering of all the streaks */
    133    for(i=0; i<MAXSIZE; i++) xusage[i] = 0;
    134    for(i=0; i<MAXSIZE; i++) boxid[i] = -1;
     136   for(i=0; i<myMAXSIZE; i++) xusage[i] = 0;
     137   for(i=0; i<myMAXSIZE; i++) boxid[i] = -1;
    135138   for(k=0; k<nbox; k++) {
    136       if(box[k].exfit >= MAXSIZE-1) continue;
     139      if(box[k].exfit >= myMAXSIZE-1) continue;
    137140      for(i=box[k].sxfit; i<=box[k].exfit; i++) xusage[i] += 1;
    138141   }
    139142
    140143/* Identify clusters */
    141    for(xs=0; xs<MAXSIZE-1; xs++) {
     144   for(xs=0; xs<myMAXSIZE-1; xs++) {
    142145      if(xusage[xs] == 0) continue;
    143       for(xe=xs, lapmax=0; xe<MAXSIZE-1; xe++) {
     146      for(xe=xs, lapmax=0; xe<myMAXSIZE-1; xe++) {
    144147         if(xusage[xe+1] == 0) break;
    145148         lapmax = MAX(lapmax, xusage[xe]);

  • branches/eam_branches/ipp-20140904/ippMonitor/czartool/czartool/Plotter.pm

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/addstar_run.pl

    r34750 r37804  
    9494# Recipes to use based on reduction class
    9595$reduction = 'DEFAULT' unless defined $reduction;
    96 if ($stage =~/skycal/) {
    97     $reduction = 'DEFAULT';
    98 }
     96#if ($stage =~/diff/) {
     97#    $reduction = 'ADDSTAR';
     98#}
     99#if ($stage =~/fullforce/) {
     100#    $reduction = 'ADDSTAR';
     101#}
     102
     103if ($stage =~/fullforce/) {
     104    $reduction='DEFAULT';
     105    #hardwired because why not
     106}
     107
    99108my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR');
    100109# XXX This is now not used: do we still need it?
    … …  
    132141    print "using $fpaObjects for $stage\n";
    133142}
     143if ($stage =~/diff/) {
     144    print "using $fpaObjects for $stage\n";
     145}
     146if ($stage =~/fullforce/)  {
     147    print "using $fpaObjects for $stage\n";
     148}
     149
    134150
    135151
    … …  
    177193}
    178194
    179 if (($stage =~/stack/ || $stage =~ /skycal/)) {
     195if (($stage =~/stack/ || $stage =~ /skycal/ || $stage =~/diff/ || $stage=~/fullforce/)) {
    180196    $fpaObjects =~ s/smf$/cmf/;
    181197    $fpaObjectsAlt =~ s/smf$/cmf/;
    … …  
    305321                $command .= " -quick-airmass";
    306322            }  #careful here - this matches staticsky and staticsky_multi
     323            if ($stage =~ /diff/) {
     324                $command .= " -accept-astrom";
     325            }
     326            if ($stage =~ /fullforce/) {
     327                $command .= " -accept-astrom -xrad";
     328            }
    307329
    308330            my $mjd_addstar_start = DateTime->now->mjd;   # MJD of starting script

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/ipp_cleanup.pl

    r35906 r37804  
    588588                addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
    589589                addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id );
     590                addFilename(\@files, "PSPHOT.RESID",$path_base,"fpa");
    590591                if ($mode eq "goto_purged") {
    591592                    # additional files to remove for 'purge' mode

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/lap_science.pl

    r37319 r37804  
    622622            $complete_fstacks++;
    623623        }
     624        elsif (($stack->{final_state})&&
     625               ($stack->{final_state} eq 'wait')) {
     626            $complete_fstacks++; # Not the best either, but we've noted an issue, and tried to move on.  LAP should respect this.
     627        }
    624628        elsif (($stack->{final_state})&&($stack->{final_state} eq 'new')&&
    625629               ($stack->{final_fault} >= 4)&&($stack->{final_fault} != 32767)) {
    … …  
    757761        }
    758762        elsif ((defined($exposure->{warpRun_state}))&&($exposure->{warpRun_state} eq 'full')&&
    759                ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.2)) {
     763               ($exposure->{warp_bad_quality} / $exposure->{warp_component_count} > 0.3)) {
    760764            printf("QUALITY: $exposure->{exp_id} has bad warp quality: %d / %d\n",
    761765                   $exposure->{warp_bad_quality} , $exposure->{warp_component_count});

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/permcheck.pl

    r37139 r37804  
    2121#my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
    2222my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
     23my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
    2324my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
    2425my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
    … …  
    4142pod2usage( -msg => "Required options: --dbname", -exitval => 2 )
    4243    unless $dbname;
    43 pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 )
     44pod2usage( -msg => "Required options: --stage [camera|warp|stack|skycal]", -exitval => 2 )
    4445    unless $stage;
    4546pod2usage( -msg => "missing key", exitval => 2 )
    … …  
    5859                           'ippb06' => 1
    5960);
     61
     62if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes.
     63    %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1,
     64                     'stsci03' => 1, 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1,
     65                     'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1,
     66                     'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1,
     67                     'stsci13' => 1, 'stsci14' => 1, 'stsci15' => 1, 'stsci16' => 1,
     68                     'stsci17' => 1, 'stsci18' => 1, 'stsci19' => 1);
     69    %backup_destinations = %backup_hosts;
     70}
     71
    6072my $backup_Nvols = 3;
    6173
    … …  
    113125        &my_die("Unable to parse metadata from stagetool", $stage_id);
    114126}
     127elsif ($stage eq 'warp') {
     128    my $cmd = "$warptool -warped -warp_id $stage_id -dbname $dbname";
     129    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     130        run(command => $cmd, verbose => 0);
     131    unless ($success) {
     132        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
     133        &my_die("Unable to perform stagetool: $error_code", $stage_id);
     134    }
     135    $files = $mdcParser->parse_list(join "", @$stdout_buf) or
     136        &my_die("Unable to parse metadata from stagetool", $stage_id);
     137}
    115138
    116139my %components = ('camera' => ['PSASTRO.OUTPUT','PSASTRO.OUTPUT.MASK'],
     140                  'warp'   => ['PSWARP.OUTPUT','PSWARP.OUTPUT.MASK','PSWARP.OUTPUT.VARIANCE'],
    117141                  'stack'  => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP',
    118142                               'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'],
    … …  
    124148    my $data_state = $entry->{state};
    125149    my $hostname = $entry->{hostname};
     150    my $quality  = $entry->{quality};
     151    if ($quality != 0) { next; }
     152    print "# $path_base $data_state $hostname $quality\n";
    126153    foreach my $product (@{ $components{$stage} }) {
    127154        my @keys = ();
    … …  
    148175            my $stat = $neb->stat($key);
    149176            die "nebulous key: $key not found" unless $stat;   
    150             my $instances = $neb->find_instances($key, 'any');
     177            my $instances;
     178            eval {
     179                $instances = $neb->find_instances($key, 'any');
     180            };
     181            unless (defined($instances)) { print "## skipping due to zero instances\n"; next; }
    151182            die "no instances found" unless $instances;   
    152183           
    … …  
    256287                    $quality[0] = 1;
    257288                    print "cp $good_file $diskfiles[0]\n";
    258                     system("cp $good_file $diskfiles[0]");
     289                    vsystem("cp $good_file $diskfiles[0]");
    259290                }
    260291                # Begin my best validation thought
    … …  
    275306                    if ($md5sums[$i] eq $md5sum) { # Found it.
    276307                        print "cp $diskfiles[$i] $diskfiles[0]\n";
    277                         system("cp $diskfiles[$i] $diskfiles[0]");
     308                        vsystem("cp $diskfiles[$i] $diskfiles[0]");
    278309                        # Begin my best validation thought
    279310                        {
    … …  
    291322            if ($Ngood == 1) { # We have only one version
    292323                if ($quality_mask & 1) { # ANd it's on a backup volume
    293                     print "neb-replicate $key\n";
    294                     if ($do_ops) {
    295                         $neb->replicate($key) or die "failed to replicate the single valid copy";
    296                         if ($@) { die $@; }
     324                    if ($user_copies > 1) { # And we want more than one copy.
     325                        print "neb-replicate $key\n";
     326                        if ($do_ops) {
     327                            $neb->replicate($key) or die "failed to replicate the single valid copy";
     328                            if ($@) { die $@; }
     329                        }
    297330                    }
    298331                }
    … …  
    305338                       
    306339                        # Begin my best validation thought
    307                         system("sync") == 0 or die "Couldn't sync?";
     340                        vsystem("sync") == 0 or die "Couldn't sync?";
    308341                        my $uris = $neb->find_instances($key,$rep_vol);
    309342                        @$uris = map {URI->new($_)->file if $_} @$uris;
    … …  
    325358                       
    326359                        # Begin my best validation thought
    327                         system("sync") == 0 or die "Couldn't sync?";
     360                        vsystem("sync") == 0 or die "Couldn't sync?";
    328361                        my $uris = $neb->find_instances($key,$rep_vol);
    329362                        @$uris = map {URI->new($_)->file if $_} @$uris;
    … …  
    343376                    }
    344377                }
    345                 for (my $i = 0; $i <= $#diskfiles; $i++) {
     378            }
     379#XXX NEW DEBUG THIS:  This should iterate over diskfiles that are not marked with a quality = 2, and cull them. 
     380# For the N>=2, we set that for the primary, and
     381# for both cases, we do not have a diskfile entry for the newly replicated copy. 
     382# Therefore, this should cull down to the correct number.
     383            for (my $i = 0; $i <= $#diskfiles; $i++) {
    346384#           print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n";
    347                     if ($quality[$i] != 1) {
    348                         if ($existance[$i] == 0) { # This disk file doesn't exist.
    349                             system("touch $diskfiles[$i]");
    350                         }
    351                         my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
    352                         print "neb-cull --volume $cull_vol $key\n";
    353                         if ($do_ops) {
    354                             # The tilde here is to force hard volumes.  Don't touch it.
    355                             $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
    356                             if ($@) { die "$@"; }
    357                         }
    358                     }
    359                 }
    360             }
     385                if ($quality[$i] != 1) {
     386                    if ($existance[$i] == 0) { # This disk file doesn't exist.
     387                        vsystem("touch $diskfiles[$i]");
     388                    }
     389                    my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i];
     390                    print "neb-cull --volume $cull_vol $key\n";
     391                    if ($do_ops) {
     392                        # The tilde here is to force hard volumes.  Don't touch it.
     393                        $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance";
     394                        if ($@) { die "$@"; }
     395                    }
     396                }
     397            }
     398            # }
    361399        } # end keys for this product
    362400    } # end product for this entry
    363401} # end entry for this id
     402
     403sub vsystem {
     404    my $cmd = shift;
     405    print "$cmd\n";
     406    if ($do_ops) {
     407        system($cmd);
     408    }
     409}
    364410
    365411sub local_md5sum {
    … …  
    379425
    380426
     427
    381428sub parse_volume {
    382429    my $filename = shift(@_);

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/publish_file.pl

    r36839 r37804  
    252252                     warp_id => $comp->{warp1},
    253253                     diff_id => $comp->{diff_id},
     254                     camera => $comp->{camera},
    254255                     output_format => $comp->{output_format},
    255256                     direction => 1,
    … …  
    291292                         warp_id => $comp->{warp2},
    292293                         diff_id => $comp->{diff_id},
     294                         camera => $comp->{camera},
     295                         # missing output_format?
    293296                         direction => 0,
    294297                         comment => $reg_comp->{comment},
    … …  
    359362    my $name = shift;           # Name of component
    360363
    361     $data->{zp} = $comp->{zp} unless defined $data->{zp};
    362     $data->{zp_err} = $comp->{zp_err} unless defined $data->{zp_err};
    363     $data->{astrom} = $comp->{astrom} unless defined $data->{astrom};
    364     $data->{exp_name} = $comp->{exp_name} unless defined $data->{exp_name};
    365     $data->{exp_id} = $comp->{exp_id} unless defined $data->{exp_id};
    366     $data->{chip_id} = $comp->{chip_id} unless defined $data->{chip_id};
    367     $data->{cam_id} = $comp->{cam_id} unless defined $data->{cam_id};
    368     $data->{fake_id} = $comp->{fake_id} unless defined $data->{fake_id};
    369     $data->{warp_id} = $comp->{warp_id} unless defined $data->{warp_id};
    370     $data->{diff_id} = $comp->{diff_id} unless defined $data->{diff_id};
     364    $data->{zp}        = $comp->{zp}        unless defined $data->{zp};
     365    $data->{zp_err}    = $comp->{zp_err}    unless defined $data->{zp_err};
     366    $data->{astrom}    = $comp->{astrom}    unless defined $data->{astrom};
     367    $data->{exp_name}  = $comp->{exp_name}  unless defined $data->{exp_name};
     368    $data->{exp_id}    = $comp->{exp_id}    unless defined $data->{exp_id};
     369    $data->{chip_id}   = $comp->{chip_id}   unless defined $data->{chip_id};
     370    $data->{cam_id}    = $comp->{cam_id}    unless defined $data->{cam_id};
     371    $data->{fake_id}   = $comp->{fake_id}   unless defined $data->{fake_id};
     372    $data->{warp_id}   = $comp->{warp_id}   unless defined $data->{warp_id};
     373    $data->{diff_id}   = $comp->{diff_id}   unless defined $data->{diff_id};
     374    $data->{camera}    = $comp->{camera}    unless defined $data->{camera};
     375    # missing output_format?
    371376    $data->{direction} = $comp->{direction} unless defined $data->{direction};
    372     $data->{comment} = $comp->{comment} unless defined $data->{comment};
    373     $data->{obsmode} = $comp->{obsmode} unless defined $data->{obsmode};
    374     $data->{difftype} = $comp->{difftype} unless defined $data->{difftype};
    375     $data->{sky} = $comp->{sky} unless defined $data->{sky};
    376     $data->{shutoutc} = $comp->{shutoutc} unless defined $data->{shutoutc};
     377    $data->{comment}   = $comp->{comment}  unless defined $data->{comment};
     378    $data->{obsmode}   = $comp->{obsmode}  unless defined $data->{obsmode};
     379    $data->{difftype}  = $comp->{difftype} unless defined $data->{difftype};
     380    $data->{sky}       = $comp->{sky}      unless defined $data->{sky};
     381    $data->{shutoutc}  = $comp->{shutoutc} unless defined $data->{shutoutc};
    377382
    378383    &my_die("zp value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{zp} and $comp->{zp} != $data->{zp};
    … …  
    386391    &my_die("warp_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{warp_id} and $comp->{warp_id} != $data->{warp_id};
    387392    &my_die("diff_id value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{diff_id} and $comp->{diff_id} != $data->{diff_id};
     393    &my_die("camera value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{camera} and $comp->{camera} ne $data->{camera};
    388394    &my_die("direction value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{direction} and $comp->{direction} != $data->{direction};
    389395    &my_die("comment value for $name doesn't match", $pub_id, $PS_EXIT_SYS_ERROR) if defined $data->{comment} and $comp->{comment} ne $data->{comment};
    … …  
    414420    $command .= " -warp_id " . $data->{warp_id} if defined $data->{warp_id};
    415421    $command .= " -diff_id " . $data->{diff_id} if defined $data->{diff_id};
     422    $command .= " -camera " . $data->{camera} if defined $data->{camera};
    416423    $command .= " -inverse" if defined $data->{direction} and $data->{direction} == 0;
    417424    $command .= " -zp " . $data->{zp} if defined $data->{zp};
    … …  
    420427    $command .= " -version " . $data->{output_format} if defined $data->{output_format};
    421428    $command .= " -comment \"" . $data->{comment} . "\"" if defined $data->{comment};
    422     $command .= " -obsmode " . $data->{obsmode} if defined $data->{obsmode};
     429    $command .= " -obsmode \"" . $data->{obsmode} . "\"" if defined $data->{obsmode};
    423430    $command .= " -difftype " . $data->{difftype} if defined $data->{difftype};
    424431    $command .= " -sky " . $data->{sky} if defined $data->{sky};

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/queuestaticsky.pl

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/rawcheck.pl

    r36685 r37804  
    6262my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1,
    6363                    'ippb02' => 1, 'ippb03' => 1,
    64                     'ippb04' => 1, 'ippb05' => 1
     64                    'ippb04' => 1, 'ippb05' => 1,
     65                    'ippb06' => 1
    6566    );
    66 my %backup_destinations = ('ippb04' => 1,
    67                            'ippb05' => 1);
     67my %backup_destinations = (# 'ippb04' => 1,  # full
     68                           # 'ippb05' => 1,  # full
     69                           'ippb06' => 1
     70    );
    6871my $backup_Nvols = 3;
    6972

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/sc_remote_exec.pl

  • branches/eam_branches/ipp-20140904/ippScripts/scripts/stack_bkg_mk_mdc.pl

    r36810 r37804  
    1313
    1414my ($dbname,$stack_id,$label,$camera,$filter);
     15
     16my $ipprc =  PS::IPP::Config->new(); # IPP Configuration
     17my $siteConfig = $ipprc->{_siteConfig};
     18
    1519#$dbname = 'gpc1';
    1620#$camera = 'GPC1';
    … …  
    2933
    3034use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
    31 my $dbserver = 'ippdb01';
    32 my $dbuser = 'ippuser';
    33 my $dbpass = 'ippuser';
     35my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
     36my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "RO_DBUSER");
     37my $dbpass = metadataLookupStr($ipprc->{_siteConfig}, "RO_DBPASSWORD");
     38die "database configuration not set up" unless defined($dbserver);
     39die "database configuration not set up" unless defined($dbuser);
     40die "database configuration not set up" unless defined($dbpass);
    3441my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
    3542                   "mysql_socket=" . DB_SOCKET(),
    … …  
    3845    ) or die "Unable to connect to database $DBI::errstr\n";
    3946
    40 my $ipprc = PS::IPP::Config->new( $camera );
     47$ipprc = PS::IPP::Config->new( $camera );
    4148my $mdcParser = PS::IPP::Metadata::Config->new;
    4249

  • branches/eam_branches/ipp-20140904/ippTasks/addstar.pro

    r35305 r37804  
    5252    active false
    5353  end
     54  task addstar.revert.diff
     55    active false
     56  end
     57  task addstar.revert.fullforce
     58    active false
     59  end
    5460end
    5561
    … …  
    6571  end
    6672  task addstar.revert.skycal
     73    active true
     74  end
     75  task addstar.revert.diff
     76    active true
     77  end
     78  task addstar.revert.fullforce
    6779    active true
    6880  end
    … …  
    141153$addstar_revert_DB_SSM = 0
    142154$addstar_revert_DB_SC = 0
     155$addstar_revert_DB_FF = 0
     156$addstar_revert_DB_DF = 0
    143157
    144158#$LOADEXEC_ADD = 300
    … …  
    275289        sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
    276290    end
     291    if ("$STAGE" == "diff")
     292        sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
     293    end
     294    if ("$STAGE" == "fullforce")
     295        sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID
     296    end
    277297
    278298 
    … …  
    293313   if ("$STAGE" == "skycal")
    294314      $run = $run --stage_id $STAGE_ID
     315    end
     316   if ("$STAGE" == "diff")
     317      $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1
     318    end
     319   if ("$STAGE" == "fullforce")
     320      $run = $run --stage_id $STAGE_ID --stage_extra1 $STAGE_EXTRA1
    295321    end
    296322
    … …  
    526552  end
    527553end
     554
     555
     556task addstar.revert.diff
     557  host         local
     558
     559  periods      -poll 5.0
     560  periods      -exec 60.0
     561  periods      -timeout 1200
     562  npending     1
     563  active        false
     564
     565  stdout NULL
     566  stderr $LOGDIR/revert.log
     567
     568  task.exec
     569    if ($LABEL:n == 0) break
     570    $run = addtool -revertprocessedexp -stage diff
     571    if ($DB:n == 0)
     572      option DEFAULT
     573    else
     574      # save the DB name for the exit tasks                                                                   
     575      option $DB:$addstar_revert_DB_DF
     576      $run = $run -dbname $DB:$addstar_revert_DB_DF
     577      $addstar_revert_DB_DF ++
     578      if ($addstar_revert_DB_DF >= $DB:n) set addstar_revert_DB_DF = 0
     579    end
     580     add_poll_labels run
     581    command $run
     582  end
     583
     584  # success                                                                                                   
     585  task.exit    0
     586  end
     587
     588  # locked list                                                                                               
     589  task.exit    default
     590    showcommand failure
     591  end
     592
     593  task.exit    crash
     594    showcommand crash
     595  end
     596
     597  # operation times out?                                                                                     
     598  task.exit    timeout
     599    showcommand timeout
     600  end
     601end
     602
     603
     604task addstar.revert.fullforce
     605  host         local
     606
     607  periods      -poll 5.0
     608  periods      -exec 60.0
     609  periods      -timeout 1200
     610  npending     1
     611  active        false
     612
     613  stdout NULL
     614  stderr $LOGDIR/revert.log
     615
     616  task.exec
     617    if ($LABEL:n == 0) break
     618    $run = addtool -revertprocessedexp -stage fullforce
     619    if ($DB:n == 0)
     620      option DEFAULT
     621    else
     622      # save the DB name for the exit tasks                                                                   
     623      option $DB:$addstar_revert_DB_FF
     624      $run = $run -dbname $DB:$addstar_revert_DB_FF
     625      $addstar_revert_DB_FF ++
     626      if ($addstar_revert_DB_FF >= $DB:n) set addstar_revert_DB_FF = 0
     627    end
     628     add_poll_labels run
     629    command $run
     630  end
     631
     632  # success                                                                                                   
     633  task.exit    0
     634  end
     635
     636  # locked list                                                                                               
     637  task.exit    default
     638    showcommand failure
     639  end
     640
     641  task.exit    crash
     642    showcommand crash
     643  end
     644
     645  # operation times out?                                                                                     
     646  task.exit    timeout
     647    showcommand timeout
     648  end
     649end

  • branches/eam_branches/ipp-20140904/ippTasks/register.pro

    r35830 r37804  
    3636    active false
    3737  end
     38  task register.exp.revert
     39    active false
     40  end
    3841  task register.exp.load
    3942    active true
    … …  
    6063    active false
    6164  end
     65  task register.exp.revert
     66    active false
     67  end
    6268  task register.exp.load
    6369    active false
    … …  
    7884    active true
    7985  end
     86  task register.exp.revert
     87    active true
     88  end
    8089end
    8190
    8291macro register.revert.off
    8392  task register.imfile.revert
     93    active false
     94  end
     95  task register.exp.revert
    8496    active false
    8597  end
    … …  
    202214$regPendingBurntoolImfile_DB = 0
    203215$regRevertImfile_DB = 0
     216$regRevertExp_DB = 0
    204217$regPendingExp_DB = 0
    205218$reg_datePAGE = 0
    … …  
    371384      $regRevertImfile_DB ++
    372385      if ($regRevertImfile_DB >= $DB:n) set regRevertImfile_DB = 0
     386    end
     387    add_poll_args run
     388    command $run
     389  end
     390
     391  # success
     392  task.exit    0
     393  end
     394
     395  # locked list
     396  task.exit    default
     397    showcommand failure
     398  end
     399
     400  task.exit    crash
     401    showcommand crash
     402  end
     403
     404  # operation times out
     405  task.exit    timeout
     406    showcommand timeout
     407  end
     408end
     409
     410
     411
     412task register.exp.revert
     413  host         local
     414  periods      -poll 60.0
     415  periods      -exec 1800.0
     416  periods      -timeout 120.0
     417  npending     1
     418
     419  stdout       NULL
     420  stderr       $LOGDIR/register.exp.revert.log
     421
     422  task.exec
     423# Only revert fault 2, because other faults likely mean something horrible is happening.
     424    $run = regtool -revertprocessedexp -fault 2
     425    if ($DB:n == 0)
     426      option DEFAULT
     427    else
     428      option $DB:$regRevertExp_DB
     429      $run = $run -dbname $DB:$regRevertExp_DB
     430      $regRevertExp_DB ++
     431      if ($regRevertExp_DB >= $DB:n) set regRevertExp_DB = 0
    373432    end
    374433    add_poll_args run

  • branches/eam_branches/ipp-20140904/ippTasks/stack.pro

    r37076 r37804  
    5353    active false
    5454  end
    55   task stack.summary.load
    56     active true
    57   end
    58   task stack.summary.run
    59     active true
    60   end
    6155end
    6256
    … …  
    7266    active false
    7367  end
    74   task stack.summary.load
    75     active false
    76   end
    77   task stack.summary.run
    78     active false
    79   end
    80 end
    81 
     68end
     69
     70# Same for summary stages.
    8271macro stack.summary.on
    8372  task stack.summary.load
    … …  
    313302  periods      -exec $LOADEXEC
    314303  periods      -timeout 1200
    315   trange       07:00:00 08:00:00 -nmax 1
     304  active       false
     305#  trange       07:00:00 08:00:00 -nmax 1
    316306  npending     1
    317307
    … …  
    368358  periods      -exec $RUNEXEC
    369359  periods      -timeout 60
     360  active       false
    370361
    371362  task.exec

  • branches/eam_branches/ipp-20140904/ippTasks/survey.pro

    r37076 r37804  
    178178# user functions to manipulate warp/stack diff labels
    179179macro survey.add.WSdiff
    180   if ($0 != 6)
    181     echo "USAGE: survey.add.WSdiff (warp label) (stack label) (dist_group) (workdir base) (target label)"
     180  if ($0 != 7)
     181    echo "USAGE: survey.add.WSdiff (key) (warp label) (stack label) (dist_group) (workdir base) (target label)"
    182182    break
    183183  end
    184184  book newpage SURVEY_DIFF_WARPSTACK $1
    185   book setword SURVEY_DIFF_WARPSTACK $1 STACK_LABEL $2
    186   book setword SURVEY_DIFF_WARPSTACK $1 DIST_GROUP $3
    187   book setword SURVEY_DIFF_WARPSTACK $1 WORKDIR $4
    188   book setword SURVEY_DIFF_WARPSTACK $1 TARGET_LABEL $5
     185  book setword SURVEY_DIFF_WARPSTACK $1 WARP_LABEL $2
     186  book setword SURVEY_DIFF_WARPSTACK $1 STACK_LABEL $3
     187  book setword SURVEY_DIFF_WARPSTACK $1 DIST_GROUP $4
     188  book setword SURVEY_DIFF_WARPSTACK $1 WORKDIR $5
     189  book setword SURVEY_DIFF_WARPSTACK $1 TARGET_LABEL $6
    189190  book setword SURVEY_DIFF_WARPSTACK $1 STATE PENDING
    190191end
    … …  
    697698  periods      -exec $SURVEY_EXEC
    698699  periods      -timeout $SURVEY_TIMEOUT
     700  ## time in UT -- want to start ~0900 HST and run until next night starts. 24h boundary so two trange needed?
     701  trange        19:00:00 23:59:00
     702  trange        00:00:00 05:00:00
    699703  npending     1
    700704
    … …  
    710714    endif
    711715
    712     book getpage SURVEY_DIFF_WARPSTACK 0 -var label -key STATE NEW
    713     if ("$label" == "NULL")
     716    book getpage SURVEY_DIFF_WARPSTACK 0 -var wsKey -key STATE NEW
     717    if ("$wsKey" == "NULL")
    714718      # All labels have been done --- reset
    715719#      echo "Resetting labels"
    716720      for i 0 $N
    717         book getpage SURVEY_DIFF_WARPSTACK $i -var label
    718         book setword SURVEY_DIFF_WARPSTACK $label STATE NEW
     721        book getpage SURVEY_DIFF_WARPSTACK $i -var wsKey
     722        book setword SURVEY_DIFF_WARPSTACK $wsKey STATE NEW
    719723      end
    720       book getpage SURVEY_DIFF_WARPSTACK 0 -var label -key STATE NEW
     724      book getpage SURVEY_DIFF_WARPSTACK 0 -var wsKey -key STATE NEW
    721725
    722726      # Select different database
    … …  
    725729    end
    726730
    727     book setword SURVEY_DIFF_WARPSTACK $label STATE DONE
    728     book getword SURVEY_DIFF_WARPSTACK $label STACK_LABEL -var stack_label
    729     book getword SURVEY_DIFF_WARPSTACK $label WORKDIR -var workdir
    730     book getword SURVEY_DIFF_WARPSTACK $label DIST_GROUP -var dist_group
    731     book getword SURVEY_DIFF_WARPSTACK $label TARGET_LABEL -var target_label
     731    book setword SURVEY_DIFF_WARPSTACK $wsKey STATE DONE
     732    book getword SURVEY_DIFF_WARPSTACK $wsKey WARP_LABEL -var warp_label
     733    book getword SURVEY_DIFF_WARPSTACK $wsKey STACK_LABEL -var stack_label
     734    book getword SURVEY_DIFF_WARPSTACK $wsKey WORKDIR -var workdir
     735    book getword SURVEY_DIFF_WARPSTACK $wsKey DIST_GROUP -var dist_group
     736    book getword SURVEY_DIFF_WARPSTACK $wsKey TARGET_LABEL -var target_label
    732737
    733738    $year = `date +%Y`
    … …  
    737742# Do we need anything else here?
    738743    $run = difftool -definewarpstack -good_frac 0.2
    739     $run = $run -warp_label $label
     744    $run = $run -warp_label $warp_label
    740745    $run = $run -stack_label $stack_label
    741746    $run = $run -set_dist_group $dist_group
    … …  
    10121017        $run = $run -uncensored
    10131018    end
    1014 
     1019    if ("$stage" == "fullforce")
     1020        $run = $run -uncensored
     1021    end
     1022    if ("$stage" == "diff")
     1023        $run = $run -uncensored)
     1024    end
    10151025    # we need to handle isp/gpc1: gpc1 wants destreaked, isp wants uncensored.  Perhaps the best way is by default do destreaked (the most paranoid), and if db = isp (is this the best way?) then do uncensored
    10161026   

  • branches/eam_branches/ipp-20140904/ippTools/share/Makefile.am

    r37251 r37804  
    88        addtool_find_cam_id_dvo.sql \
    99        addtool_find_cam_id.sql \
     10        addtool_find_diff_id.sql \
     11        addtool_find_diff_id_dvo.sql \
     12        addtool_find_ff_id.sql \
     13        addtool_find_ff_id_dvo.sql \
    1014        addtool_find_skycal_id_dvo.sql \
    1115        addtool_find_skycal_id.sql \
    … …  
    1620        addtool_find_pendingexp.sql \
    1721        addtool_find_pendingexp_cam.sql \
     22        addtool_find_pendingexp_diff.sql \
     23        addtool_find_pendingexp_ff.sql \
    1824        addtool_find_pendingexp_skycal.sql \
    1925        addtool_find_pendingexp_stack.sql \
    … …  
    2127        addtool_find_pendingmergeprocess.sql \
    2228        addtool_find_processedexp_cam.sql \
     29        addtool_find_processedexp_diff.sql \
     30        addtool_find_processedexp_ff.sql \
    2331        addtool_find_processedexp_skycal.sql \
    2432        addtool_find_processedexp_stack.sql \
    … …  
    2937        addtool_pendingcleanuprun.sql \
    3038        addtool_queue_cam_id.sql \
     39        addtool_queue_diff_id.sql \
     40        addtool_queue_ff_id.sql \
    3141        addtool_queue_stack_id.sql \
    3242        addtool_queue_sky_id_multi.sql \
    … …  
    3545        addtool_revertminidvodbprocessed.sql \
    3646        addtool_revertprocessedexp_cam.sql \
     47        addtool_revertprocessedexp_diff.sql \
     48        addtool_revertprocessedexp_ff.sql \
    3749        addtool_revertprocessedexp_skycal.sql \
    3850        addtool_revertprocessedexp_stack.sql \
    … …  
    471483        laptool_revertgroup.sql \
    472484        laptool_filtersforgroup.sql \
     485        laptool_revertsass.sql \
    473486        vptool_find_rawexp.sql \
    474487        vptool_pendingimfile.sql \

  • branches/eam_branches/ipp-20140904/ippTools/share/fftool_result.sql

    r36633 r37804  
    1111    stackRun.skycell_id,
    1212    stackRun.filter,
     13    rawExp.exp_id,
     14    rawExp.exp_name,
    1315    rawExp.camera
    1416FROM fullForceRun

  • branches/eam_branches/ipp-20140904/ippTools/share/laptool_exposures.sql

    r31511 r37804  
    1 select DISTINCT V3.*,
    2        diffRun.diff_id,diffRun.state as diffRun_state,
    3        coalesce(CONVERT(sum(diffSkyfile.quality != 0),SIGNED),0) AS diff_bad_quality,
    4        coalesce(CONVERT(count(diffSkyfile.diff_id),SIGNED),0) AS diff_component_count
    5        FROM
     1-- Join the diff information
     2-- select DISTINCT V3.*,
     3--       diffRun.diff_id,diffRun.state as diffRun_state,
     4--       coalesce(CONVERT(sum(diffSkyfile.quality != 0),SIGNED),0) AS diff_bad_quality,
     5--       coalesce(CONVERT(count(diffSkyfile.diff_id),SIGNED),0) AS diff_component_count
     6--       FROM
     7select V3.*,-1 as diff_id ,"full" as diffRun_state,0 as diff_bad_quality,0 AS diff_component_count FROM
     8-- Join the warp information
    69( select V2.*,
    710       warpRun.warp_id,warpRun.state as warpRun_state,
    … …  
    1013       warpRun.magicked
    1114       FROM
     15-- Join the camera and fake information
    1216( select V1.*,
    1317       camRun.cam_id,camRun.state as camRun_state,
    … …  
    1519       coalesce(CONVERT(count(camProcessedExp.cam_id),SIGNED),0) AS cam_component_count,
    1620       fakeRun.fake_id,fakeRun.state as fakeRun_state FROM
     21-- Get all the lap, exposure, chip stage information
    1722( SELECT DISTINCT
    1823       lap_id,lapRun.tess_id,projection_cell,filter,lapRun.state as lapRun_state, lapRun.registered, lapRun.fault, lapRun.label, lapRun.dist_group,
    … …  
    2631       GROUP BY lap_id,exp_id
    2732       ) AS V1
     33-- End lap/exposure/chip
    2834       LEFT JOIN camRun USING(chip_id) LEFT JOIN camProcessedExp USING(cam_id)
    2935       LEFT JOIN fakeRun USING(cam_id)
    3036       GROUP BY lap_id,exp_id
    3137  ) AS V2
     38-- End camera/fake
    3239  LEFT JOIN warpRun USING(fake_id) LEFT JOIN warpSkyfile USING(warp_id)
    3340  GROUP BY lap_id,exp_id
    3441) AS V3
    35 LEFT JOIN
    36   (SELECT DISTINCT diff_id,warp1,warp2 FROM diffInputSkyfile) AS DI ON
    37   (DI.warp1 = warp_id OR DI.warp2 = warp_id)
    38 LEFT JOIN diffRun USING(diff_id)
    39 LEFT JOIN diffSkyfile USING(diff_id)
     42-- End warp
     43-- LEFT JOIN
     44--  (SELECT DISTINCT diff_id,warp1,warp2 FROM diffInputSkyfile) AS DI ON
     45--  (DI.warp1 = warp_id OR DI.warp2 = warp_id)
     46-- LEFT JOIN diffRun USING(diff_id)
     47-- LEFT JOIN diffSkyfile USING(diff_id)
    4048GROUP BY lap_id,exp_id
    4149

  • branches/eam_branches/ipp-20140904/ippTools/share/pstamptool_pendingdependent.sql

    r36167 r37804  
    22    pstampDependent.*,
    33    pstampRequest.label,
     4    MIN(req_id) as first_req_id,
    45    concat( pstampDependent.outdir, '/checkdep.', dep_id, '.log') as logfile,
    56    IFNULL(Label.priority, 10000) AS priority

  • branches/eam_branches/ipp-20140904/ippTools/src/addtool.c

    r35304 r37804  
    122122    PXOPT_COPY_S64(config->args, where,  "-sky_id",    "staticskyRun.sky_id", "==");
    123123    PXOPT_COPY_S64(config->args, where,  "-skycal_id",  "skycalRun.skycal_id", "==");
     124    PXOPT_COPY_S64(config->args, where,  "-diff_id",  "diffRun.diff_id", "==");
     125    PXOPT_COPY_S64(config->args, where,  "-ff_id",  "fullForceRun.ff_id", "==");
     126
    124127    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
    125128    if (strcmp(stage, "cam")== 0) {
    … …  
    146149      PXOPT_COPY_STR(config->args, where,  "-reduction", "skycalRun.reduction",  "==");
    147150    }
    148 
     151    if (strcmp(stage, "diff")==0) {
     152      pxAddLabelSearchArgs (config, where, "-label",     "diffRun.label", "=="); //define using skycalRun label
     153      pxAddLabelSearchArgs (config, where, "-data_group","diffRun.data_group", "==");
     154      PXOPT_COPY_STR(config->args, where,  "-reduction", "diffRun.reduction",  "==");
     155    }
     156    if (strcmp(stage, "fullforce")==0) {
     157      pxAddLabelSearchArgs (config, where, "-label",     "fullForceRun.label", "=="); //define using skycalRun label
     158      pxAddLabelSearchArgs (config, where, "-data_group","fullForceRun.data_group", "==");
     159      PXOPT_COPY_STR(config->args, where,  "-reduction", "fullForceRun.reduction",  "==");
     160    }
     161
     162 
     163 
    149164    if (!psListLength(where->list)) {
    150165        psFree(where);
    … …  
    258273    }
    259274
    260 
     275    if (strcmp(stage,"diff") == 0) {
     276      if (dvodb ) {
     277        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_diff_id_dvo.sql\n%s\n", dvodb,stage);
     278        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
     279        bare_query = pxDataGet("addtool_find_diff_id_dvo.sql");
     280        // user supplied dvodb                                                                                                                                             
     281        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
     282      } else {
     283        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_diff_id.sql\n%s\n",stage);
     284        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
     285        bare_query = pxDataGet("addtool_find_diff_id.sql");
     286        // inherit dvodb from skycalRun, avoid matching NULL                                                                                                                   
     287        psStringAppend(&dvodb_string, "(diffRun.dvodb IS NOT NULL AND previous_dvodb = diffRun.dvodb)");
     288        // this is silly, there is no dvodb in skycalRun...?
     289      }
     290    }
     291
     292    if (strcmp(stage,"fullforce") == 0) {
     293      if (dvodb ) {
     294        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_ff_id_dvo.sql\n%s\n", dvodb,stage);
     295        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
     296        bare_query = pxDataGet("addtool_find_ff_id_dvo.sql");
     297        // user supplied dvodb                                                                                                                                             
     298        psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb);
     299      } else {
     300        psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_ff_id.sql\n%s\n",stage);
     301        // find the skycal_id of all the exposures that we want to queue up.                                                                                                   
     302        bare_query = pxDataGet("addtool_find_ff_id.sql");
     303        // inherit dvodb from skycalRun, avoid matching NULL                                                                                                                   
     304        psStringAppend(&dvodb_string, "(fullForceRun.dvodb IS NOT NULL AND previous_dvodb = fullForceRun.dvodb)");
     305        // this is silly, there is no dvodb in skycalRun...?
     306      }
     307    }
    261308
    262309
    … …  
    324371        psStringAppend(&query, " GROUP BY skycal_id, sky_id, stack_id ");  //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there
    325372      }
     373      //needs to be checked HAF xxx
     374      if (strcmp(stage,"diff") == 0) {
     375        psStringAppend(&query, " GROUP BY diff_id, diff_skyfile_id ");  //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there
     376      }
     377      //needs to be checked HAF xxx
     378      if (strcmp(stage,"fullforce") == 0) {
     379        psStringAppend(&query, " GROUP BY ff_id, warp_id ");  //this needs checking, but I think it shoul be fine? it groups by lots of stuff (including stack - we only want one of each stack in there
     380      }
     381
     382
     383
    326384
    327385    psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query);
    … …  
    476534    }
    477535
     536    if (strcmp(stage,"diff") == 0) {
     537     
     538      for (long i = 0; i < psArrayLength(output); i++) {
     539        psMetadata *md = output->data[i];
     540
     541        diffRunRow *row = diffRunObjectFromMetadata(md);
     542       
     543        if (!row) {
     544          psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into diffRun");
     545          psFree(output);
     546          return false;
     547        }
     548
     549        if (!dvodb) {  // there's no skycalRun.dvodb
     550          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, diff_id %" PRId64, row->label, row->diff_id);
     551          psFree(output);
     552          return false;
     553        }
     554        if (!workdir && !row->workdir) {
     555          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, diff_id %" PRId64, row->label, row->diff_id);
     556          psFree(output);
     557          return false;
     558        }
     559
     560        psFree(row);
     561      }
     562    }
     563
     564    if (strcmp(stage,"fullforce") == 0) {
     565      for (long i = 0; i < psArrayLength(output); i++) {
     566        psMetadata *md = output->data[i];
     567
     568        fullForceRunRow *row = fullForceRunObjectFromMetadata(md);
     569
     570        if (!row) {
     571          psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fullforceRun");
     572          psFree(output);
     573          return false;
     574        }
     575
     576        if (!dvodb) {  // there's no skycalRun.dvodb
     577          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, ff_id %" PRId64, row->label, row->ff_id);
     578          psFree(output);
     579          return false;
     580        }
     581        if (!workdir && !row->workdir) {
     582          psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, ff_id %" PRId64, row->label, row->ff_id);
     583          psFree(output);
     584          return false;
     585        }
     586
     587        psFree(row);
     588      }
     589    }
    478590
    479591
    … …  
    672784      }
    673785   
     786    if (strcmp(stage,"diff") == 0) {
     787     
     788    for (long i = 0; i < psArrayLength(output); i++) {
     789        psMetadata *md = output->data[i];
     790        psS64 stage_id =0;
     791        //      psS64 stage_extra1 = 0;
     792        diffRunRow *row = diffRunObjectFromMetadata(md);
     793        stage_id = row->diff_id;
     794        bool status = false;
     795        psS32 stage_extra1 = psMetadataLookupS32(&status, md, "diff_skyfile_id");
     796        if (!status) {
     797          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item diff_skyfile_id");
     798          return false;
     799        }
     800
     801       
     802        if (!row) {
     803            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun");
     804            psFree(output);
     805            return false;
     806        }
     807       
     808        // queue the exp
     809        if (!pxaddQueueByCamID(config,
     810                               stage,
     811                               stage_id,
     812                               stage_extra1,
     813                               workdir     ? workdir   : row->workdir,
     814                               reduction   ? reduction : row->reduction,
     815                               label       ? label     : row->label,
     816                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
     817                               dvodb       ? dvodb     : NULL,
     818                               note        ? note      : NULL,
     819                               image_only,
     820                               minidvodb,
     821                               minidvodb_group,
     822                               minidvodb_name
     823        )) {
     824            if (!psDBRollback(config->dbh)) {
     825                psError(PS_ERR_UNKNOWN, false, "database error sfg");
     826            }
     827            psError(PS_ERR_UNKNOWN, false,
     828                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
     829            psFree(row);
     830            psFree(output);
     831            return false;
     832        }
     833        psFree(row);
     834    }
     835      }
     836 
     837    if (strcmp(stage,"fullforce") == 0) {
     838    for (long i = 0; i < psArrayLength(output); i++) {
     839        psMetadata *md = output->data[i];
     840        psS64 stage_id =0;
     841       
     842        fullForceRunRow *row = fullForceRunObjectFromMetadata(md);
     843        stage_id = row->ff_id;
     844        bool status = false;
     845        psS32 stage_extra1 = psMetadataLookupS32(&status, md, "warp_id");
     846        if (!status) {
     847          psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item warp_id");
     848          return false;
     849        }
     850
     851        if (!row) {
     852            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into skycalRun");
     853            psFree(output);
     854            return false;
     855        }
     856
     857        // queue the exp
     858        if (!pxaddQueueByCamID(config,
     859                               stage,
     860                               stage_id,
     861                               stage_extra1,
     862                               workdir     ? workdir   : row->workdir,
     863                               reduction   ? reduction : row->reduction,
     864                               label       ? label     : row->label,
     865                               data_group  ? data_group : (row->data_group ? row->data_group :  (label ? label : row->label)),
     866                               dvodb       ? dvodb     : NULL,
     867                               note        ? note      : NULL,
     868                               image_only,
     869                               minidvodb,
     870                               minidvodb_group,
     871                               minidvodb_name
     872        )) {
     873            if (!psDBRollback(config->dbh)) {
     874                psError(PS_ERR_UNKNOWN, false, "database error sfg");
     875            }
     876            psError(PS_ERR_UNKNOWN, false,
     877                    "failed to trying to queue stage %s %" PRId64,stage, stage_id);
     878            psFree(row);
     879            psFree(output);
     880            return false;
     881        }
     882        psFree(row);
     883    }
     884      }
     885
     886
     887
    674888
    675889
    … …  
    695909    PXOPT_COPY_S64(config->args, where, "-sky_id",    "staticskyRun.sky_id", "==");
    696910    PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "==");
     911    PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "==");
     912    PXOPT_COPY_S64(config->args, where, "-ff_id", "fullForceRun.ff_id", "==");
     913
    697914    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
    698915    pxcamGetSearchArgs (config, where); // most search arguments based on camera
    … …  
    709926    psString query = NULL;
    710927    if (strcmp(stage, "cam")==0) {
    711       query = psStringCopy("UPDATE addRun JOIN camRun on cam_id = stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)");
     928      query = psStringCopy("UPDATE addRun JOIN camRun on (cam_id = stage_id and stage = 'cam') JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)");
    712929    }
    713930    if (strcmp(stage, "stack")==0) {
    714       query = psStringCopy("UPDATE addRun JOIN stackRun on stack_id = stage_id");
     931      query = psStringCopy("UPDATE addRun JOIN stackRun on (stack_id = stage_id and stage = 'stack')");
    715932    }
    716933    if (strcmp(stage, "staticsky")==0) {
    717       query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id");
     934      query = psStringCopy("UPDATE addRun JOIN staticskyRun on (sky_id = stage_id and stage = 'staticsky')");
    718935    }
    719936    if (strcmp(stage, "skycal")==0) {
    720       query = psStringCopy("UPDATE addRun JOIN skycalRun on skycal_id = stage_id");
     937      query = psStringCopy("UPDATE addRun JOIN skycalRun on (skycal_id = stage_id and stage = 'skycal')");
     938    }
     939    if (strcmp(stage, "diff")==0) {
     940      query = psStringCopy("UPDATE addRun JOIN diffInputSkyfile on (diff_id = stage_id and diff_skyfile_id = stage_extra1 and stage = 'diff') JOIN diffSkyfile on (diff_id, skycell_id) JOIN diffRun using (diff_id)");
     941    }
     942    if (strcmp(stage, "fullforce")==0) {
     943      query = psStringCopy("UPDATE addRun JOIN fullForceResult on (ff_id = stage_id and warp_id = stage_extra1 and stage = 'fullforce') JOIN fullForceRun on (ff_id)");
    721944    }
    722945
    … …  
    743966    PXOPT_COPY_S64(config->args, where, "-sky_id",    "staticskyRun.sky_id", "==");
    744967    PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "==");
     968    PXOPT_COPY_S64(config->args, where, "-ff_id",    "fullForceRun.ff_id", "==");
     969    PXOPT_COPY_S64(config->args, where, "-diff_id",    "diffRun.diff_id", "==");
    745970    PXOPT_LOOKUP_STR(stage,       config->args, "-stage", false, false);
    746971    pxcamGetSearchArgs (config, where);
    … …  
    764989      query = pxDataGet("addtool_find_pendingexp_skycal.sql");
    765990    }
     991    if (strcmp(stage, "diff")==0) {
     992      query = pxDataGet("addtool_find_pendingexp_diff.sql");
     993    }
     994    if (strcmp(stage, "fullforce")==0) {
     995      query = pxDataGet("addtool_find_pendingexp_ff.sql");
     996    }
    766997
    767998
    … …  
    7901021      psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1");
    7911022    }
     1023    if (strcmp(stage, "diff") == 0) {
     1024      //this group by is needed to join against all the warps (to get camera)
     1025      psStringAppend(&query, " GROUP BY %s", "diff_id, stage_extra1");
     1026    }
     1027    if (strcmp(stage, "fullforce") == 0) {
     1028      //this group by is needed to join against all the warps (to get camera)
     1029      psStringAppend(&query, " GROUP BY %s", "ff_id, stage_extra1");
     1030    }
     1031
     1032
     1033
    7921034    // treat limit == 0 as "no limit"
    7931035    if (limit) {
    … …  
    9941236    } else if (strcmp (stage,"skycal") == 0) {
    9951237      query = pxDataGet("addtool_find_processedexp_skycal.sql");
     1238    } else if (strcmp (stage,"diff") == 0) {
     1239      query = pxDataGet("addtool_find_processedexp_diff.sql");
     1240    } else if (strcmp (stage,"fullforce") == 0) {
     1241      query = pxDataGet("addtool_find_processedexp_ff.sql");
     1242
    9961243    } else {
    9971244        psFree(where);
    … …  
    11121359    if (strcmp(stage, "skycal") == 0) {
    11131360      query = pxDataGet("addtool_revertprocessedexp_skycal.sql");
     1361    }
     1362    if (strcmp(stage, "diff") == 0) {
     1363      query = pxDataGet("addtool_revertprocessedexp_diff.sql");
     1364    }
     1365    if (strcmp(stage, "fullforce") == 0) {
     1366      query = pxDataGet("addtool_revertprocessedexp_ff.sql");
    11141367    }
    11151368

  • branches/eam_branches/ipp-20140904/ippTools/src/addtoolConfig.c

    r35228 r37804  
    5555    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-skycal_id",          0, "search by skycal_id", 0);
    5656    pxcamSetSearchArgs(definebyqueryArgs);
     57    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id",            0, "search by diff_id", 0);
     58    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-ff_id",              0, "search by ff_id", 0);
    5759    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL);
    5860    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by camRun data_group", NULL);
    … …  
    8385    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id",                 0, "search by sky_id", 0);
    8486    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-skycal_id",                 0, "search by skycal_id", 0);
     87    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id",                0, "search by diff_id",0);
     88    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-ff_id",                  0, "search by _ff_id",0);
    8589    pxcamSetSearchArgs(updaterunArgs);
    8690    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",                  0, "search by addRun label", NULL);
    … …  
    101105    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id",            0, "search by add_id", 0);
    102106    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id",            0, "search by cam_id", 0);
    103     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id",                 0, "search by stack_id", 0);
    104     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id",                 0, "search by sky_id", 0);
    105     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id",                 0, "search by skycal_id", 0);
     107    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id",                0, "search by stack_id", 0);
     108    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id",                  0, "search by sky_id", 0);
     109    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-skycal_id",               0, "search by skycal_id", 0);
     110    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-diff_id",                 0, "search by diff_id",0);
     111    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-ff_id",                   0, "search by ff_id",0);
    106112    pxcamSetSearchArgs(pendingexpArgs);
    107113    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL);
    … …  
    236242    psMetadata *modes = psMetadataAlloc();
    237243
    238     PXOPT_ADD_MODE("-definebyquery",        "create runs from cam stage",           ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
     244    PXOPT_ADD_MODE("-definebyquery",        "create runs from cam/skycal/forcedwarp/diff stage",           ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
    239245    PXOPT_ADD_MODE("-updaterun",            "change add run properties",            ADDTOOL_MODE_UPDATERUN,      updaterunArgs);
    240246    PXOPT_ADD_MODE("-pendingexp",           "show pending exps",                    ADDTOOL_MODE_PENDINGEXP,    pendingexpArgs);

  • branches/eam_branches/ipp-20140904/ippTools/src/chiptool.c

    r36431 r37804  
    14541454    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
    14551455    PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
    1456     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
    1457     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
     1456    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "==");
     1457    PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "==");
    14581458
    14591459    psString query = pxDataGet("chiptool_run.sql");

  • branches/eam_branches/ipp-20140904/ippTools/src/fftool.c

    r36665 r37804  
    859859    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE");
    860860    PXOPT_COPY_STR(config->args, where, "-filter",     "stackRun.filter", "LIKE");
    861     PXOPT_COPY_S16(config->args, where, "-fault",      "staticskyResult.fault", "==");
     861    PXOPT_COPY_S16(config->args, where, "-fault",      "fullForceSummary.fault", "==");
    862862    pxskycellAddWhere(config, where);
    863863

  • branches/eam_branches/ipp-20140904/ippTools/src/laptool.c

    r36555 r37804  
    2525static bool pendingrunMode(pxConfig *config);
    2626static bool updaterunMode(pxConfig *config);
     27static bool revertsassMode(pxConfig *config);
    2728
    2829// Exposure level
    … …  
    6970    MODECASE(LAPTOOL_MODE_PENDINGRUN,    pendingrunMode);
    7071    MODECASE(LAPTOOL_MODE_UPDATERUN,     updaterunMode);
    71 
     72    MODECASE(LAPTOOL_MODE_REVERTSASS,    revertsassMode);
     73   
    7274    MODECASE(LAPTOOL_MODE_PENDINGEXP,    pendingexpMode);
    7375    MODECASE(LAPTOOL_MODE_EXPOSURES,     exposuresMode);
    … …  
    695697}
    696698
     699static bool revertsassMode(pxConfig *config)
     700{
     701  PS_ASSERT_PTR_NON_NULL(config, false);
     702
     703  psMetadata *where = psMetadataAlloc();
     704  PXOPT_COPY_S64(config->args, where, "-lap_id", "lap_id", "==");
     705  PXOPT_COPY_S64(config->args, where, "-sass_id", "final_sass_id", "==");
     706  if (!psListLength(where->list)) {
     707    psFree(where);
     708    psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     709    return false;
     710  }
     711
     712  psString query = pxDataGet("laptool_revertsass.sql");
     713  if (!query) {
     714    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     715    return(false);
     716  }
     717  psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     718  psStringAppend(&query, "\n WHERE %s", whereClause);
     719  psFree(whereClause);
     720
     721  if (!psDBTransaction(config->dbh)) {
     722    psError(PS_ERR_UNKNOWN, false, "database error");
     723    return false;
     724  }
     725 
     726  if (!p_psDBRunQuery(config->dbh, query)) {
     727    psError(PS_ERR_UNKNOWN, false, "database error");
     728    psFree(query);
     729    return false;
     730  }
     731  psFree(query);
     732
     733  long numUpdated = psDBAffectedRows(config->dbh);
     734  if (numUpdated != 1) {
     735    psError(PS_ERR_UNKNOWN, false, "should have updated only one row. updated %ld", numUpdated);
     736    return false;
     737  }
     738
     739  if (!psDBCommit(config->dbh)) {
     740    psError(PS_ERR_UNKNOWN, false, "database error");
     741    return false;
     742  }
     743 
     744  return true;
     745}
    697746// Exposure level
    698747

  • branches/eam_branches/ipp-20140904/ippTools/src/laptool.h

    r35566 r37804  
    1919  LAPTOOL_MODE_EXPOSURES,
    2020  LAPTOOL_MODE_STACKS,
     21  LAPTOOL_MODE_REVERTSASS,
    2122  LAPTOOL_MODE_UPDATEEXP,
    2223  LAPTOOL_MODE_DIFFCHECK,

  • branches/eam_branches/ipp-20140904/ippTools/src/laptoolConfig.c

    r35568 r37804  
    117117  ADD_OPT(S64, updaterunArgs, "-set_final_sass_id",           "set final stack sass_id", 0);
    118118 
     119  // -revertsass
     120  psMetadata *revertsassArgs = psMetadataAlloc();
     121  ADD_OPT(S64, revertsassArgs, "-lap_id",                     "search by lap run id", 0);
     122  ADD_OPT(S64, revertsassArgs, "-sass_id",                    "sass id that is being removed", 0);
     123
    119124  // -pendingexp
    120125  psMetadata *pendingexpArgs = psMetadataAlloc();
    … …  
    136141  ADD_OPT(Bool,stacksArgs, "-simple",                         "use the simple output format", false);
    137142  ADD_OPT(U64, stacksArgs, "-limit",                          "limit result set to N items", 0);
    138  
     143         
    139144  // -updateexp
    140145  psMetadata *updateexpArgs = psMetadataAlloc();
    … …  
    227232  PXOPT_ADD_MODE("-listrun",                 "", LAPTOOL_MODE_PENDINGRUN,       listrunArgs);
    228233  PXOPT_ADD_MODE("-updaterun",               "", LAPTOOL_MODE_UPDATERUN,        updaterunArgs);
     234  PXOPT_ADD_MODE("-revertsass",              "mode to remove the sass_id", LAPTOOL_MODE_REVERTSASS,   revertsassArgs);
    229235  PXOPT_ADD_MODE("-pendingexp",              "", LAPTOOL_MODE_PENDINGEXP,       pendingexpArgs);
    230236  PXOPT_ADD_MODE("-exposures",               "", LAPTOOL_MODE_EXPOSURES,        exposuresArgs);

  • branches/eam_branches/ipp-20140904/ippTools/src/pxadd.c

    r34265 r37804  
    164164        psMemSetPersistent(query, true);
    165165      }
     166      else if (strcmp(stage,"diff") == 0) {
     167        query = pxDataGet("addtool_queue_diff_id.sql");
     168        psMemSetPersistent(query, true);
     169      }
     170      else if (strcmp(stage,"fullforce") == 0) {
     171        query = pxDataGet("addtool_queue_ff_id.sql");
     172        psMemSetPersistent(query, true);
     173      }
     174
     175
    166176      else
    167177        {

  • branches/eam_branches/ipp-20140904/ippTools/src/stacktool.c

    r37104 r37804  
    4141static bool sumskyfileMode(pxConfig *config);
    4242static bool sassskyfileMode(pxConfig *config);
     43static bool updatesassMode(pxConfig *config);
    4344static bool revertsumskyfileMode(pxConfig *config);
    4445static bool tosummaryMode(pxConfig *config);
    … …  
    8283        MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE,         addsumskyfileMode);
    8384        MODECASE(STACKTOOL_MODE_SUMSKYFILE,            sumskyfileMode);
    84         MODECASE(STACKTOOL_MODE_SASSSKYFILE,            sassskyfileMode);
     85        MODECASE(STACKTOOL_MODE_SASSSKYFILE,           sassskyfileMode);
     86        MODECASE(STACKTOOL_MODE_UPDATESASS,            updatesassMode);
    8587        MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE,      revertsumskyfileMode);
    8688        MODECASE(STACKTOOL_MODE_TOSUMMARY,             tosummaryMode);
    … …  
    756758    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "==");
    757759    PXOPT_COPY_STR(config->args, where, "-state",     "stackRun.state",     "==");
    758     PXOPT_COPY_S64(config->args, where, "-sass_id",   "stackAssociationMap.sass_id",  "==");
     760
    759761    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     762    PXOPT_LOOKUP_S64(sass_id, config->args, "-sass_id", false, false);
     763
     764    if (sass_id) {
     765      PXOPT_COPY_S64(config->args, where, "-sass_id",   "stackAssociationMap.sass_id",  "==");
     766    }
     767   
    760768    if (!psListLength(where->list)) {
    761769        psFree(where);
    … …  
    771779        PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
    772780    }
    773 
     781    if (sass_id) {
     782      psStringAppend(&query, " JOIN stackAssociationMap USING(stack_id)");
     783    }
     784   
    774785    // pxUpdateRun gets parameters from config->args and updates
    775786    bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true, false);
    … …  
    12951306    PXOPT_COPY_STR(config->args, where, "-data_group", "stackAssociation.data_group", "LIKE");
    12961307
    1297 //  The following three selectors are incompatible with the sql so omit them
    1298 //    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
    1299 //     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
    1300 //    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
    1301 
    13021308    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
    13031309
    … …  
    13691375}
    13701376
     1377static bool updatesassMode(pxConfig *config)
     1378{
     1379  PS_ASSERT_PTR_NON_NULL(config, false);
     1380
     1381  psMetadata *where = psMetadataAlloc();
     1382  PXOPT_COPY_S64(config->args, where, "-sass_id", "sass_id", "==");
     1383  PXOPT_COPY_STR(config->args, where, "-data_group", "data_group", "==");
     1384  PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "==");
     1385  PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
     1386  PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
     1387  if (!psListLength(where->list)) {
     1388    psFree(where);
     1389    psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
     1390    return false;
     1391  }
     1392
     1393  psMetadata *values = psMetadataAlloc();
     1394  PXOPT_COPY_STR(config->args, values, "-set_data_group", "data_group", "==");
     1395  if (!psListLength(values->list)) {
     1396    psFree(values);
     1397    psError(PXTOOLS_ERR_CONFIG, false, "data_group to assign not found");
     1398    return false;
     1399  }
     1400
     1401  if (!psDBTransaction(config->dbh)) {
     1402    psError(PS_ERR_UNKNOWN, false, "database error");
     1403    return false;
     1404  }
     1405 
     1406  long rows = psDBUpdateRows(config->dbh, "stackAssociation", where, values);
     1407  psFree(values);
     1408  psFree(where);
     1409
     1410  if (rows != 1) {
     1411    psError(PS_ERR_UNKNOWN, false, "should have updated only one row. updated %ld", rows);
     1412    return false;
     1413  }
     1414
     1415  if (!psDBCommit(config->dbh)) {
     1416    psError(PS_ERR_UNKNOWN, false, "database error");
     1417    return false;
     1418  }
     1419
     1420  return true;
     1421}
    13711422
    13721423static bool revertsumskyfileMode(pxConfig *config)

  • branches/eam_branches/ipp-20140904/ippTools/src/stacktool.h

    r36361 r37804  
    3535    STACKTOOL_MODE_SUMSKYFILE,
    3636    STACKTOOL_MODE_SASSSKYFILE,
     37    STACKTOOL_MODE_UPDATESASS,
    3738    STACKTOOL_MODE_REVERTSUMSKYFILE,
    3839    STACKTOOL_MODE_TOSUMMARY,

  • branches/eam_branches/ipp-20140904/ippTools/src/stacktoolConfig.c

    r37104 r37804  
    238238    psMetadataAddBool(sassskyfileArgs, PS_LIST_TAIL, "-all",  0,            "enable search without arguments", false);
    239239
     240    // -updatesass
     241    psMetadata *updatesassArgs = psMetadataAlloc();
     242    psMetadataAddS64(updatesassArgs, PS_LIST_TAIL, "-sass_id", 0,           "search by stack association id", 0);
     243    psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-data_group", 0,        "search by data group", 0);
     244    psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-projection_cell", 0,   "search by projection cell", 0);
     245    psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-tess_id", 0,           "search by tess_id", 0);
     246    psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-filter", 0,            "search by filter", 0);
     247
     248    psMetadataAddStr(updatesassArgs, PS_LIST_TAIL, "-set_data_group", 0,    "data_group to assign", 0);
     249
    240250    // -revertsumskyfile
    241251    psMetadata *revertsumskyfileArgs= psMetadataAlloc();
    … …  
    348358    PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     STACKTOOL_MODE_DONECLEANUP,          donecleanupArgs);
    349359
    350     PXOPT_ADD_MODE("-sassskyfile",      "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE,      sassskyfileArgs);
     360    PXOPT_ADD_MODE("-sassskyfile",          "list results of stackAssociation", STACKTOOL_MODE_SASSSKYFILE,      sassskyfileArgs);
     361    PXOPT_ADD_MODE("-updatesass",           "update data_group of stackAssociation", STACKTOOL_MODE_UPDATESASS,  updatesassArgs);
    351362    PXOPT_ADD_MODE("-tosummary",            "show runs that can be summarized", STACKTOOL_MODE_TOSUMMARY, tosummaryArgs);
    352363    PXOPT_ADD_MODE("-summary",              "show runs that have been summarized", STACKTOOL_MODE_SUMMARY, summaryArgs);

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageAddNoise.c

    r31155 r37804  
    1818    return true;
    1919  }
     20
     21  // psTimerStart("add.noise");
    2022
    2123  // grizy variances to add to turn MD exposure -> 3pi, calculated from the DRM
    … …  
    9698  psFree(stats);
    9799 
     100  // psLogMsg ("ppImage", 5, "add noise: %f sec\n", psTimerMark ("add.noise"));
     101
    98102  return true;
    99103}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageAuxiliaryMask.c

    r36169 r37804  
    6767        return true;
    6868    }
     69
     70    // psTimerStart("aux.mask");
    6971
    7072    bool status;
    … …  
    208210    }
    209211
     212    // psLogMsg ("ppImage", 5, "auxiliary mask: %f sec\n", psTimerMark ("aux.mask"));
     213
    210214    return true;
    211215}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageCheckCTE.c

    r24117 r37804  
    1717        return true;
    1818    }
     19
     20    // psTimerStart("check.cte");
    1921
    2022    // add recipe options supplied on command line
    … …  
    106108    psFree (statsDefaults);
    107109
     110    // psLogMsg ("ppImage", 5, "check CTE: %f sec\n", psTimerMark ("check.cte"));
     111
    108112    return true;
    109113}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageCheckNoise.c

    r33667 r37804  
    1717        return true;
    1818    }
     19
     20    // psTimerStart("check.noise");
    1921
    2022    // add recipe options supplied on command line
    … …  
    124126    psFree (statsDefaults);
    125127
     128    // psLogMsg ("ppImage", 5, "check noise: %f sec\n", psTimerMark ("check.noise"));
     129
    126130    return true;
    127131}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageDetrendFringe.c

    r26895 r37804  
    327327    pmCell *cell = NULL;
    328328
     329    // psTimerStart("apply.fringe");
     330
    329331    assert (options->doFringe); // do not call if not needed
    330332    assert (inputView->chip != -1);
    … …  
    386388    }
    387389
     390    // psLogMsg ("ppImage", 5, "apply fringe: %f sec\n", psTimerMark ("apply.fringe"));
     391
    388392    psFree (view);
    389393    return true;

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageDetrendPattern.c

    r33243 r37804  
    3535    // chip.
    3636
     37    // psTimerStart("apply.pattern");
    3738    if (options->doPatternRow) {
     39
    3840        bool status;
    3941        pmHDU *hdu = pmHDUFromChip(chip);
    … …  
    193195
    194196 pattern_done:
     197
     198    // psLogMsg ("ppImage", 5, "apply pattern: %f sec\n", psTimerMark ("apply.pattern"));
     199
    195200    return(true);
    196201}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageDetrendReadout.c

    r35917 r37804  
    77bool ppImageDetrendReadout(pmConfig *config, ppImageOptions *options, pmFPAview *view)
    88{
     9  // psTimerStart("detrend.readout");
     10
    911    // construct a view for the detrend images (which have only one readout)
    1012    pmFPAview *detview = pmFPAviewAlloc(0);
    … …  
    3537      }
    3638    }
     39    // psLogMsg ("ppImage", 6, "check gain: %f sec\n", psTimerMark ("detrend.readout"));
    3740
    3841    // Check to see if we're in a chip that contains video
    … …  
    5659    }
    5760    done_video_check:
     61    // psLogMsg ("ppImage", 6, "check video: %f sec\n", psTimerMark ("detrend.readout"));
    5862
    5963    // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine.
    … …  
    6670          return false;
    6771        }
     72        // psLogMsg ("ppImage", 6, "generate mask: %f sec\n", psTimerMark ("detrend.readout"));
    6873    }
    6974    // apply the externally supplied mask to the input->mask pixels
    … …  
    7782      }
    7883      pmMaskBadPixels(input, mask, options->maskValue);
     84      // psLogMsg ("ppImage", 6, "apply mask: %f sec\n", psTimerMark ("detrend.readout"));
    7985    }
    8086
    … …  
    8389        // extern bool ppImageBurntoolApply(pmConfig *, ppImageOptions *, pmFPAview *, pmReadout *);
    8490        ppImageBurntoolApply(config, options, view, input);
     91        // psLogMsg ("ppImage", 6, "apply burntool: %f sec\n", psTimerMark ("detrend.readout"));
    8592    }
    8693   
    … …  
    93100            ppImageBurntoolMask(config,options,view,input);
    94101        }
     102        // psLogMsg ("ppImage", 6, "apply burntool mask: %f sec\n", psTimerMark ("detrend.readout"));
    95103    }
    96104
    … …  
    102110        return false;
    103111      }
     112      // psLogMsg ("ppImage", 6, "subtract overscan: %f sec\n", psTimerMark ("detrend.readout"));
    104113    }
    105114
    … …  
    111120        return(false);
    112121      }
     122      // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout"));
    113123    }
    114124
    … …  
    148158            return false;
    149159        }
     160        // psLogMsg ("ppImage", 6, "apply bias: %f sec\n", psTimerMark ("detrend.readout"));
    150161    }
    151162   
    … …  
    218229        }
    219230        psFree (noiseImage);
     231        // psLogMsg ("ppImage", 6, "generate variance: %f sec\n", psTimerMark ("detrend.readout"));
    220232    }
    221233
    … …  
    226238            return false;
    227239        }
     240        // psLogMsg ("ppImage", 6, "apply dark: %f sec\n", psTimerMark ("detrend.readout"));
    228241    }
    229242
    … …  
    235248            return false;
    236249        }
     250        // psLogMsg ("ppImage", 6, "mask remnance: %f sec\n", psTimerMark ("detrend.readout"));
    237251    }
    238252
    … …  
    244258            return false;
    245259        }
     260        // psLogMsg ("ppImage", 6, "shutter correction: %f sec\n", psTimerMark ("detrend.readout"));
    246261    }
    247262
    … …  
    253268            return false;
    254269        }
     270        // psLogMsg ("ppImage", 6, "apply flat: %f sec\n", psTimerMark ("detrend.readout"));
    255271    }
    256272
    … …  
    276292
    277293        psBinaryOp(input->image, input->image, "*", psScalarAlloc(norm, PS_TYPE_F32));
     294        // psLogMsg ("ppImage", 6, "renormalize: %f sec\n", psTimerMark ("detrend.readout"));
    278295    }
    279296
    … …  
    324341
    325342        psFree (classID);
     343
     344        // psLogMsg ("ppImage", 6, "renormalize by class: %f sec\n", psTimerMark ("detrend.readout"));
    326345    }
    327346# endif
    … …  
    333352            return false;
    334353        }
     354        // psLogMsg ("ppImage", 6, "measure fringe: %f sec\n", psTimerMark ("detrend.readout"));
    335355    }
    336356
    337357    ppImageMemoryDump("detrend");
    338358
     359    // psLogMsg ("ppImage", 5, "detrend readout: %f sec\n", psTimerMark ("detrend.readout"));
    339360    psFree(detview);
    340361    return true;

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageMosaic.c

    r28292 r37804  
    99{
    1010    bool status;                        // Status of MD lookup
     11
     12    // psTimerStart("mosaic.chip");
    1113
    1214    pmFPAfile *in = psMetadataLookupPtr(&status, config->files, inFile); // Input file
    … …  
    6365    }
    6466
     67    // psLogMsg ("ppImage", 5, "mosaic chip: %f sec\n", psTimerMark ("mosaic.chip"));
     68
    6569    return status;
    6670}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageParityFlip.c

    r25875 r37804  
    1919
    2020    if (!options->applyParity) return true;
     21
     22    // psTimerStart("parity.flip");
    2123
    2224    // find the currently selected readout
    … …  
    152154        }
    153155    }
     156    // psLogMsg ("ppImage", 5, "parity flip: %f sec\n", psTimerMark ("parity.flip"));
    154157
    155158    FIX_CONCEPT(cell->concepts, "CELL.XPARITY", S32, xParityTarget);

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImagePixelStats.c

    r19928 r37804  
    1010{
    1111    bool mdok;              // Status of MD lookup
     12
     13    // psTimerStart("pixel.stats");
    1214
    1315    // loop over the cells/readouts for this chip
    … …  
    8183    }
    8284
     85    // psLogMsg ("ppImage", 5, "measure pixel stats: %f sec\n", psTimerMark ("pixel.stats"));
     86
    8387    psFree (view);
    8488    return true;

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageRebinReadout.c

    r20439 r37804  
    1212    pmFPAfile *outFile = psMetadataLookupPtr(NULL, config->files, outName);
    1313    if (outFile == NULL) return false;
     14
     15    // psTimerStart("rebin.chip");
    1416
    1517    // XXX double check that chip != -1?
    … …  
    4042    }
    4143
     44    // psLogMsg ("ppImage", 5, "rebin chip for %s: %f sec\n", outName, psTimerMark ("rebin.chip"));
     45
    4246    return true;
    4347}

  • branches/eam_branches/ipp-20140904/ppImage/src/ppImageSetMaskBits.c

    r25829 r37804  
    1111        return false;
    1212    }
     13
     14    // psTimerStart("set.mask.bits");
    1315
    1416    // at this point we know we have valid values for required entries SAT, BAD, FLAT, BLANK:
    … …  
    5759                           options->markValue);
    5860
     61    // psLogMsg ("ppImage", 5, "set mask bits: %f sec\n", psTimerMark ("set.mask.bits"));
     62
    5963    return true;
    6064}

  • branches/eam_branches/ipp-20140904/ppTranslate/src/ppMops.h

    r34824 r37804  
    66
    77#define IN_EXTNAME "SkyChip.psf"        // Extension name for data in input
    8 #define OBSERVATORY_CODE "F51"          // IAU Observatory Code
     8// #define OBSERVATORY_CODE "F51"          // IAU Observatory Code
    99#define OUT_EXTNAME "MOPS_TRANSIENT_DETECTIONS" // Extension name for data in output
    1010#define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_SATURATED | \
    … …  
    2323  psS64 warp_id;                      // Warp stage identifier
    2424  psS64 diff_id;                      // Diff stage identifier
     25  psString camera;                    // Camera name
     26  psString obscode;                   // observatory code (derived from camera)
    2527  bool positive;                      // Sense of subtraction, T=positive, F=negative
    2628  float zp, zpErr;                    // Magnitude zero point and error
    … …  
    5961  psVector *raExtErr, *decExtErr;     // Error in sky coordinates for RA_EXT and DEC_EXT
    6062  psVector *mask;                     // Mask for detections
    61     float platescale; //Plate scale at centroid
     63  float platescale;                   // Plate scale at centroid
     64  psString fpashutoutc;               // FPA shutoutc
    6265} ppMopsDetections;
    6366

  • branches/eam_branches/ipp-20140904/ppTranslate/src/ppMopsArguments.c

    r34474 r37804  
    4242static void mopsArgumentsFree(ppMopsArguments *args)
    4343{
     44    psFree(args->camera);
     45    psFree(args->obscode);
    4446    psFree(args->input);
    4547    psFree(args->exp_name);
    … …  
    6365    args->warp_id = 0;
    6466    args->diff_id = 0;
     67    args->camera = NULL;
     68    args->obscode = NULL;
    6569    args->zp = NAN;
    6670    args->positive = true;
    … …  
    9498    psMetadataAddS64(arguments, PS_LIST_TAIL, "-warp_id", 0, "Warp stage identifier", 0);
    9599    psMetadataAddS64(arguments, PS_LIST_TAIL, "-diff_id", 0, "Diff stage identifier", 0);
     100    psMetadataAddStr(arguments, PS_LIST_TAIL, "-camera", 0, "Camera name", "No comment provided");
    96101    psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Inverse subtraction?", false);
    97102    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Magnitude zero point", NAN);
    … …  
    126131    args->warp_id = psMetadataLookupS64(NULL, arguments, "-warp_id");
    127132    args->diff_id = psMetadataLookupS64(NULL, arguments, "-diff_id");
     133    args->camera = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-camera"));
    128134    args->positive = !psMetadataLookupBool(NULL, arguments, "-inverse"); // NOTE: negated
    129135    args->zp = psMetadataLookupF32(NULL, arguments, "-zp");
    … …  
    137143    args->shutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-shutoutc"));
    138144
     145    if (!args->camera) {
     146        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "camera name not provided.");
     147        return NULL;
     148    }
     149    if (!strcasecmp(args->camera, "GPC1")) {
     150      args->obscode = psStringCopy("F51");
     151    }
     152    if (!strcasecmp(args->camera, "GPC2")) {
     153      args->obscode = psStringCopy("F52");
     154    }
     155    if (!args->obscode) {
     156        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "unknown camera name.");
     157        return NULL;
     158    }
     159
    139160    psTrace("ppMops.args", 1, "Done parsing command-line arguments\n");
    140161

  • branches/eam_branches/ipp-20140904/ppTranslate/src/ppMopsDetections.c

    r34824 r37804  
    2020    psFree(det->decErr);
    2121    psFree(det->mask);
     22    psFree(det->fpashutoutc);
    2223    return;
    2324}
    … …  
    4849    det->mask = NULL;
    4950    det->diffSkyfileId = 0;
     51    det->fpashutoutc = NULL;
    5052    return det;
    5153}

  • branches/eam_branches/ipp-20140904/ppTranslate/src/ppMopsRead.c

    r34824 r37804  
    105105    det->naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns
    106106    det->naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); // Number of rows
     107    det->fpashutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.SHUTOUTC"));
    107108    psFree(header);
    108109
    … …  
    241242    long numGood = 0;               // Number of good rows
    242243    for (long row = 0; row < size; row++) {
    243       psU32 flags = flagsV->data.U32[row]; // psFitsTableGetU32(NULL, table, row, "FLAGS");
     244      //psU32 flags = flagsV->data.U32[row]; // psFitsTableGetU32(NULL, table, row, "FLAGS");
     245      psU64 flags = flagsV->data.U64[row]; // table reads in as U64
    244246      if (flags & SOURCE_MASK) {
    245         psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %ud", row, i, flags);
     247        psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %lud", row, i, flags);
    246248        det->mask->data.U8[row] = 0xFF;
    247249        continue;

  • branches/eam_branches/ipp-20140904/ppTranslate/src/ppMopsWrite.c

    r34779 r37804  
    8989    psWarning("no inputs with surviving detections. output header will be incomplete");
    9090  }
    91   psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE);
     91  psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", args->obscode);
    9292  psMetadataAddF32(header, PS_LIST_TAIL, "MAGZP", 0, "Magnitude zero point", args->zp);
    9393  //MOPS want the name ZEROPOINT for MAGZP
    … …  
    102102  psMetadataAddF32(header, PS_LIST_TAIL, "PLTSCALE_EXT", 0, "Plate scale at centroid", det->platescale);
    103103  psMetadataAddF32(header, PS_LIST_TAIL, "PLTSCALE", 0, "Plate scale at centroid", det->platescale);
     104  psMetadataAddStr(header, PS_LIST_TAIL, "FPA.SHUTOUTC", 0, "Time of exposure", det->fpashutoutc);
    104105
    105106  //field in header that tells about the CMF version

  • branches/eam_branches/ipp-20140904/psLib/src/imageops/psImageMapFit.c

    r34153 r37804  
    1818
    1919#include <stdio.h>
     20#include <stdlib.h>
    2021#include "psError.h"
    2122#include "psAbort.h"
    … …  
    3536#include "psImageStructManip.h"
    3637#include "psImageMap.h"
     38#include "psSparse.h"
    3739// #include "psImagePixelInterpolate.h"
    3840// #include "psImageUnbin.h"
    … …  
    118120    psImageInit (A, 0.0);
    119121    psVectorInit (B, 0.0);
    120 
     122   
    121123    // we are looping over the Nx,Ny image map elements;
    122124    // the matrix equation contains Nx*Ny rows and columns
    … …  
    262264            int I = n + Nx * m;
    263265            B->data.F32[I] = fi_rx_ry + fi_rx_py + fi_px_ry + fi_px_py;
    264 
     266           
    265267            // insert these values into their corresponding locations in A, B
    266268            // float Sum = 0.0;
    … …  
    273275                    int J = (n + jn) + Nx * (m + jm);
    274276                    A->data.F32[J][I] = sA[jn][jm];
     277                   
    275278                    // fprintf (stderr, "A %d %d (%d %d : %d %d): %f\n", I, J, n, m, n + jn, m + jm, sA[jn][jm]);
    276279                    // Sum += sA[jn][jm];
    … …  
    320323        return true;
    321324    }
    322 
     325   
    323326    // set bad values to NaN
    324327    for (int i = 0; i < Nx*Ny; i++) {
    … …  
    341344    psFree (B);
    342345    psFree (Empty);
    343 
    344346    *pGoodFit = true;
    345347    return true;
    … …  
    402404        psS32 Nkeep = 0;
    403405        if (!psImageMapFit(pGoodFit, map, mask, maskValue, x, y, f, df)) {
     406            psError(PS_ERR_UNKNOWN, false, "Could not fit image map.\n");
     407            psFree(resid);
     408            if (!inMask) psFree (mask);
     409            return false;
     410        }
     411        if (!*pGoodFit) {
     412            psWarning ("bad fit to image map, try something else");
     413            psFree(resid);
     414            if (!inMask) psFree (mask);
     415            return true;
     416        }
     417
     418        psVector *fit = psImageMapEvalVector(map, mask, maskValue, x, y);
     419        if (fit == NULL) {
     420            psError(PS_ERR_UNKNOWN, false, "Failure in psImageMapEvalVector().\n");
     421            psFree(resid);
     422            if (!inMask) psFree (mask);
     423            return false;
     424        }
     425        for (int i = 0 ; i < f->n ; i++) {
     426            resid->data.F32[i] = (f->data.F32[i] - fit->data.F32[i]);
     427        }
     428
     429        if (!psVectorStats(stats, resid, NULL, mask, maskValue)) {
     430            psError(PS_ERR_UNKNOWN, false, "Failure to compute statistics on the resid vector.\n");
     431            psFree(resid);
     432            psFree(fit);
     433            if (!inMask) psFree (mask);
     434            return false;
     435        }
     436
     437        double meanValue = psStatsGetValue (stats, meanOption);
     438        double stdevValue = psStatsGetValue (stats, stdevOption);
     439
     440        psTrace("psLib.imageops", 5, "Mean is %f\n", meanValue);
     441        psTrace("psLib.imageops", 5, "Stdev is %f\n", stdevValue);
     442        psF32 minClipValue = -minClipSigma*stdevValue;
     443        psF32 maxClipValue = +maxClipSigma*stdevValue;
     444
     445        // set mask if pts are not valid
     446        // we are masking out any point which is out of range
     447        // recovery is not allowed with this scheme
     448        for (psS32 i = 0; i < resid->n; i++) {
     449            // XXX this prevents recovery of previously masked values
     450            if (mask->data.PS_TYPE_VECTOR_MASK_DATA[i] & maskValue) {
     451                continue;
     452            }
     453
     454            if ((resid->data.F32[i] - meanValue > maxClipValue) || (resid->data.F32[i] - meanValue < minClipValue)) {
     455                psTrace("psLib.imageops", 6, "Masking element %d  : %f vs %f : resid is %f\n", i, f->data.F32[i], fit->data.F32[i], resid->data.F32[i]);
     456                mask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= 0x01;
     457                continue;
     458            }
     459            Nkeep++;
     460        }
     461
     462        // We should probably exit this loop if no new elements were masked since the fit won't
     463        // change.
     464        psTrace("psLib.imageops", 6, "keeping %d of %ld pts for fit\n", Nkeep, x->n);
     465        stats->clippedNvalues = Nkeep;
     466        psFree(fit);
     467    }
     468
     469    // Free local temporary variables
     470    psFree(resid);
     471    if (!inMask) psFree (mask);
     472    *pGoodFit = true; // XXX probably don't need to set this (set by psImageMapFit)
     473    return true;
     474}
     475
     476// CZW: 2014-10-09
     477// Sparse versions of MapFit and MapFitClip that assume the matrices are not filled.
     478bool psImageMapFitSparse(bool *pGoodFit, psImageMap *map, const psVector *mask, psVectorMaskType maskValue,
     479                   const psVector *x, const psVector *y, const psVector *f, const psVector *df)
     480{
     481    // XXX Add Asserts
     482
     483    *pGoodFit = false;
     484
     485    // dimensions of the output map image
     486    int Nx = map->binning->nXruff;
     487    int Ny = map->binning->nYruff;
     488
     489   
     490    // no spatial information, just calculate mean & stdev
     491    if ((Nx == 1) && (Ny == 1)) {
     492        psStatsInit(map->stats);
     493
     494        // the user has supplied one of various stats option pairs,
     495        psStatsOptions mean = psStatsMeanOption(map->stats->options);
     496        psStatsOptions stdev = psStatsStdevOption(map->stats->options);
     497        if (!psStatsSingleOption(mean)) {
     498            psError(PS_ERR_UNKNOWN, true, "no valid mean stats option selected");
     499            return false;
     500        }
     501        if (!psStatsSingleOption(stdev)) {
     502            psError(PS_ERR_UNKNOWN, true, "no valid stdev stats option selected");
     503            return false;
     504        }
     505
     506        // XXX does ROBUST_MEDIAN work with weight?
     507        if (!psVectorStats(map->stats, f, NULL, mask, maskValue)) {
     508            psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
     509            return false;
     510        }
     511
     512        map->map->data.F32[0][0]   = psStatsGetValue(map->stats, mean);
     513        map->error->data.F32[0][0] = psStatsGetValue(map->stats, stdev);
     514        if (isfinite(map->map->data.F32[0][0]) && isfinite( map->error->data.F32[0][0])) {
     515            *pGoodFit = true;
     516        }
     517        return true;
     518    }
     519
     520    if (Nx == 1) {
     521        bool status;
     522        status = psImageMapFit1DinY (pGoodFit, map, mask, maskValue, x, y, f, df);
     523        return status;
     524    }
     525    if (Ny == 1) {
     526        bool status;
     527        status = psImageMapFit1DinX (pGoodFit, map, mask, maskValue, x, y, f, df);
     528        return status;
     529    }
     530
     531    // set up the redirection table so we can use sA[-1][-1], etc
     532    // XXX psKernel does this for you --- PAP.
     533    float SAm[3][3], *SAv[3], **sA;
     534
     535    for (int i = 0; i < 3; i++) {
     536        SAv[i] = SAm[i] + 1;
     537    }
     538    sA = SAv + 1;
     539
     540    // elements of the matrix equation Ax = B; we are solving for the vector x
     541    // psImage *A = psImageAlloc (Nx*Ny, Nx*Ny, PS_TYPE_F32);
     542    // psVector *B = psVectorAlloc (Nx*Ny, PS_TYPE_F32);
     543
     544    // psImageInit (A, 0.0);
     545    // psVectorInit (B, 0.0);
     546
     547    // CZW: call to psSparseAlloc
     548    // It should match old A, and each element of that should only touch four others.
     549    psSparse *Asparse = psSparseAlloc(Nx * Ny, 4 * Nx * Ny);
     550   
     551    // we are looping over the Nx,Ny image map elements;
     552    // the matrix equation contains Nx*Ny rows and columns
     553
     554    for (int n = 0; n < Nx; n++) {
     555        for (int m = 0; m < Ny; m++) {
     556            // define & init summing variables
     557            float rx_rx_ry_ry = 0;
     558            float rx_rx_dy_ry = 0;
     559            float dx_rx_ry_ry = 0;
     560            float dx_rx_dy_ry = 0;
     561            float fi_rx_ry    = 0;
     562            float rx_rx_py_py = 0;
     563            float rx_rx_qy_py = 0;
     564            float dx_rx_py_py = 0;
     565            float dx_rx_qy_py = 0;
     566            float fi_rx_py    = 0;
     567            float px_px_ry_ry = 0;
     568            float px_px_dy_ry = 0;
     569            float qx_px_ry_ry = 0;
     570            float qx_px_dy_ry = 0;
     571            float fi_px_ry    = 0;
     572            float px_px_py_py = 0;
     573            float px_px_qy_py = 0;
     574            float qx_px_py_py = 0;
     575            float qx_px_qy_py = 0;
     576            float fi_px_py    = 0;
     577
     578            // generate the sums for the fitting matrix element I,J
     579            // I = n + nX*m
     580            // J = (n + jn) + nX*(m + jm)
     581            for (int i = 0; i < x->n; i++) {
     582
     583                if (mask && (mask->data.PS_TYPE_VECTOR_MASK_DATA[i] & maskValue)) continue;
     584
     585                // base coordinate offset for this point (x,y) relative to this map element (n,m)
     586                float dx = psImageBinningGetRuffX (map->binning, x->data.F32[i]) - (n + 0.5);
     587                float dy = psImageBinningGetRuffY (map->binning, y->data.F32[i]) - (m + 0.5);
     588
     589                // edge cases to include:
     590                bool edgeX = false;
     591                edgeX |= ((n == 1) && (dx < -1.0));
     592                edgeX |= ((n == Nx - 2) && (dx > +1.0));
     593
     594                bool edgeY = false;
     595                edgeY |= ((m == 1) && (dy < -1.0));
     596                edgeY |= ((m == Ny - 2) && (dy > +1.0));
     597
     598                // skip points outside of 2x2 grid centered on n,m:
     599                if (!edgeX && (fabs(dx) > 1.0)) continue;
     600                if (!edgeY && (fabs(dy) > 1.0)) continue;
     601
     602                // related offset values
     603                float rx = 1.0 - dx;
     604                float ry = 1.0 - dy;
     605                float px = 1.0 + dx;
     606                float py = 1.0 + dy;
     607                float qx = -dx;
     608                float qy = -dy;
     609
     610                // data value & weight for this point
     611                float fi = f->data.F32[i];
     612                if (!isfinite(fi)) continue;
     613
     614                float wt = 1.0;
     615                if (df != NULL) {
     616                    if (df->data.F32[i] == 0.0) {
     617                        wt = 0.0;
     618                    } else {
     619                        if (!isfinite(df->data.F32[i])) continue;
     620                        wt = 1.0 / PS_SQR(df->data.F32[i]); // XXX test for dz == NULL or dz_i = 0
     621                    }
     622                }
     623
     624                // sum the appropriate elements for the different quadrants
     625
     626                int Qx = (dx >= 0) ? 1 : 0;
     627                if (n ==      0) Qx = 1;
     628                if (n == Nx - 1) Qx = 0;
     629
     630                int Qy = (dy >= 0) ? 1 : 0;
     631                if (m ==      0) Qy = 1;
     632                if (m == Ny - 1) Qy = 0;
     633
     634                assert (isfinite(fi));
     635                assert (isfinite(wt));
     636                assert (isfinite(rx));
     637                assert (isfinite(ry));
     638
     639                // points at offset 1,1
     640                if ((Qx == 1) && (Qy == 1)) {
     641                    rx_rx_ry_ry += rx*rx*ry*ry*wt;
     642                    rx_rx_dy_ry += rx*rx*dy*ry*wt;
     643                    dx_rx_ry_ry += dx*rx*ry*ry*wt;
     644                    dx_rx_dy_ry += dx*rx*dy*ry*wt;
     645                    fi_rx_ry    += fi*rx*ry*wt;
     646                }
     647                // points at offset 1,0
     648                if ((Qx == 1) && (Qy == 0)) {
     649                    rx_rx_py_py += rx*rx*py*py*wt;
     650                    rx_rx_qy_py += rx*rx*qy*py*wt;
     651                    dx_rx_py_py += dx*rx*py*py*wt;
     652                    dx_rx_qy_py += dx*rx*qy*py*wt;
     653                    fi_rx_py    += fi*rx*py*wt;
     654                }
     655                // points at offset 0,1
     656                if ((Qx == 0) && (Qy == 1)) {
     657                    px_px_ry_ry += px*px*ry*ry*wt;
     658                    px_px_dy_ry += px*px*dy*ry*wt;
     659                    qx_px_ry_ry += qx*px*ry*ry*wt;
     660                    qx_px_dy_ry += qx*px*dy*ry*wt;
     661                    fi_px_ry    += fi*px*ry*wt;
     662                }
     663                // points at offset 0,0
     664                if ((Qx == 0) && (Qy == 0)) {
     665                    px_px_py_py += px*px*py*py*wt;
     666                    px_px_qy_py += px*px*qy*py*wt;
     667                    qx_px_py_py += qx*px*py*py*wt;
     668                    qx_px_qy_py += qx*px*qy*py*wt;
     669                    fi_px_py    += fi*px*py*wt;
     670                }
     671            }
     672
     673            // the chi-square derivatives have elements of the form g(n+jn,m+jm)*A(jn,jm),
     674            // jn,jm = -1 to +1. Convert the sums above into the correct coefficients
     675            sA[-1][-1] = qx_px_qy_py;
     676            sA[-1][ 0] = qx_px_ry_ry + qx_px_py_py;
     677            sA[-1][+1] = qx_px_dy_ry;
     678            sA[ 0][-1] = rx_rx_qy_py + px_px_qy_py;
     679            sA[ 0][ 0] = rx_rx_ry_ry + px_px_ry_ry + rx_rx_py_py + px_px_py_py;
     680            sA[ 0][+1] = rx_rx_dy_ry + px_px_dy_ry;
     681            sA[+1][-1] = dx_rx_qy_py;
     682            sA[+1][ 0] = dx_rx_ry_ry + dx_rx_py_py;
     683            sA[+1][+1] = dx_rx_dy_ry;
     684
     685            // I[ 0][ 0] = index for this n,m element:
     686            int I = n + Nx * m;
     687            //            B->data.F32[I] = fi_rx_ry + fi_rx_py + fi_px_ry + fi_px_py;
     688            // CZW: call to psSparseVector Element
     689            if (fi_rx_ry + fi_rx_py + fi_px_ry + fi_px_py == 0.0) {
     690              psSparseVectorElement(Asparse, I, 1.0);
     691            }
     692            else {
     693              psSparseVectorElement(Asparse, I, fi_rx_ry + fi_rx_py + fi_px_ry + fi_px_py);
     694            }
     695           
     696            //      printf("ADDING: %d %g \n",I, fi_rx_ry + fi_rx_py + fi_px_ry + fi_px_py);
     697            // insert these values into their corresponding locations in A, B
     698            for (int jn = -1; jn <= +1; jn++) {
     699                if (n + jn <   0) continue;
     700                if (n + jn >= Nx) continue;
     701                for (int jm = -1; jm <= +1; jm++) {
     702                    if (m + jm <   0) continue;
     703                    if (m + jm >= Ny) continue;
     704                    int J = (n + jn) + Nx * (m + jm);
     705                    //              printf("A: %d %d %g\n",J,I,sA[jn][jm]);
     706                    // A->data.F32[J][I] = sA[jn][jm];
     707                    // CZW: call to psSparseMatrixElement
     708                    if (J < I) { continue; }
     709                    psSparseMatrixElement(Asparse,J,I,sA[jn][jm]); // Ensure J < I?
     710                   
     711                }
     712            }
     713        }
     714    }
     715
     716    // test for empty diagonal elements (unconstained cells), mark, and set pivots to 1.0
     717    // CZW: I'm not totally sure how to check these in the sparse context.
     718    // Iterate over all ii pairs, and manually check the structure?
     719#if (0)
     720    psVector *Empty = psVectorAlloc (Nx*Ny, PS_TYPE_S8);
     721    psVectorInit (Empty, 0);
     722    for (int i = 0; i < Nx*Ny; i++) {
     723        if (A->data.F32[i][i] == 0.0) {
     724            Empty->data.S8[i] = 1;
     725            for (int j = 0; j < Nx*Ny; j++) {
     726                A->data.F32[i][j] = 0.0;
     727                A->data.F32[j][i] = 0.0;
     728            }
     729            A->data.F32[i][i] = 1.0;
     730            B->data.F32[i] = 0.0;
     731        }
     732    }
     733#endif
     734
     735    // CZW: call to psSparseSolve
     736    psVector *solution = psVectorAlloc(Nx*Ny, PS_TYPE_F32);
     737    psSparseConstraint Constraint;
     738    Constraint.paramDelta = 1e-3;
     739    Constraint.paramMin   = -1e5;
     740    Constraint.paramMax   = 1e5;
     741    solution = psSparseSolve(solution, Constraint, Asparse, 1000);
     742    if (!solution) {
     743      psFree(solution);
     744      psFree(Asparse);
     745      return(false);
     746    }
     747
     748#if (0)
     749    // CZW: This is a continuation of the above information
     750    // set bad values to NaN
     751    for (int i = 0; i < Nx*Ny; i++) {
     752        if (Empty->data.S8[i]) {
     753            B->data.F32[i] = NAN;
     754            A->data.F32[i][i] = 0;
     755        }
     756    }
     757#endif
     758
     759    for (int n = 0; n < Nx; n++) {
     760        for (int m = 0; m < Ny; m++) {
     761            int I = n + Nx * m;
     762            map->map->data.F32[m][n] = solution->data.F32[I];
     763            map->error->data.F32[m][n] = NAN; // sqrt(A->data.F32[I][I]); // CZW: fix this to be a real error.
     764        }
     765    }
     766
     767    //    psFree (A);
     768    //    psFree (B);
     769    //    psFree (Empty);
     770    // CZW: free things
     771    psFree(solution);
     772    psFree(Asparse);
     773   
     774    *pGoodFit = true;
     775    return true;
     776}
     777
     778// measure residuals on each pass and clip outliers based on stats
     779bool psImageMapClipFitSparse(bool *pGoodFit, psImageMap *map, psStats *stats, psVector *inMask, psVectorMaskType maskValue,
     780                             const psVector *x, const psVector *y, const psVector *f, const psVector *df)
     781{
     782    // XXX add in full PS_ASSERTS
     783    psAssert(map, "impossible");
     784    psAssert(stats, "impossible");
     785    psAssert(x, "impossible");
     786    psAssert(y, "impossible");
     787    psAssert(f, "impossible");
     788
     789    *pGoodFit = false;
     790
     791    // the user supplies one of various stats option pairs,
     792    // determine the desired mean and stdev STATS options:
     793    psStatsOptions meanOption = psStatsMeanOption(stats->options);
     794    psStatsOptions stdevOption = psStatsStdevOption(stats->options);
     795    if (!psStatsSingleOption(meanOption)) {
     796        psError(PS_ERR_UNKNOWN, true, "no valid mean stats option selected");
     797        return false;
     798    }
     799    if (!psStatsSingleOption(stdevOption)) {
     800        psError(PS_ERR_UNKNOWN, true, "no valid stdev stats option selected");
     801        return false;
     802    }
     803
     804    // clipping range defined by min and max and/or clipSigma
     805    psF32 minClipSigma;
     806    psF32 maxClipSigma;
     807    if (isfinite(stats->max)) {
     808        maxClipSigma = fabs(stats->max);
     809    } else {
     810        maxClipSigma = fabs(stats->clipSigma);
     811    }
     812    if (isfinite(stats->min)) {
     813        minClipSigma = fabs(stats->min);
     814    } else {
     815        minClipSigma = fabs(stats->clipSigma);
     816    }
     817
     818    psVector *mask = inMask;
     819    if (!inMask) {
     820        mask = psVectorAlloc (x->n, PS_TYPE_VECTOR_MASK);
     821        psVectorInit (mask, 0);
     822    }
     823
     824    // vector to store residuals
     825    psVector *resid = psVectorAlloc(f->n, PS_TYPE_F32);
     826
     827    psTrace("psLib.imageops", 4, "stats->clipIter is %d\n", stats->clipIter);
     828    psTrace("psLib.imageops", 4, "(minClipSigma, maxClipSigma) is (%.2f, %.2f)\n", minClipSigma, maxClipSigma);
     829
     830    for (psS32 N = 0; N < stats->clipIter; N++) {
     831        psTrace("psLib.imageops", 6, "Loop iteration %d.  Calling psImageMapFit()\n", N);
     832        psS32 Nkeep = 0;
     833        if (!psImageMapFitSparse(pGoodFit, map, mask, maskValue, x, y, f, df)) {
    404834            psError(PS_ERR_UNKNOWN, false, "Could not fit image map.\n");
    405835            psFree(resid);

  • branches/eam_branches/ipp-20140904/psLib/src/imageops/psImageMapFit.h

    r30044 r37804  
    2020// fit the image map to a set of points
    2121bool psImageMapClipFit(bool *pGoodFit,
     22                       psImageMap *map,
     23                       psStats *stats,
     24                       psVector *mask,  // WARNING: Mask is modified!
     25                       psVectorMaskType maskValue,
     26                       const psVector *x,
     27                       const psVector *y,
     28                       const psVector *f,
     29                       const psVector *df
     30    );
     31
     32
     33// fit the image map to a set of points, using sparse matrix tools
     34bool psImageMapFitSparse(bool *pGoodFit,
     35                   psImageMap *map,
     36                   const psVector *mask,
     37                   psVectorMaskType maskValue, //
     38                   const psVector *x,
     39                   const psVector *y,
     40                   const psVector *f,
     41                   const psVector *df
     42    );
     43
     44// fit the image map to a set of points using sparse matrix tools
     45bool psImageMapClipFitSparse(bool *pGoodFit,
    2246                       psImageMap *map,
    2347                       psStats *stats,

  • branches/eam_branches/ipp-20140904/psLib/src/math/psMinimizePolyFit.c

    r36296 r37804  
    700700
    701701    // Define values that may be used by PS_POLYNOMIAL_ORD form.
     702    // these scaling values put the dynamic range of the data into some vaguely sensible location
    702703    bool scale = false;
     704    bool median_zero = false;
    703705    double median = 0.0;
    704706    double sigma = 1.0;
    705     psVector *sorted = NULL;
    706     psVector *z64 = NULL;
    707    
     707
    708708    switch (poly->type) {
    709709    case PS_POLYNOMIAL_ORD:
    710710      if ((f64->n < 10000)&&(poly->nX > 1)) {
    711711        scale = true;
    712         sorted = psVectorSort(NULL,x64);
     712
     713        // generate a subset of the unmasked values:
     714        psVector *tmp = psVectorAllocEmpty (x64->n, PS_TYPE_F64);
     715        for (int itmp = 0; itmp < x64->n; itmp++) {
     716          if (mask && (mask->data.PS_TYPE_VECTOR_MASK_DATA[itmp] && maskValue)) continue;
     717          psVectorAppend (tmp, x64->data.F64[itmp]);
     718        }
     719        psVector *sorted = psVectorSort(NULL,tmp);
    713720        median = sorted->data.F64[sorted->n / 2];
    714721        // CZW: I'm not bothering to scale this because it doesn't really matter.
    715722        sigma  = (sorted->data.F64[3 * sorted->n / 4] - sorted->data.F64[sorted->n / 4]);
    716723        psFree(sorted);
     724        psFree(tmp);
    717725
    718726        if ((!isfinite(median))||
    … …  
    721729          sigma = 1.0;
    722730        }
    723         if ((median == 0.0)&&(sigma == 1.0)) {
    724           scale = false;
     731        if (fabs(median) < 1e-10) {
     732          // CZW 2014-10-21: This median is small and close to zero.  This can cause issues with the
     733          // scaling,
     734          median_zero = true;
     735          if ((sigma == 1.0)||(fabs(sigma) <= 1e-10)) {
     736            // Don't bother scaling if sigma is unity (it's already scaled) or if the sigma calculation has gone wrong.
     737            scale = false;
     738          }
    725739        }
     740
    726741        // I can't see a way to not clobber x if it's already F64, so make a copy.x
    727         z64 = psVectorCopy(NULL,x64,PS_TYPE_F64);
     742        psVector *z64 = psVectorCopy(NULL,x64,PS_TYPE_F64);
    728743        psBinaryOp(z64,z64,"-",psScalarAlloc(median,PS_TYPE_F64));
    729744        psBinaryOp(z64,z64,"/",psScalarAlloc(sigma,PS_TYPE_F64));
    … …  
    733748#endif
    734749
    735        
    736750        result = VectorFitPolynomial1DOrd(poly, mask, maskValue, f64, fErr64, z64);
    737751        psFree(z64); // Done with this.
    … …  
    758772#endif
    759773        }
     774
    760775        for (psS32 i = 0; i <= poly->nX; i++) {
    761776          poly->coeff[i] = 0.0;
    762777          poly->coeffErr[i] = 0.0;
    763          
    764           for (psS32 j = 0; j <= poly->nX; j++) {
     778          if (median_zero) {  // If the median is zero, the obtained solution just needs to be scaled by the sigma values.
     779            poly->coeff[i]    = Zcoeff[i] * pow(1.0 / sigma,i);
     780            poly->coeffErr[i] = ZcoeffErr[i] * pow(1.0 / sigma,i);
     781          }
     782          else { // Otherwise, do the correct transformations by expanding the (x-m)/s terms.
     783            for (psS32 j = 0; j <= poly->nX; j++) {
    765784#if (CZW)
    766             printf("        %d %d %f %f %f %f => %f\n",
    767                    i,j,Zcoeff[j],
    768                    pow(1.0 / sigma,j) * pow(-1,j - i),
    769                    pow(median,j - i),
    770                    1.0 * psBinomialCoeff(j,i),
    771                    Zcoeff[j] * pow(1.0 / sigma,j) * pow(-1,j  -i) * pow(median,j - i) * 1.0 * psBinomialCoeff(j,i)
    772                    );
     785              printf("        %d %d %f %f %f %f => %f\n",
     786                     i,j,Zcoeff[j],
     787                     pow(1.0 / sigma,j) * pow(-1,j - i),
     788                     pow(median,j - i),
     789                     1.0 * psBinomialCoeff(j,i),
     790                     Zcoeff[j] * pow(1.0 / sigma,j) * pow(-1,j  -i) * pow(median,j - i) * 1.0 * psBinomialCoeff(j,i)
     791                     );
    773792#endif
    774             poly->coeff[i] += Zcoeff[j] * pow(1.0 / sigma,j) * pow(-1,j - i) * pow(median,j - i) * psBinomialCoeff(j,i);
    775             poly->coeffErr[i] += pow(ZcoeffErr[j] * pow(1.0 / sigma,j) * pow(-1,j - i) * pow(median,j - 1) * psBinomialCoeff(j,i),2);
     793              poly->coeff[i] += Zcoeff[j] * pow(1.0 / sigma,j) * pow(-1,j - i) * pow(median,j - i) * psBinomialCoeff(j,i);
     794              poly->coeffErr[i] += pow(ZcoeffErr[j] * pow(1.0 / sigma,j) * pow(-1,j - i) * pow(median,j - 1) * psBinomialCoeff(j,i),2);
     795            }
     796            poly->coeffErr[i] = sqrt(poly->coeffErr[i]);
    776797          }
    777           poly->coeffErr[i] = sqrt(poly->coeffErr[i]);
    778798#if (CZW)
    779799          printf("poly1d: unscaled parameters: %d %f %f\n",
    … …  
    784804        psFree(ZcoeffErr);
    785805
    786       }
     806      } // End scaling block.
    787807       
    788808        break;

  • branches/eam_branches/ipp-20140904/psLib/src/mathtypes/psImage.c

    r23443 r37804  
    9191    }
    9292
    93     long numBytes = numRows * numCols * elementSize;
     93    size_t numBytes = (size_t) numRows * (size_t) numCols * (size_t) elementSize;
    9494
    9595    psImage* image = (psImage* ) p_psAlloc(file, lineno, func, sizeof(psImage));

  • branches/eam_branches/ipp-20140904/psModules/src/concepts/pmConcepts.c

    r36834 r37804  
    312312        conceptRegisterEnum("FPA.TIMESYS", "Time system", p_pmConceptParse_TIMESYS, p_pmConceptFormat_TIMESYS, p_pmConceptCopy_TIMESYS, false, PM_FPA_LEVEL_FPA);
    313313        conceptRegisterTime("FPA.TIME", "Time of exposure", false, PM_FPA_LEVEL_FPA);
     314
     315        conceptRegisterStr("FPA.SHUTOUTC", "Time of exposure", NULL,NULL,NULL,false, PM_FPA_LEVEL_FPA);
     316
    314317        conceptRegisterF32("FPA.TEMP", "Temperature of focal plane", NULL, NULL, NULL, false, PM_FPA_LEVEL_FPA);
    315318        conceptRegisterF32("FPA.M1X", "Primary Mirror X Position", NULL, NULL, NULL, false, PM_FPA_LEVEL_FPA);

  • branches/eam_branches/ipp-20140904/psModules/src/imcombine/pmSubtraction.c

    r36650 r37804  
    995995
    996996    }
     997
     998    if (0) {
     999      FILE *f = fopen ("vector.dat", "w");
     1000      psAssert (f, "should not fail to open test file");
     1001     
     1002      for (int qq = 0; qq < match->chisq->n; qq++) {
     1003        fprintf (f, "%f %f : %f %f : %d\n",
     1004                 match->chisq->data.F32[qq], match->fluxes->data.F32[qq],
     1005                 logchisq->data.F32[qq], logflux->data.F32[qq],
     1006                 match->stampMask->data.PS_TYPE_VECTOR_MASK_DATA[qq]);
     1007      }
     1008      fclose (f);
     1009    }
     1010
     1011
    9971012    bool result = psVectorClipFitPolynomial1D(model, stats, match->stampMask, 0xff, logchisq, NULL, logflux);
    9981013    if (!result) {

  • branches/eam_branches/ipp-20140904/psModules/src/objects/pmSourceExtendedPars.c

    r36375 r37804  
    226226    pars->petrosianR50 = NAN;
    227227    pars->petrosianR50Err = NAN;
     228    pars->ghalfLightRadius = NAN;
     229    pars->gRT = NAN;
     230    pars->gRA = NAN;
     231    pars->gS2 = NAN;
     232    pars->gA = NAN;
     233    pars->gbumpy = NAN;
    228234    return pars;
    229235}

  • branches/eam_branches/ipp-20140904/psModules/src/objects/pmSourceExtendedPars.h

    r36747 r37804  
    6565    float petrosianR50Err;
    6666    float petrosianFill;
     67    float ghalfLightRadius;                 
     68    float gRT;                           // total residual (from eliptically symmetric model) Simard 2002
     69    float gRA;                           // assymetric residual (from eliptically symmetric model) Simard 2002
     70    float gS2;                           // (un) smoothness  (Simard, 2011 Cheng 2011)
     71    float gA;                            // assymetry index (Gyory & Bell 2010)
     72    float gbumpy;                        // Blakeslee bumpiness
    6773} pmSourceExtendedPars;
    6874

  • branches/eam_branches/ipp-20140904/psModules/src/objects/pmSourceIO_CMF.c.in

    r37325 r37804  
    319319
    320320    // define PSF model type
    321     int modelType = pmModelClassGetType ("PS_MODEL_GAUSS");
     321    int defaultModelType = pmModelClassGetType ("PS_MODEL_GAUSS");
     322    int modelType = -1;
    322323
    323324    // if header does not define the model, default to a gaussian
    … …  
    326327        modelType = pmModelClassGetType (PSF_NAME);
    327328    }
    328     assert (modelType > -1);
     329    // work around bug in psphotFullForce
     330    if (modelType < 0) {
     331        modelType = defaultModelType;
     332    }
     333    // assert (modelType > -1);
    329334
    330335    // do we expect to find lensing parameters?
    … …  
    456461          @>PS1_V4@ source->lensingOBJ->shear->e2  = psMetadataLookupF32 (&status, row, "LENS_E2_SH_OBJ");
    457462        }
     463
     464        @>PS1_V4@ source->chipNum = psMetadataLookupS16 (&status, row, "SRC_CHIP_NUM");
     465        @>PS1_V4@ source->chipX = psMetadataLookupS16 (&status, row, "SRC_CHIP_X");
     466        @>PS1_V4@ source->chipY = psMetadataLookupS16 (&status, row, "SRC_CHIP_Y");
    458467
    459468        if (haveLensPSF) {
    … …  
    639648                psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS_90_ERR", PS_DATA_F32, "Petrosian R90 Error (pix)", extpars->petrosianR90Err);
    640649                psMetadataAdd (row, PS_LIST_TAIL, "PETRO_FILL",          PS_DATA_F32, "Petrosian Fill Factor", extpars->petrosianFill);
     650                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "HALF_LIGHT_RADIUS", 0, "Half Light Radius", extpars->ghalfLightRadius);
     651                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_RT", 0, "total residual Sersic Model", extpars->gRT);
     652                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_RA", 0, "assymetric residual Sersic Model", extpars->gRA);
     653                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_S2", 0, "(un)smoothness paramter residual Sersic Model", extpars->gS2);
     654                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_A", 0, "assymetry index", extpars->gA);
     655                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_BUMPY", 0, "bumpiness", extpars->gbumpy);
    641656            } else {
    642657                psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG",        PS_DATA_F32, "Petrosian Magnitude",       NAN);
    … …  
    649664                psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS_90_ERR", PS_DATA_F32, "Petrosian R90 Error (pix)",NAN);
    650665                psMetadataAdd (row, PS_LIST_TAIL, "PETRO_FILL",          PS_DATA_F32, "Petrosian Fill Factor", NAN);
     666                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "HALF_LIGHT_RADIUS", 0, "Half Light Radius", NAN);
     667                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_RT", 0, "total residual Sersic Model", NAN);
     668                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_RA", 0, "assymetric residual Sersic Model", NAN);
     669                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_S2", 0, "(un)smoothness paramter residual Sersic Model", NAN);
     670                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_A", 0, "assymetry index", NAN);
     671                @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "G_BUMPY", 0, "bumpiness", NAN);
    651672            }
    652673        }
    … …  
    848869        extpars->petrosianR90Err   = psMetadataLookupF32(&status, row, "PETRO_RADIUS_90_ERR");
    849870        extpars->petrosianFill     = psMetadataLookupF32(&status, row, "PETRO_FILL");
     871        extpars->ghalfLightRadius  = psMetadataLookupF32(&status, row, "HALF_LIGHT_RADIUS");
     872        extpars->gRT               = psMetadataLookupF32(&status, row, "G_RT");
     873        extpars->gRA               = psMetadataLookupF32(&status, row, "G_RA");
     874        extpars->gS2               = psMetadataLookupF32(&status, row, "G_S2");
     875        extpars->gA                = psMetadataLookupF32(&status, row, "G_A");
     876        extpars->gbumpy            = psMetadataLookupF32(&status, row, "G_BUMPY");
    850877
    851878        psVector *radSB   = psMetadataLookupVector(&status, row, "PROF_SB");
    … …  
    9951022            psMetadataAddF32 (row, PS_LIST_TAIL, "X_EXT_SIG",        0, "Sigma in EXT x coordinate",                  xErr);
    9961023            psMetadataAddF32 (row, PS_LIST_TAIL, "Y_EXT_SIG",        0, "Sigma in EXT y coordinate",                  yErr);
     1024            @>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "SKY_EXT",        0, "EXT fit sky value",                  PAR ? PAR[PM_PAR_SKY] : NAN);
    9971025            @>PS1_DV2,>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "RA_EXT",           0, "EXT model ra coordinate",                    raPos);
    9981026            @>PS1_DV2,>PS1_SV3@ psMetadataAddF32 (row, PS_LIST_TAIL, "DEC_EXT",          0, "EXT model dec coordinate",                   decPos);

  • branches/eam_branches/ipp-20140904/psastro/src/psastro.h

    r36837 r37804  
    154154
    155155bool              psastroZeroPoint (pmConfig *config);
    156 psVector         *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime);
     156psVector         *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime, float refminMag, float refmaxMag);
    157157bool              psastroZeroPointAnalysis (psMetadata *header, psVector *dMag, float zeropt, psMetadata *recipe);
    158158bool              psastroZeroPointFromRecipe (float *zeropt, float *exptime, float *ghostMaxMag, pmFPA *fpa, psMetadata *recipe);
     159bool              psastroZeroPointRefMagLimitFromRecipe (float *refminMag, float *refmaxMag, pmFPA *fpa, psMetadata *recipe);
    159160
    160161psStats          *psastroStatsPercentile (psVector *myVector, psMetadata *recipe);

  • branches/eam_branches/ipp-20140904/psastro/src/psastroChooseRefstars.c

    r35715 r37804  
    6666    psF32 clampMagMin = psMetadataLookupF32 (&status, recipe, "REFSTAR_CLAMP_MAG_MIN");
    6767    if (!status) clampMagMin = -5.0;
     68    //MEH adding max option as well -- really should not be hardcoded..
     69    psF32 clampMagMax = psMetadataLookupF32 (&status, recipe, "REFSTAR_CLAMP_MAG_MAX");
     70    if (!status) clampMagMax = 30.0;
     71    psWarning("Will skip/clamp refstar with brighter/fainter magnitude %f/%f\n",clampMagMin,clampMagMax);
     72
    6873
    6974    // de-activate all files except PSASTRO.REFSTARS
    … …  
    109114
    110115                    if (ref->Mag < clampMagMin) {
    111                         psWarning("Skipping refstar with abusrd magnitude %f ra: %f dec: %f\n",
    112                             ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d));
     116                        //psWarning("Skipping refstar with abusrd magnitude %f ra: %f dec: %f\n",
     117                        //    ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d));
     118                        psTrace ("psastro", 6, "Skipping by clamp refstar with bright magnitude %f ra: %f dec: %f\n",
     119                                 ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d));
     120                        goto skip;
     121                    }
     122                    //MEH option for max mag clamp also -- need to change psWarning (to log) to trace and above for bright..
     123                    if (ref->Mag > clampMagMax) {
     124                        //psWarning("Skipping by clamp refstar with faint magnitude %f ra: %f dec: %f\n",
     125                        //    ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d));
     126                        psTrace ("psastro", 6, "Skipping by clamp refstar with faint magnitude %f ra: %f dec: %f\n",
     127                                 ref->Mag, RAD_TO_DEG(ref->sky->r), RAD_TO_DEG(ref->sky->d));
    113128                        goto skip;
    114129                    }

  • branches/eam_branches/ipp-20140904/psastro/src/psastroZeroPoint.c

    r33643 r37804  
    2020    pmCell *cell = NULL;
    2121    pmReadout *readout = NULL;
     22    float refminMag,refmaxMag;
    2223
    2324    // select the current recipe
    … …  
    5960    }
    6061
     62    // MEH -- RefMagLimit for ZP with photcode option (merr probably better but not an option)
     63    // -- defualt is to use general config value (no hardcode..)
     64    if (!psastroZeroPointRefMagLimitFromRecipe (&refminMag, &refmaxMag, fpa, recipe)) {
     65        psLogMsg ("psastro", PS_LOG_INFO, "failed to load ref min,maxMag data from recipe");
     66        return false;
     67    }
     68
    6169    // if we measure the zero point by exposure, accumulate the dMag values here:
    6270    psVector *dMag = NULL;
     71    // MEH -- also the number of refcat sources used
     72    int zpt_nref = 0;
    6373
    6474    float fpaZP = 0.0;                  // Average zero point
    … …  
    8191
    8292                // calculate dMag for the matched stars
    83                 dMag = psastroZeroPointReadoutAccum (dMag, readout, exptime);
    84 
     93                //dMag = psastroZeroPointReadoutAccum (dMag, readout, exptime);
     94                // MEH -- additional limits should be available for ZP if just robust mean and w/ stack skycal
     95                dMag = psastroZeroPointReadoutAccum (dMag, readout, exptime, refminMag, refmaxMag);
    8596                if (!byExposure) {
    8697                    // calculate dMag for the matched stars just for this readout (well, chip)
    8798                    psMetadata *header = psMetadataLookupMetadata (&status, readout->analysis, "PSASTRO.HEADER");
    8899                    psastroZeroPointAnalysis (header, dMag, zeropt, recipe);
     100                    // MEH -- per readout N refcat, min/max mag  -- added here, not part of median/edge process
     101                    zpt_nref += dMag->n;
     102                    psMetadataAddS32 (header, PS_LIST_TAIL, "ZPT_NREF", PS_META_REPLACE, "N refcat sources for zero point", dMag->n);
     103                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MIN", PS_META_REPLACE, "min refcat mag for zero point", refminMag);
     104                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MAX", PS_META_REPLACE, "max refcat mag for zero point", refmaxMag);
    89105                    psFree (dMag);
    90106                    dMag = NULL;
    … …  
    95111                        numZP++;
    96112                    }
     113
    97114
    98115
    … …  
    110127        }
    111128        psastroZeroPointAnalysis (header, dMag, zeropt, recipe);
     129        //MEH -- all N refcat sources here
     130        zpt_nref = dMag->n;
    112131        psFree (dMag);
    113132        dMag = NULL;
    … …  
    141160                    // calculate dMag for the matched stars just for this readout (well, chip)
    142161                    psMetadata *header = psMetadataLookupMetadata (&status, readout->analysis, "PSASTRO.HEADER");
    143 
    144162                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_OBS", PS_META_REPLACE, "measured zero point",  zptObs);
    145163                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_REF", PS_META_REPLACE, "reference zero point", zptRef);
    146164                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_ERR", PS_META_REPLACE, "error on zero point",  zptErr);
    147165                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_OFF", PS_META_REPLACE, "zero point offset",    zptOff);
     166                    //MEH -- per readout  N refcat, MIN/MAX for consistency
     167                    psMetadataAddS32 (header, PS_LIST_TAIL, "ZPT_NREF", PS_META_REPLACE, "N refcat sources for zero point", zpt_nref);
     168                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MIN", PS_META_REPLACE, "min refcat mag for zero point", refminMag);
     169                    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MAX", PS_META_REPLACE, "max refcat mag for zero point", refmaxMag);
    148170                }
    149171            }
    … …  
    151173    }
    152174
     175    // MEH add N refcat, min/max mag to primary header
     176    psMetadata *header = psMetadataLookupMetadata (&status, fpa->analysis, "PSASTRO.HEADER");
     177    if (!header) {
     178        header = psMetadataAlloc ();
     179        psMetadataAddMetadata (fpa->analysis, PS_LIST_TAIL, "PSASTRO.HEADER",  PS_META_REPLACE, "psastro header stats", header);
     180        psFree (header);
     181    }
     182    psMetadataAddS32 (header, PS_LIST_TAIL, "ZPT_NREF", PS_META_REPLACE, "Total N refcat sources for zero point", zpt_nref);
     183    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MIN", PS_META_REPLACE, "min refcat mag for zero point", refminMag);
     184    psMetadataAddF32 (header, PS_LIST_TAIL, "ZPT_MAX", PS_META_REPLACE, "max refcat mag for zero point", refmaxMag);
    153185
    154186    psMetadataItem *item = psMetadataLookup(fpa->concepts, "FPA.ZP");
    … …  
    162194 * accumulate the dMag values from this readout
    163195 */
    164 psVector *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime) {
     196psVector *psastroZeroPointReadoutAccum(psVector *dMag, pmReadout *readout, float exptime, float refminMag, float refmaxMag) {
    165197
    166198    bool status;
    … …  
    191223      // XXX I should be applying the airmass term here or in psastroLoadRefstars
    192224      // (ie, ref->Mag would be a prediction for a given star.
    193       float value = ref->Mag - raw->Mag - 2.5*log10(exptime);
    194       psVectorAppend (dMag, value);
     225      // MEH -- only append if in allowed ref mag range -- may want merr but not available..
     226      if ( ref->Mag >= refminMag && ref->Mag <= refmaxMag ){
     227        float value = ref->Mag - raw->Mag - 2.5*log10(exptime);
     228        psVectorAppend (dMag, value);
     229      }
    195230    }
    196231    return dMag;
    … …  
    490525                continue;
    491526            }
    492         }
     527            //MEH null is a pain.. so only log if set
     528            psLogMsg ("psastro", PS_LOG_INFO, "found GHOST_MAX_MAG %f",*ghostMaxMag);
     529        }
     530        //MEH what zpt is set to
     531        psLogMsg ("psastro", PS_LOG_INFO, "found ZEROPT  %f",*zeropt);
    493532        psFree (iter);
    494533        return true;
    … …  
    497536    return false;
    498537}
     538
     539
     540
     541bool psastroZeroPointRefMagLimitFromRecipe (float *refminMag, float *refmaxMag, pmFPA *fpa, psMetadata *recipe) {
     542
     543    bool status;
     544    float tmpmin,tmpmax;
     545
     546    // MEH -- duplicate psastroZeroPointFromRecipe for ZeroPoint adding RefMag limits with photcode option for extra restrictions
     547    // must have non-photcode dependent setting -- use as default rather than a hardcoded value
     548    *refminMag = psMetadataLookupF32 (&status, recipe, "REFSTAR.ZP.MIN.MAG");
     549    //if (!status) *refminMag = -5.0;
     550    if (!status) {
     551        psLogMsg ("psastro", PS_LOG_INFO, "RefMagLimit is missing REFSTAR.ZP.MIN.MAG");
     552        return false;
     553    }
     554    *refmaxMag = psMetadataLookupF32 (&status, recipe, "REFSTAR.ZP.MAX.MAG");
     555    //if (!status) *refmaxMag = 30.0;
     556    if (!status) {
     557        psLogMsg ("psastro", PS_LOG_INFO, "RefMagLimit is missing REFSTAR.ZP.MAX.MAG");
     558        return false;
     559    }
     560
     561    // select the filter; default to any photcode and mag limit otherwise
     562    char *filter = psMetadataLookupStr (&status, fpa->concepts, "FPA.FILTERID");
     563    if (!status) ESCAPE ("RefMagLimit missing FPA.FILTER in concepts");
     564
     565    // need to select the PHOTCODE.DATA folder that matches our filter
     566    psMetadataItem *item = psMetadataLookup (recipe, "PHOTCODE.DATA");
     567    if (!item) ESCAPE ("RefMagLimit PHOTCODE.DATA folders missing");
     568    if (item->type != PS_DATA_METADATA_MULTI) ESCAPE ("RefMagLimit PHOTCODE.DATA not a multi");
     569
     570    // PHOTCODE.DATA is a multi of metadata items
     571    psListIterator *iter = psListIteratorAlloc(item->data.list, PS_LIST_HEAD, false);
     572
     573    psMetadataItem *refItem = NULL;
     574    while ((refItem = psListGetAndIncrement (iter))) {
     575        if (refItem->type != PS_DATA_METADATA) ESCAPE ("RefMagLimit PHOTCODE.DATA entry is not a metadata folder");
     576
     577        char *refFilter = psMetadataLookupStr (&status, refItem->data.md, "FILTER");
     578        if (!status) {
     579            // always seems to happen once, commented out in ZeroPoint above as well
     580            //psLogMsg ("psastro", PS_LOG_INFO, "RefMagLimit PHOTCODE.DATA recipe folder is missing FILTER");
     581            continue;
     582        }
     583
     584        // does this entry match the current filter?
     585        if (strcmp (refFilter, filter)) continue;
     586
     587        psLogMsg ("psastro", PS_LOG_DETAIL, "RefMagLimit PHOTCODE.DATA found for filter %s", filter);
     588
     589        // -- may set one or other or both -- no reason to log if found or often not
     590        tmpmin = psMetadataLookupF32 (&status, refItem->data.md, "REFSTAR.ZP.MIN.MAG");
     591        if (status) {
     592            *refminMag = tmpmin;
     593        }
     594        tmpmax = psMetadataLookupF32 (&status, refItem->data.md, "REFSTAR.ZP.MAX.MAG");
     595        if (status) {
     596            *refmaxMag = tmpmax;
     597        }
     598        // once finds filter, should free and return true but multiple groups cause issue so just go through all groups now
     599        //psFree (iter);
     600        //return true;
     601    }
     602
     603    // log out set values
     604    psLogMsg ("psastro", PS_LOG_INFO, "using REFSTAR.ZP.MIN.MAG %f",*refminMag);
     605    psLogMsg ("psastro", PS_LOG_INFO, "using REFSTAR.ZP.MAX.MAG %f",*refmaxMag);
     606
     607    psFree (iter);
     608    return true;
     609}
     610

  • branches/eam_branches/ipp-20140904/psphot/src/Makefile.am

    r37321 r37804  
    260260        psphotSetNFrames.c             \
    261261        psphotSourceMemory.c           \
    262         psphotChipParams.c
     262        psphotChipParams.c              \
     263        psphotGalaxyParams.c
    263264
    264265# not currently used

  • branches/eam_branches/ipp-20140904/psphot/src/psphot.h

    r37321 r37804  
    138138bool            psphotExtendedSourceFits_Threaded (psThreadJob *job);
    139139
     140bool            psphotGalaxyParams (pmConfig *config, const pmFPAview *view, const char *filerule);
     141bool            psphotGalaxyParamsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe);
     142bool            psphotGalaxyParams_Threaded (psThreadJob *job);
     143
    140144bool            psphotApResid (pmConfig *config, const pmFPAview *view, const char *filerule);
    141145bool            psphotApResidReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe);
    142146
    143147bool            psphotMagnitudes (pmConfig *config, const pmFPAview *view, const char *filerule);
    144 bool            psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf);
     148bool            psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf, int index);
    145149bool            psphotMagnitudes_Threaded (psThreadJob *job);
    146150
    … …  
    168172// in psphotChoosePSF.c:
    169173bool            psphotPSFstats (pmReadout *readout, pmPSF *psf);
     174bool            psphotPSFstatsSources (pmReadout *readout, psArray *sources, pmPSF *psf);
    170175bool            psphotMomentsStats (pmReadout *readout, psArray *sources);
    171176
    … …  
    232237int             psphotSaveImage (psMetadata *header, psImage *image, char *filename);
    233238bool            psphotDumpConfig (pmConfig *config);
    234 pmReadout      *psphotSelectBackground (pmConfig *config, const pmFPAview *view);
    235 pmReadout      *psphotSelectBackgroundStdev (pmConfig *config, const pmFPAview *view);
     239pmReadout      *psphotSelectBackground (pmConfig *config, const pmFPAview *view, int index);
     240pmReadout      *psphotSelectBackgroundStdev (pmConfig *config, const pmFPAview *view, int index);
    236241
    237242// PSF / DBL / EXT evaluation functions

  • branches/eam_branches/ipp-20140904/psphot/src/psphotChooseAnalysisOptions.c

    r36453 r37804  
    606606    psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld objects: %f sec\n", objects->n, psTimerMark ("psphot.options"));
    607607
     608    psFree(exptime);
     609    psFree(zeropt);
     610    psFree(chips);
     611    psFree(inputFilters);
     612    psFree(petroFluxLim);
     613    psFree(extFitFluxLim);
    608614    psFree(toGal);
    609615

  • branches/eam_branches/ipp-20140904/psphot/src/psphotChoosePSF.c

    r36863 r37804  
    11# include "psphotInternal.h"
     2bool psphotPSFstatsSources (pmReadout *readout, psArray *sources, pmPSF *psf);
    23
    34// generate a PSF model for inputs without PSF models already loaded
    … …  
    388389    psFree (models);
    389390
    390     if (!psphotPSFstats (readout, psf)) {
     391    // if (!psphotPSFstats (readout, psf)) {
     392
     393    if (!psphotPSFstatsSources (readout, sources, psf)) {
    391394        psError(PSPHOT_ERR_PSF, false, "cannot measure PSF shape terms");
    392395        psFree(options);
    … …  
    466469
    467470            if (modelPSF->params->n > 7) {
    468               psVectorAppend (psfExtra1, modelPSF->params->data.F32[7]);
     471                psVectorAppend (psfExtra1, modelPSF->params->data.F32[7]);
    469472            }
    470473            if (modelPSF->params->n > 8) {
    471               psVectorAppend (psfExtra2, modelPSF->params->data.F32[8]);
     474                psVectorAppend (psfExtra2, modelPSF->params->data.F32[8]);
    472475            }
    473476            psFree (modelPSF);
    474477
    475             // float fwhmtest = pmPSFtoFWHM(psf, xc, yc);
    476             // fprintf (stderr, "fwhm: %f, %f : %f\n", FWHM_MAJOR, FWHM_MINOR, fwhmtest);
     478            float fwhmtest = pmPSFtoFWHM(psf, xc, yc);
     479            fprintf (stderr, "fwhm: %f, %f : %f\n", FWHM_MAJOR, FWHM_MINOR, fwhmtest);
    477480        }
    478481    }
    … …  
    522525
    523526    if (psfExtra1->n) {
    524       if (!psVectorStats (stats, psfExtra1, NULL, NULL, 0)) {
    525         psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 1");
     527        if (!psVectorStats (stats, psfExtra1, NULL, NULL, 0)) {
     528            psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 1");
     529            goto escape;
     530        }
     531        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT1", PS_META_REPLACE, "PSF extra param 1", stats->sampleMean);
     532        psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 1: %f +/- %f", stats->sampleMean, stats->sampleStdev);
     533    }
     534
     535    if (psfExtra2->n) {
     536        if (!psVectorStats (stats, psfExtra2, NULL, NULL, 0)) {
     537            psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 2");
     538            goto escape;
     539        }
     540        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT2", PS_META_REPLACE, "PSF extra param 2", stats->sampleMean);
     541        psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 2: %f +/- %f", stats->sampleMean, stats->sampleStdev);
     542    }
     543
     544    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE",    PS_META_REPLACE, "PSF angle",           axes.theta);
     545    psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", psf->nPSFstars);
     546
     547    // psLogMsg ("psphot", PS_LOG_DETAIL, "PSF angle: %f, nstars: %d", axes.theta, psf->nPSFstars);
     548
     549    char *psfModelName = pmModelClassGetName(psf->type);
     550    psMetadataAddStr(readout->analysis,  PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "PSF Model Name", psfModelName);
     551    psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSF_OK",   PS_META_REPLACE, "Valid PSF Model?", true);
     552
     553    int nParams = pmModelClassParameterCount(psf->type);
     554    psMetadataAddS32(readout->analysis, PS_LIST_TAIL, "PSF_NPAR",   PS_META_REPLACE, "Number of PSF parameters", nParams);
     555
     556    psFree (fwhmMajor);
     557    psFree (fwhmMinor);
     558    psFree (psfExtra1);
     559    psFree (psfExtra2);
     560    psFree (stats);
     561    return true;
     562
     563escape:
     564    psFree (fwhmMajor);
     565    psFree (fwhmMinor);
     566    psFree (psfExtra1);
     567    psFree (psfExtra2);
     568    psFree (stats);
     569    return false;
     570}
     571
     572// measure average parameters of the PSF model
     573bool psphotPSFstatsSources (pmReadout *readout, psArray *sources, pmPSF *psf) {
     574
     575    psEllipseShape shape;
     576    psEllipseAxes axes;
     577
     578    PS_ASSERT_PTR_NON_NULL(readout, false);
     579    PS_ASSERT_PTR_NON_NULL(psf, false);
     580
     581    psImage *image = readout->image;
     582    PS_ASSERT_PTR_NON_NULL(image, false);
     583
     584    // XXX a minor hack: measure the values for a grid of points across the image, determine mean, UQ, LQ, etc:
     585    psVector *fwhmMajor = psVectorAllocEmpty (100, PS_DATA_F32);
     586    psVector *fwhmMinor = psVectorAllocEmpty (100, PS_DATA_F32);
     587    psVector *psfExtra1 = psVectorAllocEmpty (100, PS_DATA_F32);
     588    psVector *psfExtra2 = psVectorAllocEmpty (100, PS_DATA_F32);
     589
     590    for (int i = 0; i < sources->n; i++) {
     591
     592        pmSource *source = sources->data[i];
     593
     594        if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue;
     595
     596        float xc = source->peak->xf;
     597        float yc = source->peak->yf;
     598
     599        // create modelPSF from this model
     600        pmModel *modelPSF = pmModelFromPSFforXY (psf, xc, yc, 1.0);
     601        if (!modelPSF) {
     602            fprintf (stderr, "?");
     603            continue;
     604        }
     605
     606        // get the model full-width at half-max
     607        float FWHM_MAJOR = 2*modelPSF->class->modelRadius (modelPSF->params, 0.5);
     608
     609        // XXX make sure this is consistent with the re-definition of PM_PAR_SXX
     610        shape.sx  = modelPSF->params->data.F32[PM_PAR_SXX];
     611        shape.sy  = modelPSF->params->data.F32[PM_PAR_SYY];
     612        shape.sxy = modelPSF->params->data.F32[PM_PAR_SXY];
     613        axes = psEllipseShapeToAxes (shape, 20.0);
     614
     615        float FWHM_MINOR = FWHM_MAJOR * (axes.minor / axes.major);
     616        if (!isfinite(FWHM_MAJOR) || !isfinite(FWHM_MINOR)) {
     617            fprintf (stderr, "!");
     618            psFree (modelPSF);
     619            continue;
     620        }
     621        psVectorAppend (fwhmMajor, FWHM_MAJOR);
     622        psVectorAppend (fwhmMinor, FWHM_MINOR);
     623
     624        if (modelPSF->params->n > 7) {
     625            psVectorAppend (psfExtra1, modelPSF->params->data.F32[7]);
     626        }
     627        if (modelPSF->params->n > 8) {
     628            psVectorAppend (psfExtra2, modelPSF->params->data.F32[8]);
     629        }
     630        psFree (modelPSF);
     631
     632        float fwhmtest = pmPSFtoFWHM(psf, xc, yc);
     633        fprintf (stderr, "fwhm: %f, %f : %f\n", FWHM_MAJOR, FWHM_MINOR, fwhmtest);
     634    }
     635
     636    psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_SAMPLE_QUARTILE);
     637    if (!psVectorStats (stats, fwhmMajor, NULL, NULL, 0)) {
     638        psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MAJOR");
    526639        goto escape;
    527       }
    528       psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT1", PS_META_REPLACE, "PSF extra param 1", stats->sampleMean);
    529       psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 1: %f +/- %f", stats->sampleMean, stats->sampleStdev);
     640    }
     641
     642    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MAJ",   PS_META_REPLACE, "PSF FWHM Major axis (mean)", stats->sampleMean);
     643    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_SG",   PS_META_REPLACE, "PSF FWHM Major axis (sigma)", stats->sampleStdev);
     644    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_LQ",   PS_META_REPLACE, "PSF FWHM Major axis (lower quartile)", stats->sampleLQ);
     645    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_UQ",   PS_META_REPLACE, "PSF FWHM Major axis (upper quartile)", stats->sampleUQ);
     646
     647    float fwhmMaj = stats->sampleMean;  // FWHM on major axis
     648
     649    if (!psVectorStats (stats, fwhmMinor, NULL, NULL, 0)) {
     650        psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MINOR");
     651        goto escape;
     652    }
     653    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MIN",   PS_META_REPLACE, "PSF FWHM Minor axis (mean)", stats->sampleMean);
     654    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_SG",   PS_META_REPLACE, "PSF FWHM Minor axis (sigma)", stats->sampleStdev);
     655    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_LQ",   PS_META_REPLACE, "PSF FWHM Minor axis (lower quartile)", stats->sampleLQ);
     656    psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_UQ",   PS_META_REPLACE, "PSF FWHM Minor axis (upper quartile)", stats->sampleUQ);
     657
     658    float fwhmMin = stats->sampleMean;  // FWHM on minor axis
     659
     660    if (readout->parent) {
     661
     662        // we now have 2 possible measurements of the seeing : the PSF model based version and
     663        // the Moments based value we need to define a definitive "CHIP.SEEING" value.  This
     664        // value is used by the PSF-matching programs (ppSub and ppStack).  The moments-based
     665        // value is probably more representative of the value needed by those tools (it is also
     666        // more stable?)
     667        bool status = false;
     668        float fwhmMajorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1");
     669        float fwhmMinorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1");
     670
     671        pmChip *chip = readout->parent->parent; // Parent chip
     672        psAssert(chip, "Cell should be attached to a chip.");
     673        psMetadataItem *item = psMetadataLookup(chip->concepts, "CHIP.SEEING"); // Item with chip
     674        item->data.F32 = 0.5 * (fwhmMajorMoments + fwhmMinorMoments);
     675
     676        psLogMsg ("psphot", PS_LOG_DETAIL, "fwhm (psf): %f,%f (moments): %f,%f", fwhmMaj, fwhmMin, fwhmMajorMoments, fwhmMinorMoments);
     677    }
     678
     679    if (psfExtra1->n) {
     680        if (!psVectorStats (stats, psfExtra1, NULL, NULL, 0)) {
     681            psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 1");
     682            goto escape;
     683        }
     684        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT1", PS_META_REPLACE, "PSF extra param 1", stats->sampleMean);
     685        psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 1: %f +/- %f", stats->sampleMean, stats->sampleStdev);
    530686    }
    531687
    532688    if (psfExtra2->n) {
    533       if (!psVectorStats (stats, psfExtra2, NULL, NULL, 0)) {
    534         psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 2");
    535         goto escape;
    536       }
    537       psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT2", PS_META_REPLACE, "PSF extra param 2", stats->sampleMean);
    538       psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 2: %f +/- %f", stats->sampleMean, stats->sampleStdev);
     689        if (!psVectorStats (stats, psfExtra2, NULL, NULL, 0)) {
     690            psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 2");
     691            goto escape;
     692        }
     693        psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT2", PS_META_REPLACE, "PSF extra param 2", stats->sampleMean);
     694        psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 2: %f +/- %f", stats->sampleMean, stats->sampleStdev);
    539695    }
    540696

  • branches/eam_branches/ipp-20140904/psphot/src/psphotCleanup.c

    r36555 r37804  
    2525    // don't display memory leaks unless trace is on.
    2626    // fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "psphot");
    27     fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, stdout, false), "psphot");
     27    fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks2 (0, NULL, stdout, false, 500), "psphot");
    2828#endif
    2929    return;

  • branches/eam_branches/ipp-20140904/psphot/src/psphotEfficiency.c

    r36863 r37804  
    502502
    503503    // measure the magnitudes and fluxes for the sources
    504     if (!psphotMagnitudesReadout(config, recipe, view, readout, fakeSourcesAll, psf)) {
     504    if (!psphotMagnitudesReadout(config, recipe, view, readout, fakeSourcesAll, psf, index)) {
    505505        psError(PS_ERR_UNKNOWN, false, "Unable to measure magnitudes of fake sources.");
    506506        psFree(fakeSources);

  • branches/eam_branches/ipp-20140904/psphot/src/psphotExtendedSourceFits.c

    r36375 r37804  
    629629          pmSourceCacheModel (source, maskVal);
    630630
    631 # if (PS_TRACE_ON)
     631# if (0 && PS_TRACE_ON)
    632632          pmModel *model = source->modelFits->data[0];
    633633          int flags = 0xffffffff;

  • branches/eam_branches/ipp-20140904/psphot/src/psphotFitSourcesLinear.c

    r34418 r37804  
    263263            }
    264264        }
     265        if (modelSum == 0 ||  maskedSum == 0) {
     266            // no pixels at all drop the modelPSF (probably only an issue for external sources
     267            // which are created from a fake peak (full force))
     268            psFree(source->modelPSF);
     269            continue;
     270        }
    265271        if (modelSum < 0.5) continue; // skip sources with no model constraint (somewhat arbitrary limit)
    266272        if (modelSum < 0.8) {

  • branches/eam_branches/ipp-20140904/psphot/src/psphotFullForceReadout.c

    r37321 r37804  
    117117    psphotGalaxyShape (config, view, filerule);
    118118
     119    // measure aperture photometry corrections
     120    if (!psphotApResid (config, view, filerule)) {
     121        psLogMsg ("psphot", 3, "failed on psphotApResid");
     122        return psphotReadoutCleanup (config, view, filerule);
     123    }
     124
    119125    // calculate source magnitudes (psf mag and ap mag)
    120126    psphotMagnitudes(config, view, filerule);

  • branches/eam_branches/ipp-20140904/psphot/src/psphotMagnitudes.c

    r34404 r37804  
    3838        psAssert (psf, "missing psf?");
    3939
    40         if (!psphotMagnitudesReadout (config, recipe, view, readout, sources, psf)) {
     40        if (!psphotMagnitudesReadout (config, recipe, view, readout, sources, psf, i)) {
    4141            psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i);
    4242            return false;
    … …  
    4646}
    4747
    48 bool psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf) {
     48bool psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf, int index) {
    4949
    5050    bool status = false;
    … …  
    6565
    6666    // if backModel or backStdev are missing, the values of sky and/or skyErr will be set to NAN
    67     pmReadout *backModel = psphotSelectBackground (config, view);
    68     pmReadout *backStdev = psphotSelectBackgroundStdev (config, view);
     67    pmReadout *backModel = psphotSelectBackground (config, view, index);
     68    pmReadout *backStdev = psphotSelectBackgroundStdev (config, view, index);
    6969
    7070    // bit-masks to test for good/bad pixels

  • branches/eam_branches/ipp-20140904/psphot/src/psphotOutput.c

    r35769 r37804  
    2929}
    3030
    31 pmReadout *psphotSelectBackground (pmConfig *config, const pmFPAview *view) {
    32 
    33     bool status;
    34    
    35 
    36     pmFPAfile *file = psMetadataLookupPtr (&status, config->files, psphotGetFilerule("PSPHOT.BACKMDL"));
     31pmReadout *psphotSelectBackground (pmConfig *config, const pmFPAview *view, int index) {
     32
     33    pmFPAfile *file = pmFPAfileSelectSingle (config->files, psphotGetFilerule("PSPHOT.BACKMDL"), index);
    3734    if (!file) return NULL;
    3835
    … …  
    4138}
    4239
    43 pmReadout *psphotSelectBackgroundStdev (pmConfig *config, const pmFPAview *view) {
    44 
    45     bool status;
    46 
    47     pmFPAfile *file = psMetadataLookupPtr (&status, config->files, psphotGetFilerule("PSPHOT.BACKMDL.STDEV"));
     40pmReadout *psphotSelectBackgroundStdev (pmConfig *config, const pmFPAview *view, int index) {
     41
     42    pmFPAfile *file = pmFPAfileSelectSingle (config->files, psphotGetFilerule("PSPHOT.BACKMDL.STDEV"), index);
    4843    if (!file) return NULL;
    4944

  • branches/eam_branches/ipp-20140904/psphot/src/psphotPetrosianAnalysis.c

    r31154 r37804  
    1212    // XXX temporary user-supplied systematic sky noise measurement (derive from background model)
    1313    float skynoise = psMetadataLookupF32 (&status, recipe, "SKY.NOISE");
    14 
    15 # if (0)
    16     // if backModel or backStdev are missing, the values of sky and/or skyErr will be set to NAN
    17     // XXX use this to set skynoise
    18     pmReadout *backModel = psphotSelectBackground (config, view);
    19     pmReadout *backStdev = psphotSelectBackgroundStdev (config, view);
    20 # endif
    2114
    2215    // S/N limit to perform full non-linear fits

  • branches/eam_branches/ipp-20140904/psphot/src/psphotReadout.c

    r36623 r37804  
    325325    psphotExtendedSourceAnalysis (config, view, filerule); // pass 1 (detections->allSources)
    326326    psphotExtendedSourceFits (config, view, filerule); // pass 1 (detections->allSources)
     327    // measure some parameters for galaxy science
     328    psphotGalaxyParams (config, view, filerule);
    327329    psphotRadialApertures(config, view, filerule, 0);
    328330

  • branches/eam_branches/ipp-20140904/psphot/src/psphotReadoutCleanup.c

    r35771 r37804  
    5757    // use the psf-model to measure FWHM stats
    5858    if (psf) {
    59         if (!psphotPSFstats (readout, psf)) {
     59      if (!psphotPSFstatsSources (readout, sources, psf)) {
    6060            psError(PSPHOT_ERR_PROG, false, "Failed to measure PSF shape parameters");
    6161            return false;

  • branches/eam_branches/ipp-20140904/psphot/src/psphotSetNFrames.c

    r33691 r37804  
    4545
    4646    pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS");
    47     psAssert (detections, "missing detections?");
    48     psAssert (detections->allSources, "missing sources?");
     47    if (!detections) {
     48        psLogMsg ("psphot", PS_LOG_INFO, "No detections image input %d", index);
     49        return true;
     50    }
    4951
    5052    psArray *sources = detections->allSources;
     53    if (!sources) {
     54        psLogMsg ("psphot", PS_LOG_INFO, "No sources for input %d", index);
     55        return true;
     56    }
    5157
    5258    int col0 = expnum->col0;

  • branches/eam_branches/ipp-20140904/psphot/src/psphotSetThreads.c

    r37321 r37804  
    9292    psFree(task);
    9393
     94    task = psThreadTaskAlloc("PSPHOT_GALAXY_PARAMS", 11);
     95    task->function = &psphotGalaxyParams_Threaded;
     96    psThreadTaskAdd(task);
     97    psFree(task);
     98
    9499    return true;
    95100}

  • branches/eam_branches/ipp-20140904/psphot/src/psphotSourceStats.c

    r36375 r37804  
    185185            psThreadJob *job = psThreadJobAlloc ("PSPHOT_SOURCE_STATS");
    186186
    187             psArrayAdd(job->args, 1, cells->data[j]); // sources
    188 
    189             PS_ARRAY_ADD_SCALAR(job->args, INNER,   PS_TYPE_F32);
    190             PS_ARRAY_ADD_SCALAR(job->args, MIN_SN,  PS_TYPE_F32);
    191             PS_ARRAY_ADD_SCALAR(job->args, RADIUS,  PS_TYPE_F32);
    192             PS_ARRAY_ADD_SCALAR(job->args, SIGMA,   PS_TYPE_F32);
    193             PS_ARRAY_ADD_SCALAR(job->args, MIN_KRON_RADIUS, PS_TYPE_F32);
    194 
    195             PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK);
    196             PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK);
    197 
    198             PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nmoments
    199             PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail
    200             PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfaint
     187            psArrayAdd(job->args, 1, cells->data[j]); // 0 : sources
     188
     189            PS_ARRAY_ADD_SCALAR(job->args, INNER,   PS_TYPE_F32); // 1
     190            PS_ARRAY_ADD_SCALAR(job->args, MIN_SN,  PS_TYPE_F32); // 2
     191            PS_ARRAY_ADD_SCALAR(job->args, RADIUS,  PS_TYPE_F32); // 3
     192            PS_ARRAY_ADD_SCALAR(job->args, SIGMA,   PS_TYPE_F32); // 4
     193            PS_ARRAY_ADD_SCALAR(job->args, MIN_KRON_RADIUS, PS_TYPE_F32); // 5
     194
     195            PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); // 6
     196            PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); // 7
     197
     198            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // 8  : this is used as a return value for Nmoments
     199            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // 9  : this is used as a return value for Nfail
     200            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // 10 : this is used as a return value for Nfaint
    201201
    202202            if (!psThreadJobAddPending(job)) {
    … …  
    316316
    317317            // XXX: this must match the above
     318            psAbort ("this code is broken: must match psphotSourceStatsReadout above");
    318319            psArrayAdd(job->args, 1, cells->data[j]); // sources
    319320            psArrayAdd(job->args, 1, recipe);
    … …  
    472473
    473474    // change the value of a scalar on the array (wrap this and put it in psArray.h)
    474     scalar = job->args->data[7];
     475    scalar = job->args->data[8];
    475476    scalar->data.S32 = Nmoments;
    476477
    477     scalar = job->args->data[8];
     478    scalar = job->args->data[9];
    478479    scalar->data.S32 = Nfail;
    479480
    480     scalar = job->args->data[9];
     481    scalar = job->args->data[10];
    481482    scalar->data.S32 = Nfaint;
    482483

  • branches/eam_branches/ipp-20140904/psphot/src/psphotStackParseCamera.c

    r34528 r37804  
    262262    }
    263263
     264#ifdef MAKE_CHISQ_IMAGE
    264265    // generate an pmFPAimage for the chisqImage
    265266    {
    … …  
    293294        chisqVariance->save = saveChisq;
    294295    }
     296#else
     297    (void) saveChisq;
     298#endif   // MAKE_CHISQ_IMAGE
    295299
    296300    psTrace("psphot", 1, "Done with psphotStackParseCamera...\n");

  • branches/eam_branches/ipp-20140904/psphot/src/psphotStackReadout.c

    r36630 r37804  
    341341    // measure non-linear extended source models (exponential, deVaucouleur, Sersic) (sources sorted by S/N)
    342342    psphotExtendedSourceFits (config, view, STACK_RAW); // pass 1 (detections->allSources)
     343
     344    // measure some parameters for galxy science
     345    psphotGalaxyParams (config, view, STACK_RAW); // pass 1 (detections->allSources)
    343346
    344347    // create source children for the OUT filerule (for radial aperture photometry and output)
    … …  
    354357        return psphotReadoutCleanup (config, view, STACK_RAW);
    355358    }
     359    // These arrays are no longer needed. The inputs' sources arrays have references to the data
     360    psFree (objects);
     361    psFree (objectsOut);
    356362
    357363    bool radial_apertures = psMetadataLookupBool(NULL, recipe, "RADIAL_APERTURES");
    … …  
    410416    // measure aperture photometry corrections
    411417    if (!psphotApResid (config, view, STACK_RAW)) {
    412         psFree (objects);
    413         psFree (objectsOut);
    414418        psLogMsg ("psphot", 3, "failed on psphotApResid");
    415419        return psphotReadoutCleanup (config, view, STACK_RAW);
    … …  
    432436
    433437    logMemStats("final");
    434 #if (1)
     438#if (0)
    435439    psphotSourceMemory(config, view, STACK_RAW);
    436440    psphotSourceMemory(config, view, STACK_OUT);
    … …  
    448452    psphotStackRemoveChisqFromInputs(config, STACK_RAW);
    449453#endif
    450 
    451     psFree (objects);
    452     psFree (objectsOut);
    453454
    454455    // create the exported-metadata and free local data

  • branches/eam_branches/ipp-20140904/pstamp/scripts/pstamp_checkdependent.pl

    r37354 r37804  
    2424my $ra_max_3PI_hours = 22.5;  # disable updates for skycells with ra greater than this (hours)
    2525my $ra_max_3PI = $ra_max_3PI_hours * 15 * 3.14149 / 180.; # rawExp.ra is in radians
     26
     27# verbose flag for parse commands
     28my $parse_verbose = 0;
    2629
    2730# XXX: put this in a module somewhere
    … …  
    4952    'dbserver=s'    =>  \$ps_dbserver,  # postage stamp server dbserver
    5053    'verbose'       =>  \$verbose,
     54    'parse-verbose' =>  \$parse_verbose,
    5155    'save-temps'    =>  \$save_temps,
    5256    'no-update'     =>  \$no_update,
    … …  
    7882}
    7983
     84print "\n==== Starting dependency checking for $dep_id $stage $stage_id $component at " . (scalar localtime) . ". ====\n";
     85
    8086if ($label) {
    8187    # rlabel is deprecated. Use one based on the supplied label parameter which is the current label
    … …  
    8490    # (and formerly in pstampparse.pl) is not particularly clean but it's simple.
    8591    my $new_rlabel = 'ps_ud_' . $label;
     92    $rlabel = "undefined" if !defined $rlabel;
    8693    if ($new_rlabel ne $rlabel) {
    8794        print "Notice: using $new_rlabel instead of $rlabel for update label.\n";
    … …  
    120127}
    121128
    122 my $it = runToolAndParseExpectOne($cmd, $verbose);
     129my $it = runToolAndParseExpectOne($cmd, $parse_verbose);
    123130
    124131my_die("no components found", $PS_EXIT_PROG_ERROR) if ( !$it);
    … …  
    136143}
    137144
     145
    138146# magic is no longer rquired
    139147$need_magic = 0;
    … …  
    142150if ((($it->{state} eq 'full') or ($it->{state} eq 'update')) and ($it->{data_state} eq 'full')) {
    143151
     152    print "\nDependency Satisfied for $stage $stage_id $component\n";
    144153    # This Dependency is satisfied. All done. Release the pstampJobs
    145154    #
    … …  
    178187        }
    179188
    180         # assume the fault is transient.
     189        # hope the fault is transient.
    181190        # fault the dependent to give the fault a chance to correct itself
    182191        my_die("Component faulted on update dep_id: $dep_id", $PS_EXIT_SYS_ERROR);
    … …  
    279288            ($fault eq $PSTAMP_GONE)) {
    280289
    281             print "chipRun state is $chip->{chip_id} has state: $state data_state: $data_state fault: $fault cannot update\n";
     290            print "chipRun state is $chip->{chip_id} has state: $state class_id $chip->{class_id} data_state: $data_state fault: $fault cannot update.\n";
    282291            my $error_code;
    283292            if (($state eq 'error_cleaned') or ($data_state ='error_cleaned')) {
    … …  
    291300        } elsif ($chip->{state} eq 'goto_cleaned') {
    292301
    293             # we don't need to check for full data_state
    294             # if ($chip->{data_state} eq 'full') {
    295302            {
     303                print "dependent chip is in state goto_cleaned, changing state to update.\n";
     304
    296305                # cleanup must not be running. Set state to update. If this chip is not 'full' it will be
    297306                # set to be updated the next time this script is invoked
    298                 # XXX: In the very rare case that this chip run is actually in the running cleanup pantasks
     307                # XXX: In the very unlikely case that this chip run is actually in the running cleanup pantasks
    299308                # queue things may get confused but we can live with that
    300309                my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
    … …  
    313322            }
    314323        } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
     324
     325            print "Setting chip imfile $chip_id $chip->{class_id} to be updated.\n";
    315326
    316327            # chiptool does more state checking to insure this isn't done prematurely.
    … …  
    343354            # we've had a number of runs in this limbo state
    344355
     356            print "Changing chip run $chip_id to state update for $chip->{class_id} which is in data_state update.\n";
    345357            my $command = "$chiptool -updaterun -set_state update -chip_id $chip_id";
    346358            $command .= " -set_label $rlabel" if $rlabel;
    … …  
    364376                return $PSTAMP_GONE;
    365377            } elsif ($fault_count > $max_fault_count) {
    366                 print "$stage $stage_id has faulted $fault_count times. Giving up\n";
     378                print "Dependency for $stage $stage_id has been found in fault state $fault_count times. Giving up. Fault: $fault\n";
    367379                # XXX: stop faulting chips for now
    368380#                $fault = $PSTAMP_GONE;
    … …  
    421433    if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or
    422434         ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) {
    423         print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs\n";
     435        print STDERR "warpRun $warp_id $skycell_id has state $state $data_state faulting jobs.\n";
    424436        my $error_code;
    425437        if (($state eq 'error_cleaned') or ($data_state eq 'error_cleaned')) {
    … …  
    430442        return $error_code
    431443    }
    432     if (($data_state eq 'full') and ($metadata->{fault})) {
     444    if ((($data_state eq 'full') or ($data_state eq 'update')) and ($metadata->{fault})) {
    433445        # fault dependent.
    434446        my $fault = $metadata->{fault};
    435         print STDERR "warp $warp_id $skycell_id faulted: $fault";
     447        print STDERR "warp $warp_id $skycell_id faulted with code: $fault.";
     448        # XXX: TODO maybe: If ($fault != 2 and $fault != $PSTAMP_GONE) revert the skycell. The warp task only reverts fault 2
     449        # and sometimes warps will fault due to nfs problems that end up with fault = 4;.
    436450        return $fault;
    437451    }
    438452    if ($metadata->{quality} ne 0) {
    439         print STDERR "warp $warp_id $skycell_id bad quality on update: $metadata->{quality}\n";
     453        print STDERR "warp $warp_id $skycell_id bad quality on update: $metadata->{quality}.\n";
    440454        return $PSTAMP_GONE;
    441455    }
    … …  
    448462        # It's "safe" to set the state to update. If the skycell is not full it will be set to update
    449463        # the next time this task runs
     464        print "Change state of warp $warp_id to update.\n";
    450465        my $command = "$warptool -updaterun -set_state update -warp_id $warp_id";
    451466        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    465480    # get the list of input chips for this skycell
    466481    my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
    467     my $data = runToolAndParse($command, $verbose);
     482    my $data = runToolAndParse($command, $parse_verbose);
    468483    if (!$data or scalar @$data == 0) {
    469484        # This happens if the chipProcessedImfile disappears which happened when earlier
    470485        # versions of chiptool -revertprocessedimfile didn't check the chipRun.state before
    471486        # deleting the row.
    472         print STDERR "failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id";
     487        print STDERR "failed to find warpSkyCellMap for warpRun $warp_id skycell_id $skycell_id";
    473488        return $PSTAMP_GONE;
    474489    }
    … …  
    478493    my $chip_id;
    479494    foreach my $chip (@$data) {
     495        # XXX: change tools to include cam_state (camRun.state). If it is not full the warp updates will never run.
    480496        my $cam_fault = $chip->{cam_fault};
    481497        if (defined $cam_fault and $cam_fault > 0) {
    … …  
    488504        $chip->{decl} = $metadata->{decl};
    489505
     506if (0) {
    490507        # if chip has been magicked before require it to be magicked again
    491508        # because the warp pending query requires it.
    492 if (0) {
    493509        if ($chip->{magicked} < 0) {
    494510            print "Input has been destreaked so we must destreak before warping\n";
    … …  
    514530        # the reason we defer setting the warp to update is so that we can handle error conditions at previous
    515531        # stages more easily.
     532        print "Chips are ready for warp $warp_id $skycell->{skycell_id}. Setting skycell to be updated.\n";
    516533        my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
    517534        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    527544        }
    528545    } elsif ($chips_ready and $skycell->{data_state} eq 'update' and $skycell->{state} ne 'update') {
     546        print "Chips are ready for warp $warp_id $skycell->{skycell_id} but run state is $skycell->{state}. Changing state to update.\n";
    529547        my $command = "$warptool -updaterun -warp_id $warp_id -set_state update";
    530548        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    566584    my $skycell_id = $skycell->{skycell_id};
    567585
    568     # if ($metadata->{state} eq 'goto_cleaned' and $metadata->{data_state} eq 'full') {
    569586    if ($metadata->{state} eq 'goto_cleaned') {
    570587        # cleanup must not be running. Set state to update. If the skycell is not 'full' it will be
    571         # set to be updated the next time this script is invoked
    572         print "changing state of diffRun $diff_id from goto_cleaned to update. skycell $skycell_id is full\n";
     588        # set to be updated the next time this script is invoked.
     589        print "Changing state of diffRun $diff_id from goto_cleaned to update.\n";
    573590        my $command = "$difftool -updaterun -set_state update -diff_id $diff_id";
    574591        $command .= " -set_label $rlabel" if $rlabel;
    … …  
    589606        # check the state of the template stack
    590607        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    591         my $stack = runToolAndParseExpectOne($command, $verbose);
     608        my $stack = runToolAndParseExpectOne($command, $parse_verbose);
    592609        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
    593610
    … …  
    597614            return $PSTAMP_GONE;
    598615        }
     616        print " Stack $skycell->{stack2} is ready.\n";
    599617
    600618        # now check the warp
    601619        $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
    602         my $warp = runToolAndParseExpectOne($command, $verbose);
     620        my $warp = runToolAndParseExpectOne($command, $parse_verbose);
    603621        my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
    604622
    … …  
    615633        }
    616634        # warps are ready fall through and queue the diff update
     635        print " Warp $skycell->{warp1} $skycell_id is ready.\n";
    617636    } elsif ($diff_mode eq $IPP_DIFF_MODE_WARP_WARP) {
    618637        my $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
    619         my $warp1 = runToolAndParseExpectOne($command, $verbose);
     638        my $warp1 = runToolAndParseExpectOne($command, $parse_verbose);
    620639        my_die("failed to find warpSkyfile for warpRun $skycell->{warp1} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp1;
    621640
    … …  
    637656        }
    638657        $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
    639         my $warp2 = runToolAndParseExpectOne($command, $verbose);
     658        my $warp2 = runToolAndParseExpectOne($command, $parse_verbose);
    640659        my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp2;
    641660
    … …  
    661680        # check the state of the input stack
    662681        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
    663         my $stack1 = runToolAndParseExpectOne($command, $verbose);
     682        my $stack1 = runToolAndParseExpectOne($command, $parse_verbose);
    664683        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
    665684
    … …  
    671690        # check the state of the template stack
    672691        $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
    673         my $stack2 = runToolAndParseExpectOne($command, $verbose);
     692        my $stack2 = runToolAndParseExpectOne($command, $parse_verbose);
    674693        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack2;
    675694
    … …  
    684703        # check the state of the input stack
    685704        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
    686         my $stack = runToolAndParseExpectOne($command, $verbose);
     705        my $stack = runToolAndParseExpectOne($command, $parse_verbose);
    687706        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
    688707
    … …  
    695714        # now check the template warp
    696715        $command = "$warptool -warped -warp_id $skycell->{warp2} -skycell_id $skycell_id";
    697         my $warp = runToolAndParseExpectOne($command, $verbose);
     716        my $warp = runToolAndParseExpectOne($command, $parse_verbose);
    698717        my_die("failed to find warpSkyfile for warpRun $skycell->{warp2} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR) if !$warp;
    699718
    … …  
    736755            print "skipping $command\n";
    737756        }
     757    } else {
     758        print " diff is in update state\n";
    738759    }
    739760
    … …  
    763784    my $now = DateTime->now->mjd;
    764785    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
    765     print "Time to run $program: $dtime_tool\n";
     786    if ($dtime_tool > 0.1) {
     787        print "Time to run $program: $dtime_tool\n";
     788    }
    766789
    767790    my $buf = join "", @$stdout_buf;
    … …  
    777800
    778801    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
    779     print "Time to parse results from $program: $dtime_parse\n";
     802    if ($dtime_parse > 0.1) {
     803        print "Time to parse results from $program: $dtime_parse\n";
     804    }
    780805
    781806    return $results;
    … …  
    848873    if (!$dsRun_state) {
    849874        my $command = "$chiptool -listrun -chip_id $stage_id";
    850         my $data = runToolAndParse($command, $verbose);
     875        my $data = runToolAndParse($command, $parse_verbose);
    851876        my $chipRun = $data->[0];
    852877        $dsRun_state = $chipRun->{dsRun_state};
    … …  
    862887            foreach my $c (@$components) {
    863888                my $command = "$magicdstool -destreakedfile -magic_ds_id $magic_ds_id -component $c";
    864                 my $dsfile = runToolAndParseExpectOne($command, $verbose);
     889                my $dsfile = runToolAndParseExpectOne($command, $parse_verbose);
    865890                if (!$dsfile) {
    866891                    my_die("failed to find magicDSFile for ${stage}Run $stage_id $c", $PS_EXIT_UNKNOWN_ERROR);

  • branches/eam_branches/ipp-20140904/pstamp/scripts/pstamp_finish.pl

  • branches/eam_branches/ipp-20140904/pstamp/scripts/pstamp_job_run.pl

  • branches/eam_branches/ipp-20140904/pstamp/scripts/pstampparse.pl

  • branches/eam_branches/ipp-20140904/pstamp/scripts/request_finish.pl

  • branches/eam_branches/ipp-20140904/tools/heather/queuesliceslap2.pl

    r37311 r37804  
    44
    55
    6 #$label = "LAP.ThreePi.20130717";
    7 $label = "LAP.ThreePi.20130717.pole3";
    8 #$label = "LAP.ThreePi.20130717.pole3.CNP";
    9 
     6$label1 = "LAP.ThreePi.20130717";
     7$label2 = "LAP.ThreePi.20130717.pole3";
     8$label3 = "LAP.ThreePi.20130717.pole3.CNP";
     9#heather done bad things to segments and slices...
     10@labels = ($label1, $label2, $label3);
    1011@segments = (0,1,2,3,4,5,6,7,8);
     12#@segments=(0,1);
    1113@segments1 = ( -35, -22,-10,2,14,26,38,50,62);
    1214@segments2 = ( -22,-10,2,14,26,38,50,62,75);
    … …  
    1416
    1517@slices = ("00","02","04","06","08","10","12","14","16","18","20","22");
    16  
     18#@slices = ("00","02","04");
     19
     20foreach $label (@labels) {
     21
    1722foreach $slice (@slices) {
    1823   
    … …  
    2126    foreach $num (@segments) {
    2227
    23         $dvodb = "LAP.PV2.20130717";
    24         $minidvodb_group = "LAP.PV2.slice$slice.$segments[$num].cam";
     28        $dvodb = "LAP.PV2.V2.20130717";
     29        $minidvodb_group = "LAP.PV2.V2.slice$slice.$segments[$num].cam";
    2530        $dvodb_label = "$minidvodb_group";
    2631        $ra_min = $slice*15.0;
    … …  
    3742
    3843}
    39 $com2 = "addtool -definebyquery  -label $label -set_dvodb $dvodb -set_minidvodb_group LAP.PV2.pole.cam -set_minidvodb -set_label LAP.PV2.pole.cam -stage cam -uncensored -ra_min 0 -ra_max 360 -decl_min 75 -decl_max 90 -dbname gpc1";
     44$com2 = "addtool -definebyquery  -label $label -set_dvodb $dvodb -set_minidvodb_group LAP.PV2.V2.pole.cam -set_minidvodb -set_label LAP.PV2.V2.pole.cam -stage cam -uncensored -ra_min 0 -ra_max 360 -decl_min 75 -decl_max 90 -dbname gpc1";
    4045print "$com2\n";
    4146`$com2`;
     47}

  • branches/eam_branches/ipp-20140904/tools/heather/queuesliceslap2skycal.pl

    r37311 r37804  
    44
    55
    6 #$label = "LAP.ThreePi.20130717";
    7 #$label = "LAP.ThreePi.20130717.pole3";
    8 $label = "LAP.ThreePi.20130717.pole3.CNP";
    9 
     6$label1 = "LAP.ThreePi.20130717";
     7$label2 = "LAP.ThreePi.20130717.pole3";
     8$label3 = "LAP.ThreePi.20130717.pole3.CNP";
     9@labels = ($label1,$label2,$label3);
    1010@segments = (0,1,2,3,4,5,6,7,8);
    1111@segments1 = ( -35, -22,-10,2,14,26,38,50,62);
    … …  
    1414
    1515@slices = ("00","02","04","06","08","10","12","14","16","18","20","22");
    16 
     16foreach $label (@labels) {
    1717foreach $slice (@slices) {
    1818   
    … …  
    2121    foreach $num (@segments) {
    2222
    23         $dvodb = "LAP.PV2.20130717";
    24         $minidvodb_group = "LAP.PV2.slice$slice.$segments[$num].skycal";
     23        $dvodb = "LAP.PV2.V2.20130717";
     24        $minidvodb_group = "LAP.PV2.V2.slice$slice.$segments[$num].skycal";
    2525        $dvodb_label = "$minidvodb_group";
    2626        $ra_min = $slice*15.0;
    … …  
    3636    }
    3737
     38
    3839}
    39 $com2 = "addtool -definebyquery  -label $label -set_dvodb $dvodb -set_minidvodb_group LAP.PV2.pole.skycal -set_minidvodb -set_label LAP.PV2.pole.skycal -stage skycal -uncensored -ra_min 0 -ra_max 360 -dec_min 75 -dec_max 90 -dbname gpc1";
     40$com2 = "addtool -definebyquery  -label $label -set_dvodb $dvodb -set_minidvodb_group LAP.PV2.V2.pole.skycal -set_minidvodb -set_label LAP.PV2.V2.pole.skycal -stage skycal -uncensored -ra_min 0 -ra_max 360 -dec_min 75 -dec_max 90 -dbname gpc1";
    4041print "$com2\n";
    4142`$com2`;
     43}

  • branches/eam_branches/ipp-20140904/tools/iasc/getit

    r37326 r37804  
    4545chdir $final_destdir or die "failed to chdir to $final_destdir";
    4646
    47 system 'perl /data/ippc18.0/home/ipp/iasc.v2/rename.stamp.outputs.pl *';
     47system 'perl /data/ippc18.0/home/ipp/iasc/rename.stamp.outputs.pl *';
    4848
    4949
Note: See TracChangeset for help on using the changeset viewer.