Index: /branches/eam_branches/ipp-20140423/ippMonitor/czartool/czartool/czarconfig.xml =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/czartool/czartool/czarconfig.xml (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/czartool/czartool/czarconfig.xml (revision 36826) @@ -27,5 +27,5 @@ gpc1 - scidbm + scidbs1 ippuser ippuser @@ -47,5 +47,5 @@ 20 MINUTE - stdscience distribution summitcopy registration stack publishing pstamp + stdscience distribution summitcopy registration stack pstamp Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/camIQstats.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/camIQstats.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/camIQstats.d (revision 36826) @@ -10,4 +10,7 @@ WHERE camProcessedExp.cam_id = camRun.cam_id WHERE camProcessedExp.fault = 0 + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp.d (revision 36826) @@ -10,4 +10,7 @@ WHERE camProcessedExp.cam_id = camRun.cam_id WHERE camProcessedExp.fault = 0 + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() # define the arguments supplied to the links below (if any) Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp_Images.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp_Images.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp_Images.d (revision 36826) @@ -3,6 +3,4 @@ FILE camProcessedExp_Images.php MENU ipp.science.dat - -UNRESTRICTED AND 0>1 # the following WHERE clauses are added to all queries joined by AND @@ -12,4 +10,8 @@ WHERE camProcessedExp.cam_id = camRun.cam_id WHERE camProcessedExp.fault = 0 + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() + # define image names to be used below # IMAGE VAR basename filerule camera class_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp_NoImages.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp_NoImages.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedExp_NoImages.d (revision 36826) @@ -10,4 +10,7 @@ WHERE camProcessedExp.cam_id = camRun.cam_id WHERE camProcessedExp.fault = 0 + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() # define the arguments supplied to the links below (if any) Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedImfile.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedImfile.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/camProcessedImfile.d (revision 36826) @@ -10,4 +10,7 @@ WHERE camProcessedExp.cam_id = camRun.cam_id WHERE chipProcessedImfile.chip_id = camRun.chip_id + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/camStageExp.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/camStageExp.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/camStageExp.d (revision 36826) @@ -6,4 +6,7 @@ WHERE camRun.chip_id = chipRun.chip_id WHERE chipRun.exp_id = rawExp.exp_id + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/chipIQstats.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/chipIQstats.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/chipIQstats.d (revision 36826) @@ -8,4 +8,7 @@ WHERE chipRun.exp_id = rawExp.exp_id WHERE chipRun.chip_id = chipProcessedImfile.chip_id + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/chipProcessedExp.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/chipProcessedExp.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/chipProcessedExp.d (revision 36826) @@ -7,4 +7,7 @@ WHERE chipRun.state != 'new' WHERE chipRun.exp_id = rawExp.exp_id + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/chipProcessedImfile.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/chipProcessedImfile.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/chipProcessedImfile.d (revision 36826) @@ -9,4 +9,7 @@ WHERE chipProcessedImfile.chip_id = chipRun.chip_id WHERE chipProcessedImfile.fault = 0 + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() #UNRESTRICTED WHERE 0=1 Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/chipStageExp.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/chipStageExp.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/chipStageExp.d (revision 36826) @@ -5,4 +5,7 @@ WHERE chipRun.exp_id = rawExp.exp_id + +RESTRICT_COUNT AND dateobs >= CURDATE() +UNRESTRICTED AND dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/diffInputSkyfile.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/diffInputSkyfile.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/diffInputSkyfile.d (revision 36826) @@ -5,4 +5,7 @@ WHERE diffInputSkyfile.diff_id = diffRun.diff_id + +RESTRICT_COUNT AND 0 = 1 +UNRESTRICTED AND 0 = 1 # field size format name show link to extras Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/diffProcessedSkyfile.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/diffProcessedSkyfile.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/diffProcessedSkyfile.d (revision 36826) @@ -7,4 +7,7 @@ # WHERE diffRun.state = 'full' # WHERE diffSkyfile.fault = 0 + +RESTRICT_COUNT AND 0 = 1 +UNRESTRICTED AND 0 = 1 ARGS ARG7 diffSkyfile.diff_id=$diffSkyfile.diff_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/diffProcessedSkyfile_Images.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/diffProcessedSkyfile_Images.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/diffProcessedSkyfile_Images.d (revision 36826) @@ -3,4 +3,7 @@ FILE diffProcessedSkyfile_Images.php MENU ipp.stack.dat + +#RESTRICT_COUNT AND dateobs >= CURDATE() +#UNRESTRICTED AND dateobs >= CURDATE() UNRESTRICTED AND 0>1 Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/diffRun.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/diffRun.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/diffRun.d (revision 36826) @@ -3,4 +3,7 @@ FILE diffRun.php MENU ipp.stack.dat + +#RESTRICT_COUNT WHERE dateobs >= CURDATE() +#UNRESTRICTED WHERE dateobs >= CURDATE() # field size format name show link to extras Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/rawBurntoolState.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/rawBurntoolState.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/rawBurntoolState.d (revision 36826) @@ -5,6 +5,7 @@ # if no query restrictions are supplied, we want to limit the query or it is extremely long running: -RESTRICT_COUNT WHERE 0>1 -UNRESTRICTED WHERE exp_id = 0 +RESTRICT_COUNT WHERE dateobs >= CURDATE() +# UNRESTRICTED WHERE exp_id = 0 +UNRESTRICTED WHERE dateobs >= CURDATE() # field size format name show link to extras Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/warpProcessedSkyfiles.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/warpProcessedSkyfiles.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/warpProcessedSkyfiles.d (revision 36826) @@ -4,5 +4,6 @@ MENU ipp.science.dat -UNRESTRICTED WHERE 0=1 +RESTRICT_COUNT WHERE dateobs >= CURDATE() +UNRESTRICTED WHERE dateobs >= CURDATE() ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: /branches/eam_branches/ipp-20140423/ippMonitor/def/warpProcessedSkyfiles_Images.d =================================================================== --- /branches/eam_branches/ipp-20140423/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 36826) @@ -4,5 +4,8 @@ MENU ipp.science.dat -UNRESTRICTED WHERE 0=1 +RESTRICT_COUNT WHERE dateobs >= CURDATE() +UNRESTRICTED WHERE dateobs >= CURDATE() + +# UNRESTRICTED WHERE 0=1 # WHERE warpSkyfile.warp_id = warpRun.warp_id Index: /branches/eam_branches/ipp-20140423/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/ipp-20140423/ippScripts/Build.PL (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippScripts/Build.PL (revision 36826) @@ -116,4 +116,5 @@ scripts/lossy_compress_imfile.pl scripts/rawcheck.pl + scripts/permcheck.pl scripts/ipp_apply_burntool.pl scripts/ipp_apply_burntool_single.pl @@ -134,4 +135,5 @@ scripts/psphot_fullforce_warp.pl scripts/psphot_fullforce_summary.pl + scripts/stack_bkg_mk_mdc.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /branches/eam_branches/ipp-20140423/ippScripts/MANIFEST =================================================================== --- /branches/eam_branches/ipp-20140423/ippScripts/MANIFEST (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippScripts/MANIFEST (revision 36826) @@ -40,4 +40,5 @@ scripts/lossy_compress_imfile.pl scripts/rawcheck.pl +scripts/permcheck.pl scripts/ipp_apply_burntool.pl scripts/ipp_apply_burntool_single.pl Index: /branches/eam_branches/ipp-20140423/ippScripts/scripts/permcheck.pl =================================================================== --- /branches/eam_branches/ipp-20140423/ippScripts/scripts/permcheck.pl (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippScripts/scripts/permcheck.pl (revision 36826) @@ -0,0 +1,414 @@ +#!/usr/bin/env perl + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use Digest::MD5; +use URI; + +my $missing_tools = 0; +#my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); + +my ($server,$dbname,$stage,$stage_id); + +$server = $ENV{'NEB_SERVER'} unless $server; +# $dbname = 'gpc1'; +GetOptions( + 'server|s=s' => \$server, + 'dbname=s' => \$dbname, + 'stage=s' => \$stage, + 'stage_id|x=s' => \$stage_id, +) || pod2usage( 2 ); + +# Option parsing +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) + unless $server; +pod2usage( -msg => "Required options: --dbname", -exitval => 2 ) + unless $dbname; +pod2usage( -msg => "Required options: --stage [camera|stack|skycal]", -exitval => 2 ) + unless $stage; +pod2usage( -msg => "missing key", exitval => 2 ) + unless defined $stage_id; + +# Global options: + +my $do_ops = 1; +my %backup_hosts = ('ippb00' => 1, 'ippb01' => 1, + 'ippb02' => 1, 'ippb03' => 1, + 'ippb04' => 1, 'ippb05' => 1 + ); +my %backup_destinations = ('ippb04' => 1, + 'ippb05' => 1); +my $backup_Nvols = 3; + +my $ipprc; +if ($dbname eq 'gpc1') { + $ipprc = PS::IPP::Config->new( "GPC1" ); +} +else { + die "Unknown camera to use."; +} + +# Set up nebulous db interface +my $neb = Nebulous::Client->new( + proxy => "$server", +); +die "can't connected to Nebulous Server: $server" + unless defined $neb; + +# Pull data from the gpc1 database +my $verbose = 0; +my $mdcParser = PS::IPP::Metadata::Config->new; + +my $files; +if ($stage eq 'camera') { + my $cmd = "$camtool -processedexp -cam_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + $files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); +} +elsif ($stage eq 'stack') { + my $cmd = "$stacktool -sumskyfile -stack_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + $files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); +} +elsif ($stage eq 'skycal') { + my $cmd = "$staticskytool -skycalresult -skycal_id $stage_id -dbname $dbname"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $cmd, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stagetool: $error_code", $stage_id); + } + $files = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from stagetool", $stage_id); +} + +my %components = ('camera' => ['PSASTRO.OUTPUT'], + 'stack' => ['PPSTACK.UNCONV.COMP','PPSTACK.UNCONV.MASK.COMP','PPSTACK.UNCONV.VARIANCE.COMP', + 'PPSTACK.UNCONV.EXP','PPSTACK.UNCONV.EXPNUM','PPSTACK.UNCONV.EXPWT.COMP'], + 'skycal' => ['PSASTRO.OUTPUT.CMF']); + + +foreach my $entry (@$files) { + my $path_base = $entry->{path_base}; + my $data_state = $entry->{state}; + my $hostname = $entry->{hostname}; + foreach my $product (@{ $components{$stage} }) { + my $key = $ipprc->filename($product,$path_base); + +# Do validation + + # neb-stat level handling + my $stat = $neb->stat($key); + die "nebulous key: $key not found" unless $stat; + my $instances = $neb->find_instances($key, 'any'); + die "no instances found" unless $instances; + + my $user_copies; + eval { + $user_copies = $neb->getxattr($key, "user.copies"); + }; + unless(defined($user_copies)) { + $user_copies = 1; + } + + my $md5sum; + my @validation; + my %md5sum_uniq; + my $existing_copies = 0; + + my @existance; + my @md5sums; + my @diskfiles = map {URI->new($_)->file if $_} @$instances; + my @diskvols; + my @diskhosts; + my @quality; + my $Ngood = 0; + my $quality_mask = 0; + + for (my $i = 0; $i <= $#diskfiles; $i++) { + if (-e $diskfiles[$i]) { + $existance[$i] = 1; + $existing_copies++; + $md5sums[$i] = local_md5sum($diskfiles[$i]); + $md5sum_uniq{$md5sums[$i]} = 1; + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + } + ($diskhosts[$i],$diskvols[$i]) = parse_volume($diskfiles[$i]); + $validation[$i] = sprintf("% 3d %32s %s %s %d", + $existance[$i], + $md5sums[$i], + $diskfiles[$i], + $diskhosts[$i],$diskvols[$i] + ); + + # Pre-parse decisions + if ($existance[$i] == 0) { + $quality[$i] = 0; + } + elsif (is_backup_volume($diskhosts[$i])) { + $quality[$i] = 1; + $quality_mask = $quality_mask | 1; + $Ngood++; + } + else { + $quality[$i] = 2; + $Ngood++; + } + } + if (scalar(keys(%md5sum_uniq)) != 1) { #unlike the raw data, we don't know the truth. + die "There are multiple md5sum values for $key"; + } + $md5sum = (keys(%md5sum_uniq))[0]; + + print "\n$key $data_state $md5sum $hostname\n"; + if (1) { + + print + "object id: ", @$stat[0], "\n", + "key: ", @$stat[1], "\n"; + print + "epoch: ", @$stat[4], "\n"; + print + "md5sum count: ", scalar(keys %md5sum_uniq), "\n"; + print + "requested instances: ", $user_copies, "\n", + "available instances: ", @$stat[6], "\n", + "existing instances: ", $existing_copies, "\n", + "total instances: ", @$stat[7], "\n", + "instance location:\n", " " x 4; + print + join("\n" . " " x 4, @validation), "\n"; + } + + # Decide what to do + if ($Ngood == 0) { + # DO something to attempt to fix this. + my $deneb_key = $key; $deneb_key =~ s/.*?gpc/gpc/; + + open(DD,"/home/panstarrs/ipp/local/bin/deneb-locate.py $deneb_key 2> /dev/null |"); + my $good_file = ''; + while (
) { + $_ =~ s/^\s+//; + my ($z,undef,$ff) = split /\s+/; + if (($ff)&&(-e $ff)) { + my $md_response = `md5sum $ff`; + if ($md_response =~ /$md5sum/) { + $good_file = (split /\s+/,$md_response)[1]; + } + } + } + close(DD); + if ($good_file eq '') { + die "No valid instance of key: $key"; + } + else { + $quality[0] = 1; + print "cp $good_file $diskfiles[0]\n"; + system("cp $good_file $diskfiles[0]"); + } + # Begin my best validation thought + { + my $tmpmd5 = local_md5sum($diskfiles[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + + $Ngood = 1; # We now hand off this single valid instance object to be handled by the Ngood=1 case. + } + + if ($quality[0] == 0) { # The first instance is bad. + # But since we're here, and not up there, there must be at least one good copy. Find it. + for (my $i = 0; $i <= $#md5sums; $i++) { + if ($md5sums[$i] eq $md5sum) { # Found it. + print "cp $diskfiles[$i] $diskfiles[0]\n"; + system("cp $diskfiles[$i] $diskfiles[0]"); + # Begin my best validation thought + { + my $tmpmd5 = local_md5sum($diskfiles[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + last; # We're done here now. + } + } + } + + if ($Ngood == 1) { # We have only one version + if ($quality_mask & 1) { # ANd it's on a backup volume + print "neb-replicate $key\n"; + if ($do_ops) { + $neb->replicate($key) or die "failed to replicate the single valid copy"; + if ($@) { die $@; } + } + } + else { # And it's not, so put one there + my $rep_vol = get_random_backup_volume(); + print "neb-replicate --volume $rep_vol $key\n"; + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate the single valid copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + # End my best validation thought. + } + } + } + else { # N >= 2 + if (!($quality_mask & 1)) { # no backup copy + my $rep_vol = get_random_backup_volume(); + print "neb-replicate --volume $rep_vol $key\n"; + if ($do_ops) { + $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + # End my best validation thought. + } + } + if (!($quality_mask & 2)) { # no copy on the requested host, so select the first as valid. + for (my $i = 0; $i <= $#diskfiles; $i++) { + if ($quality[$i] == 2) { + $quality[$i] = 1; + last; + } + } + } + for (my $i = 0; $i <= $#diskfiles; $i++) { +# print "$existance[$i] $quality[$i] $md5sums[$i] $md5sum $diskhosts[$i] $diskvols[$i]\n"; + if ($quality[$i] != 1) { + if ($existance[$i] == 0) { # This disk file doesn't exist. + system("touch $diskfiles[$i]"); + } + my $cull_vol = $diskhosts[$i] . "." . $diskvols[$i]; + print "neb-cull --volume $cull_vol $key\n"; + if ($do_ops) { + # The tilde here is to force hard volumes. Don't touch it. + $neb->cull($key,"~${cull_vol}",2) or die "failed to cull a superfluous instance"; + if ($@) { die "$@"; } + } + } + } + } + } # end product for this entry +} # end entry for this id + +sub local_md5sum { + my $filename = shift; + my $volume = (split /\//, $filename)[2]; + my $host = $volume; + $host =~ s/\.\d//; +# print "$filename $host $volume\n"; + my $response = `ssh $host md5sum $filename`; + chomp($response); + my ($sum, undef) = split /\s+/, $response; + unless(defined($sum)) { + my_die("Failed to calculate md5sum locally. $filename $host $volume"); + } + return($sum); +} + + +sub parse_volume { + my $filename = shift(@_); + my $full_volume = (split /\//, $filename)[2]; + my ($hostname,$vol_index) = split /\./, $full_volume; # /; + return($hostname,$vol_index); +} + +sub is_backup_volume { + my $hostname = shift(@_); + if (exists($backup_hosts{$hostname})) { + return(1); + } + return(0); +} + +sub get_random_backup_volume { + my $NN = scalar keys %backup_destinations; + my $backup_host = (keys %backup_destinations)[int(rand($NN))]; + my $backup_vol = int(rand($backup_Nvols)); + + return("${backup_host}.${backup_vol}"); +} + +sub my_die { + my $msg = shift(@_); + print $msg . "\n"; + foreach my $a (@_) { + print "ARG: $a\n"; + } + die; +} + + + + + + + + + + + + + + + + + + + + +__END__ + + Index: /branches/eam_branches/ipp-20140423/ippScripts/scripts/psphot_fullforce_warp.pl =================================================================== --- /branches/eam_branches/ipp-20140423/ippScripts/scripts/psphot_fullforce_warp.pl (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippScripts/scripts/psphot_fullforce_warp.pl (revision 36826) @@ -194,5 +194,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); + &my_die("Unable to perform psphotFullForce: $error_code", $ff_id, $warp_id, $skycell_id, $error_code); } Index: /branches/eam_branches/ipp-20140423/ippScripts/scripts/queuestaticsky.pl =================================================================== --- /branches/eam_branches/ipp-20140423/ippScripts/scripts/queuestaticsky.pl (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippScripts/scripts/queuestaticsky.pl (revision 36826) @@ -14,4 +14,6 @@ my ($seq_id, $tess_id, $projection_cell, $label, $dist_group, $dbname, $pretend, $simple, $verbose, $no_update); + +my $good_frac_min = 0.05; GetOptions( @@ -87,4 +89,5 @@ . " -select_skycell_id $projection_cell%" . " -set_workdir $workdirBase/$label/$datestr" + . " -select_good_frac_min $good_frac_min" . " -set_label $label" . " -set_data_group $label" Index: /branches/eam_branches/ipp-20140423/ippScripts/scripts/stack_bkg_mk_mdc.pl =================================================================== --- /branches/eam_branches/ipp-20140423/ippScripts/scripts/stack_bkg_mk_mdc.pl (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippScripts/scripts/stack_bkg_mk_mdc.pl (revision 36826) @@ -0,0 +1,91 @@ +#! /usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DBI; +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + + +my ($dbname,$stack_id,$label,$camera,$filter); +#$dbname = 'gpc1'; +#$camera = 'GPC1'; +GetOptions( + 'stack_id=s' => \$stack_id, + 'label=s' => \$label, + 'filter=s' => \$filter, + 'dbname=s' => \$dbname, + 'camera=s' => \$camera, + ) or pod2usage ( 2 ); +pod2usage( -msg => "Required options; --stack_id | --label", -exitval => 3) unless + ((defined $stack_id)||(defined $label)); +pod2usage( -msg => "Required options; --camera --dbname ", -exitval => 3) unless + (defined($camera) && defined($dbname)); +my $verbose = 0; + +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +my $dbserver = 'ippdb01'; +my $dbuser = 'ippuser'; +my $dbpass = 'ippuser'; +my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + +my $ipprc = PS::IPP::Config->new( $camera ); +my $mdcParser = PS::IPP::Metadata::Config->new; + + +my $query = "SELECT DISTINCT chip_id,cam_id,path_base FROM stackRun JOIN stackInputSkyfile USING(stack_id) JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun USING (chip_id) WHERE 1 "; + +if (defined($label)) { $query .= " AND stackRun.label = '${label}' "; } +if (defined($stack_id)) { $query .= " AND stack_id = ${stack_id} "; } +if (defined($filter)) { $query .= " AND stackRun.filter = '${filter}' "; } + +my $cams = $db->selectall_arrayref( $query ); + +#my $logDest = $ipprc->filename($logRule, $outroot, $class_id); + +print "backgroundStackInputs MULTI\n\n"; +my $index = 0; + +foreach my $cam (@{ $cams }) { + my ($chip_id,$cam_id,$cam_path) = @{ $cam }; + my $smf_file = $ipprc->filename("PSASTRO.OUTPUT",$cam_path); + print "backgroundStackInput${index} METADATA\n"; + print " chip_id S64 $chip_id\n"; + print " cam_id S64 $cam_id\n"; + print " astrom STR $smf_file\n"; + print " models METADATA\n"; + + my $command = "chiptool -processedimfile -chip_id $chip_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + die "Unable to parse metadata."; + } + my $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + die "Unabel to parse metadata: 2."; + + foreach my $imf (@$imfiles) { + my $class_id = $imf->{class_id}; + my $path_base= $imf->{path_base}; + + my $model = $ipprc->filename("PPIMAGE.BACKMDL",$path_base,$class_id); + + print " ${class_id} STR $model\n"; + } + print " END\n"; + print "END\n\n"; + + $index++; +} + Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/Makefile.am (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/Makefile.am (revision 36826) @@ -5,4 +5,8 @@ tables.P2.vot \ tables.ST.vot \ + tables.DF.vot \ + tables.DO.vot \ + tables.FW.vot \ + tables.FO.vot \ settings.xml Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/README =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/README (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/README (revision 36826) @@ -1,3 +1,4 @@ -ippToPspsDbSchema.sql : SQL commands to build the real (not scratch) ipptopsps database +ippToPspsDbSchema.sql : SQL commands to build the real (not scratch) ipptopsps +database ( the database that tracks/queues the batch creation) settings.xml : locations of things such as mysql dbs of interest tables.IN.full.vot : original version of the init batch tables, with invalid SkyCell, ProjectionCell, Regions @@ -6,2 +7,21 @@ tables.P2.vot : P2 detection batch table tables.ST.vot : stack detection batch table +tables.DF.vot : diff detection batch table +tables.DO.vot : diff object batch table +tables.FW.vot : forced warp measurement table +tables.FO.vot : forced mean object table + + +Heather's Vague Instructions: + +to create a new batch db: + +create database ipptopsps2 +cut and paste the sql file - should be no problems, I tried to reorganize them so no foriegn key constraint fails. + +to make a scratch db: + check settings.xml if necessary and recompile + mysql -h localhost -u dvo -pdvo -e "create database ipptopsps2_scratch" + ippjython setupScratchDb.py + +should be good to go! Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/ippToPspsDbSchema.sql =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/ippToPspsDbSchema.sql (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/ippToPspsDbSchema.sql (revision 36826) @@ -45,4 +45,6 @@ `stage_id` bigint(20) default NULL, `total_detections` bigint(20) default NULL, + `skychunk` varchar(400) default NULL, + `datastore_file` varchar(400) default NULL, `comment` varchar(400) default NULL, `purged` tinyint(4) NOT NULL default '0', @@ -51,24 +53,4 @@ SET character_set_client = @saved_cs_client; --- --- Table structure for table `box` --- - -DROP TABLE IF EXISTS `box`; -SET @saved_cs_client = @@character_set_client; -SET character_set_client = utf8; -CREATE TABLE `box` ( - `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, - `id` bigint(20) unsigned NOT NULL auto_increment, - `skychunk` varchar(30) default NULL, - `ra_center` float default NULL, - `dec_center` float default NULL, - `box_side` float default NULL, - `ingested` varchar(64) default 'none', - PRIMARY KEY (`id`), - KEY `fk_skychunk` (`skychunk`), - CONSTRAINT `fk_skychunk` FOREIGN KEY (`skychunk`) REFERENCES `skychunk` (`name`) ON DELETE CASCADE -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1; -SET character_set_client = @saved_cs_client; -- @@ -125,4 +107,8 @@ `queue_ST` tinyint(1) default '0', `queue_OB` tinyint(1) default '0', + `queue_DF` tinyint(1) default '0', + `queue_DO` tinyint(1) default '0', + `queue_FW` tinyint(1) default '0', + `queue_FO` tinyint(1) default '0', `active` tinyint(1) default '1', `parallel` tinyint(1) default '0', @@ -136,4 +122,25 @@ -- +-- Table structure for table `box` needs to be after skychunk or get err:150 +-- + +DROP TABLE IF EXISTS `box`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `box` ( + `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP, + `id` bigint(20) unsigned NOT NULL auto_increment, + `skychunk` varchar(30) default NULL, + `ra_center` float default NULL, + `dec_center` float default NULL, + `box_side` float default NULL, + `ingested` varchar(64) default 'none', + PRIMARY KEY (`id`), + KEY `fk_skychunk` (`skychunk`), + CONSTRAINT `fk_skychunk` FOREIGN KEY (`skychunk`) REFERENCES `skychunk` (`name`) ON DELETE CASCADE +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `detection` -- @@ -152,4 +159,20 @@ -- +-- Table structure for table `forcedwarp` +-- + +DROP TABLE IF EXISTS `forcedwarp`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `forcedwarp` ( + `batch_id` bigint(20) unsigned NOT NULL, + `exp_id` bigint(20) default NULL, + `filter` varchar(1) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `forcedwarpd_ibfk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `object` -- @@ -163,4 +186,18 @@ KEY `batch_id` (`batch_id`), CONSTRAINT `object_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; +-- +-- Table structure for table `diffobject` +-- + +DROP TABLE IF EXISTS `diffobject`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `diffobject` ( + `batch_id` bigint(20) unsigned NOT NULL, + `region` varchar(50) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `diffobject_fk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; SET character_set_client = @saved_cs_client; @@ -200,4 +237,20 @@ -- +-- Table structure for table `diff` +-- + +DROP TABLE IF EXISTS `diff`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `diff` ( + `batch_id` bigint(20) unsigned NOT NULL, + `filter` varchar(1) default NULL, + `diff_type` varchar(20) default NULL, + KEY `batch_id` (`batch_id`), + CONSTRAINT `diff_ibfk_1` FOREIGN KEY (`batch_id`) REFERENCES `batch` (`batch_id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + +-- -- Table structure for table `stripe` -- @@ -225,2 +278,19 @@ -- Dump completed on 2012-04-10 6:00:59 + + +DROP TABLE IF EXISTS `dvodb`; +SET @saved_cs_client = @@character_set_client; +SET character_set_client = utf8; +CREATE TABLE `dvodb` ( + `dvo_db` varchar(30) default NULL, + `batch_type` varchar(10) default NULL, + `stage_id` bigint(20) unsigned default NULL, + `ra_center` float default NULL, + `dec_center` float default NULL, + `uniq_key` varchar(30) default NULL, + UNIQUE KEY `uniq_key` (`uniq_key`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +SET character_set_client = @saved_cs_client; + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/parse_vot.pl =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/parse_vot.pl (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/parse_vot.pl (revision 36826) @@ -0,0 +1,120 @@ +#!/usr/bin/env perl + +# this thing will just parse the votables and print out in 'ken schema style' + +$wiki=1; +#$csv = 1; +$infield = 0; +$intable = 0; +while (<>) { + + if ($_ =~/\//; + $t =~ s/\<\/DESCRIPTION\>//; + print $t; + $infield = 1; + } + + if ($_ =~ /\<\/FIELD/) { + if ($wiki) { print "||";} + if ($csv) {print "\"";} + print "\n"; + $infield = 0; + } + + } else { + #not in field something new + if ($_ =~ /\//; + $t =~ s/\<\/DESCRIPTION\>//; + if ($csv) { + $dostuff = 1; + } else { + print "$t\n"; + print "---------------------------\n"; + } + $intable = 1; + + } + + } + } + if ($_ =~ /\<\/TABLE\>/) { + $intable = 0; + } + + + } + } + + +} Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/settings.xml =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/settings.xml (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/settings.xml (revision 36826) @@ -6,9 +6,9 @@ - SAS15 + ipptopsps_pv2_logs - ipptopsps_scratch + ipptopsps2_scratch ipp006 dvo @@ -16,338 +16,6 @@ 16 - - ipptopsps_scratch - ipp007 - dvo - dvo - 16 - - - ipptopsps_scratch - ipp008 - dvo - dvo - 16 - - - ipptopsps_scratch - ipp009 - dvo - dvo - 16 - - - ipptopsps_scratch - ipp010 - dvo - dvo - 16 - - - ipptopsps_scratch - ipp011 - dvo - dvo - 16 - - - - ipptopsps_scratch - ipp012 - dvo - dvo - 16 - - - - - - ipptopsps_scratch - ipp013 - dvo - dvo - 32 - - - - ipptopsps_scratch - ipp014 - dvo - dvo - 16 - - - - ipptopsps_scratch - ipp015 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp016 - dvo - dvo - 16 - - - ipptopsps_scratch - ipp018 - dvo - dvo - 16 - - - - ipptopsps_scratch - ipp019 - dvo - dvo - 16 - - - - ipptopsps_scratch - ipp020 - dvo - dvo - 16 - - - ipptopsps_scratch - ipp021 - dvo - dvo - 16 - - - - ipptopsps_scratch - ipp025 - dvo - dvo - 16 - - - - - - - ipptopsps_scratch - ipp037 - dvo - dvo - 16 - - - - - ipptopsps_scratch - ipp038 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp039 - dvo - dvo - 20 - - - - - ipptopsps_scratch - ipp040 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp041 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp042 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp043 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp044 - dvo - dvo - 20 - - - ipptopsps_scratch - ipp045 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp046 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp047 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp048 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp049 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp050 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp051 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp052 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp053 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp054 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp055 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp056 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp057 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp058 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp059 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp060 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp061 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp062 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp063 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp064 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp065 - dvo - dvo - 32 - - - ipptopsps_scratch - ipp066 - dvo - dvo - 32 - - ipptopsps_scratch + ipptopsps2_scratch stare00 dvo @@ -356,5 +24,5 @@ - ipptopsps_scratch + ipptopsps2_scratch stare01 dvo @@ -363,5 +31,5 @@ - ipptopsps_scratch + ipptopsps2_scratch stare02 dvo @@ -370,5 +38,5 @@ - ipptopsps_scratch + ipptopsps2_scratch stare03 dvo @@ -377,5 +45,5 @@ - ipptopsps_scratch + ipptopsps2_scratch stare04 dvo @@ -409,5 +77,5 @@ - ipptopsps + ipptopsps2 ipp006 dvo Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.DF.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.DF.vot (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.DF.vot (revision 36826) @@ -0,0 +1,306 @@ + + + + +
+ VOTable description of PSPS table DiffMeta + + + diff identifier + + + batch identifier + + + survey flag identifier + + + filter identifier + + + diff type identifier + + + positive image identifier + + + negative image identifier + + + single float givijng PrimaryCell.SkyCell identifier + + + photometry code + + + saturation magnitude level + + + analysis version index + + + exposure time of positive image + + + name of astrometric projection in RA + + + name of astrometric projection in DEC + + + RA corresponding to reference pixel + + + DEC corresponding to reference pixel + + + reference pixel value for RA + + + reference pixel value for DEC + + + scale factor for RA + + + scale factor for DEC + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + calibration modification number + + + Data release + + + + +
+ + VOTable description of PSPS table DiffDetection + + + ODM diff object identifier + + + galactic coordinates ODM object identifier + + + ODM diff detection identifier + + + diff detection meta identifer + + + detection id generated by IPP + + + object id generated by IPP + + + filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... + + + survey flag identifier + + + random diff id + + + single float givijng PrimaryCell.SkyCell identifier + + + identifier to dvo spacial region + + + Time of mid observation of positive detection in Mean Julian Days + + + measured x on CCD from PSF fit + + + measured y on CCD from PSF fit + + + estimated error in x + + + estimated error in y + + + plate scale at this location (from DVO) + + + position angle (sky-to-chip) at this location (from DVO) + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zeropoint + + + estimated extinction due to non-photometric conditions + + + exposure time (informational) + + + airmass (informational) + + + PSF instrumental flux (in analog-to-digital units/sec) or adu/sec + + + estimated error in flux + + + model PSF width in major axis + + + model PSF width in minor axis + + + model PSF orientation angle degrees East of North + + + model PSF core parameter [k in (1 + kz + z^1.66)] + + + PSF coverage factor + + + PSF-weighted fraction of pixels totally unmasked + + + reduced-chisq of the psf model fit. (ndof?, npix?) + + + PSF likelihood + + + moment XX + + + moment XY + + + moment YY + + + first radial moment + + + half radial moment + + + 3rd moment cos(theta) + + + 3rd moment sin(theta) + + + 4th moment cos(theta) + + + 4th moment sin(theta) + + + Aperture Flux + + + Aperture Flux error + + + Aperture fill factor + + + Kron flux + + + Kron flux error + + + Kron radius + + + Kron radius error + + + NPix > 3sigma + + + fPos/(fPos + fNeg) + + + nPos/(nPos + nNeg) + + + nPos/(nPos + nMask) + + + nPos/(nPos + nMask + nNeg) + + + distance to positive match source + + + distance to negative match source + + + signal to noise of pos match source + + + signal to noise of neg match source + + + PSF sky level at source + + + estimated error in sky + + + flag indicating provenance information + + + Data release when this detection was originally taken. Recalibrations do not affect this value. + + + + +
+ + VOTable description of PSPS table DiffToImage + + + diff identifier + + + hashed exposure-ccdID identifier + + + + +
+ + Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.DO.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.DO.vot (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.DO.vot (revision 36826) @@ -0,0 +1,112 @@ + + + + + + VOTable description of PSPS table DifDetObject + + + IAU Name constructed from objID position PS1.x JHHMMSS.SS+/- + + + ODM object identifier index + + + galactic coordinates ODM object identifier + + + IPP object number + + + Survey ID + + + single float giving PrimaryCell.SkyCell identifier + + + HTM index + + + Zone index + + + Random integer between 0 and 2^32 = 4,394,967,296 + + + batchID + + + identifier to dvo spatial region + + + flag indicating odd properties + + + subset of objInfoFlag - flags related to goodness of object + + + Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation + + + Right Ascension + + + Declination + + + Right Ascension error + + + Declination error + + + Cartesian x + + + Cartesian y + + + Cartesian z + + + ecliptic longitude + + + ecliptic latitude + + + galactic longitude + + + galactic latitude + + + qf perfect + + + Data release + + + total number of detection measurements in all filters + + + number of measures in g filter + + + number of measures in r filter + + + number of measures in i filter + + + number of measures in z filter + + + number of measures in y filter + + + + +
+ + Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.FO.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.FO.vot (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.FO.vot (revision 36826) @@ -0,0 +1,748 @@ + + + + + + VOTable description of PSPS table ForcedMeanObject + + + ODM object identifier index + + + galactic coordinates ODM object identifier + + + IPP object number + + + total number of detection measurements in all filters + + + batch id + + + stack image id (how to carry through IPP+DVO + + + number of detections in g filter + + + number of detections included in robust psf mean in g filter + + + number of detections included in robust mean of Kron mag in g filter + + + number of detections included in robust mean of Ap Mag in g filter + + + number of detections included in robust mean of ApFlx 5 in g filter + + + number of detections included in robust mean of ApFlx 6 in g filter + + + robust mean g psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile + + + estimated error of g psf AB magnitude mean(individual errors added in quadrature) + + + standard deviation of measurements included in the robust mean + + + mean g kron AB magnitude + + + estimated error of g kron AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + mean g aperture AB magnitude + + + estimated error of g aperture AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + Flux inside r = 5 in g unconvolved stack + + + estimated error in flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 in g unconvolved stack + + + estimated error in flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + flags for g forced + + + per-object smear polarizability X11 term + + + per-object smear polarizability X12 term + + + per-object smear polarizability X22 term + + + per-object smear polarizability E1 term + + + per-object smear polarizability E2 term + + + per-object shear polarizability X11 term + + + per-object shear polarizability X12 term + + + per-object shear polarizability X22 term + + + per-object shear polarizability E1 term + + + per-object shear polarizability E2 term + + + psf model smear polarizability X11 term + + + psf model smear polarizability X12 term + + + psf model smear polarizability X22 term + + + psf model smear polarizability E1 term + + + psf model smear polarizability E2 term + + + psf model shear polarizability X11 term + + + psf model shear polarizability X12 term + + + psf model shear polarizability X22 term + + + psf model shear polarizability E1 term + + + psf model shear polarizability E2 term + + + second moment (Mxx+Myy) + + + polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) + + + polarization, KSB-corrected 2Mxy / (Mxx + Myy) + + + stack image id (how to carry through IPP+DVO + + + number of detections in r filter + + + number of detections included in robust psf mean in r filter + + + number of detections included in robust mean of Kron mag in r filter + + + number of detections included in robust mean of Ap Mag in r filter + + + number of detections included in robust mean of ApFlx 5 in r filter + + + number of detections included in robust mean of ApFlx 6 in r filter + + + robust mean r psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile + + + estimated error of g psf AB magnitude mean(individual errors added in quadrature) + + + standard deviation of measurements included in the robust mean + + + mean r kron AB magnitude + + + estimated error of r kron AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + mean r aperture AB magnitude + + + estimated error of r aperture AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + Flux inside r = 5 in r unconvolved stack + + + estimated error in flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 in r unconvolved stack + + + estimated error in flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + flags for r forced + + + per-object smear polarizability X11 term + + + per-object smear polarizability X12 term + + + per-object smear polarizability X22 term + + + per-object smear polarizability E1 term + + + per-object smear polarizability E2 term + + + per-object shear polarizability X11 term + + + per-object shear polarizability X12 term + + + per-object shear polarizability X22 term + + + per-object shear polarizability E1 term + + + per-object shear polarizability E2 term + + + psf model smear polarizability X11 term + + + psf model smear polarizability X12 term + + + psf model smear polarizability X22 term + + + psf model smear polarizability E1 term + + + psf model smear polarizability E2 term + + + psf model shear polarizability X11 term + + + psf model shear polarizability X12 term + + + psf model shear polarizability X22 term + + + psf model shear polarizability E1 term + + + psf model shear polarizability E2 term + + + second moment (Mxx+Myy) + + + polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) + + + polarization, KSB-corrected 2Mxy / (Mxx + Myy) + + + stack image id (how to carry through IPP+DVO + + + number of detections in i filter + + + number of detections included in robust psf mean in i filter + + + number of detections included in robust mean of Kron mag in i filter + + + number of detections included in robust mean of Ap Mag in i filter + + + number of detections included in robust mean of ApFlx 5 in i filter + + + number of detections included in robust mean of ApFlx 6 in i filter + + + robust mean i psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile + + + estimated error of i psf AB magnitude mean(individual errors added in quadrature) + + + standard deviation of measurements included in the robust mean + + + mean i kron AB magnitude + + + estimated error of i kron AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + mean i aperture AB magnitude + + + estimated error of i aperture AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + Flux inside r = 5 in i unconvolved stack + + + estimated error in flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 in i unconvolved stack + + + estimated error in flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + flags for i forced + + + per-object smear polarizability X11 term + + + per-object smear polarizability X12 term + + + per-object smear polarizability X22 term + + + per-object smear polarizability E1 term + + + per-object smear polarizability E2 term + + + per-object shear polarizability X11 term + + + per-object shear polarizability X12 term + + + per-object shear polarizability X22 term + + + per-object shear polarizability E1 term + + + per-object shear polarizability E2 term + + + psf model smear polarizability X11 term + + + psf model smear polarizability X12 term + + + psf model smear polarizability X22 term + + + psf model smear polarizability E1 term + + + psf model smear polarizability E2 term + + + psf model shear polarizability X11 term + + + psf model shear polarizability X12 term + + + psf model shear polarizability X22 term + + + psf model shear polarizability E1 term + + + psf model shear polarizability E2 term + + + second moment (Mxx+Myy) + + + polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) + + + polarization, KSB-corrected 2Mxy / (Mxx + Myy) + + + stack image id (how to carry through IPP+DVO + + + number of detections in z filter + + + number of detections included in robust psf mean in z filter + + + number of detections included in robust mean of Kron mag in z filter + + + number of detections included in robust mean of Ap Mag in z filter + + + number of detections included in robust mean of ApFlx 5 in z filter + + + number of detections included in robust mean of ApFlx 6 in z filter + + + robust mean z psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile + + + estimated error of z psf AB magnitude mean(individual errors added in quadrature) + + + standard deviation of measurements included in the robust mean + + + mean z kron AB magnitude + + + estimated error of z kron AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + mean z aperture AB magnitude + + + estimated error of z aperture AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + Flux inside r = 5 in z unconvolved stack + + + estimated error in flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 in z unconvolved stack + + + estimated error in flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + flags for z forced + + + per-object smear polarizability X11 term + + + per-object smear polarizability X12 term + + + per-object smear polarizability X22 term + + + per-object smear polarizability E1 term + + + per-object smear polarizability E2 term + + + per-object shear polarizability X11 term + + + per-object shear polarizability X12 term + + + per-object shear polarizability X22 term + + + per-object shear polarizability E1 term + + + per-object shear polarizability E2 term + + + psf model smear polarizability X11 term + + + psf model smear polarizability X12 term + + + psf model smear polarizability X22 term + + + psf model smear polarizability E1 term + + + psf model smear polarizability E2 term + + + psf model shear polarizability X11 term + + + psf model shear polarizability X12 term + + + psf model shear polarizability X22 term + + + psf model shear polarizability E1 term + + + psf model shear polarizability E2 term + + + second moment (Mxx+Myy) + + + polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) + + + polarization, KSB-corrected 2Mxy / (Mxx + Myy) + + + stack image id (how to carry through IPP+DVO + + + number of detections in y filter + + + number of detections included in robust psf mean in y filter + + + number of detections included in robust mean of Kron mag in y filter + + + number of detections included in robust mean of Ap Mag in y filter + + + number of detections included in robust mean of ApFlx 5 in y filter + + + number of detections included in robust mean of ApFlx 6 in y filter + + + robust mean y psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number summary? 1/12 is ~9th percentile + + + estimated error of y psf AB magnitude mean(individual errors added in quadrature) + + + standard deviation of measurements included in the robust mean + + + mean y kron AB magnitude + + + estimated error of y kron AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + mean y aperture AB magnitude + + + estimated error of y aperture AB magnitude mean + + + standard deviation of measurements included in the robust mean + + + Flux inside r = 5 in y unconvolved stack + + + estimated error in flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 in y unconvolved stack + + + estimated error in flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + flags for y forced + + + per-object smear polarizability X11 term + + + per-object smear polarizability X12 term + + + per-object smear polarizability X22 term + + + per-object smear polarizability E1 term + + + per-object smear polarizability E2 term + + + per-object shear polarizability X11 term + + + per-object shear polarizability X12 term + + + per-object shear polarizability X22 term + + + per-object shear polarizability E1 term + + + per-object shear polarizability E2 term + + + psf model smear polarizability X11 term + + + psf model smear polarizability X12 term + + + psf model smear polarizability X22 term + + + psf model smear polarizability E1 term + + + psf model smear polarizability E2 term + + + psf model shear polarizability X11 term + + + psf model shear polarizability X12 term + + + psf model shear polarizability X22 term + + + psf model shear polarizability E1 term + + + psf model shear polarizability E2 term + + + second moment (Mxx+Myy) + + + polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) + + + polarization, KSB-corrected 2Mxy / (Mxx + Myy) + + + + +
+ + Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.FW.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.FW.vot (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.FW.vot (revision 36826) @@ -0,0 +1,355 @@ + + + + + + VOTable description of PSPS table ForcedWarpMeta + + + forced warp identifier identifier + + + batch identifier + + + survey flag identifier + + + filter identifier + + + single float giving PrimaryCell.SkyCell identifier + + + photometry code + + + saturation magnitude level + + + IPP software analysis release + + + exposure time of positive image + + + 95% completion level in mag + + + astrometric scatter for warp + + + photometric scatter for warp + + + number of astrometric reference sources + + + number of photometric reference sources + + + PSF model identifier + + + mean PSF FWHM in the stack + + + maximum PSF FWHM in the stack + + + local derived photometric zero point + + + local derived photometric color term + + + name of astrometric projection in RA + + + name of astrometric projection in DEC + + + RA corresponding to reference pixel + + + DEC corresponding to reference pixel + + + reference pixel value for RA + + + reference pixel value for DEC + + + scale factor for RA + + + scale factor for DEC + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + calibration modification number + + + Data release + + + + +
+ + VOTable description of PSPS table ForcedWarpMeasurement + + + ODM object identifier + + + galactic coordinates ODM object identifier + + + ODM detection identifier + + + object id generated by IPP + + + detection id generated by IPP + + + filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... + + + survey flag identifier + + + forced warp measurement meta identifer + + + random warp id + + + single float givijng PrimaryCell.SkyCell identifier + + + identifier to dvo spacial region + + + Time of mid observation of positive detection in Mean Julian Days + + + zeropoint (only thing needed to convert adu/sec to AB + mag), note no zpErr + + + estimated extinction due to non-photometric conditions + + + exposure time (informational) + + + airmass (informational) + + + PSF instrumental flux (in analog-to-digital units/sec) or adu/sec + + + estimated error in flux + + + model PSF width in major axis + + + model PSF width in minor axis + + + model PSF orientation angles degrees East of North + + + model PSF core parameter [k in (1 + kz + z^1.66)] + + + PSF coverage factor + + + PSF-weighted fraction of pixels totally unmasked + + + reduced-chisq of the psf model fit. (ndof?, npix?) + + + PSF likelihood + + + moment XX + + + moment XY + + + moment YY + + + first radial moment + + + half radial moment + + + 3rd moment cos(theta) + + + 3rd moment sin(theta) + + + 4th moment cos(theta) + + + 4th moment sin(theta) + + + per-object smear polarizability X11 term + + + per-object smear polarizability X12 term + + + per-object smear polarizability X22 term + + + per-object smear polarizability E1 term + + + per-object smear polarizability E2 term + + + per-object shear polarizability X11 term + + + per-object shear polarizability X12 term + + + per-object shear polarizability X22 term + + + per-object shear polarizability E1 term + + + per-object shear polarizability E2 term + + + psf model smear polarizability X11 term + + + psf model smear polarizability X12 term + + + psf model smear polarizability X22 term + + + psf model smear polarizability E1 term + + + psf model smear polarizability E2 term + + + psf model shear polarizability X11 term + + + psf model shear polarizability X12 term + + + psf model shear polarizability X22 term + + + psf model shear polarizability E1 term + + + psf model shear polarizability E2 term + + + Aperture Flux + + + Aperture Flux error + + + Aperture fill factor + + + Kron flux + + + Kron flux error + + + Kron radius + + + Kron radius error + + + Flux inside r = 5 + + + estimated error in flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error in flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + PSF sky level at source + + + estimated error in sky + + + flag indicating provenance information + + + Data release when this detection was originally taken. Recalibrations do not affect this value. + + + + +
+ + VOTable description of PSPS table ForcedWarpToImage + + + forced warp identifier + + + hashed exposure-ccdID identifier + + + + +
+ + Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.IN.full.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.IN.full.vot (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.IN.full.vot (revision 36826) @@ -49784,14 +49784,14 @@ - elements of rotation/Dcale matrix + elements of rotation/scale matrix - elements of rotation/Dcale matrix + elements of rotation/scale matrix - elements of rotation/Dcale matrix + elements of rotation/scale matrix - elements of rotation/Dcale matrix + elements of rotation/scale matrix Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.OB.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.OB.vot (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.OB.vot (revision 36826) @@ -3,303 +3,379 @@ - - VOTable description of PSPS table Object +
+ VOTable description of PSPS table ObjectThin - + + IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version + + ODM object identifier index - + + galactic coordinates ODM object identifier + + IPP object number - - projection cell identifier at discovery time - - + + Survey ID + + + single float giving PrimaryCell.SkyCell identifier + + HTM index - + Zone index - - static stack version number - - + + Random integer between 0 and 2^32 - 4,394,967,296 + + + batch id + + + identifier of dvo spatial region -dvo cat id + + + Data release version of this table + + flag indicating odd properties - + subset of objInfoFlag - flags related to goodness of object - - Survey ID - - - Right Ascension - - - Declination - - - Right Ascension error - - - Declination error - - - Cartesian x + + Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation + + + Right Ascension - best weighted RA across filters from stack WHENEVER an object is detected in the stack + + + Declination - best weighted DEC across filters from stack WHENEVER an object is detected in the stack + + + Error in stack Right Ascension, note will have a systematic floor, and is not necessarily the best ra,dec for all objects + + + Error in stack Declination, note will have a systematic floor, and is not necessarily the best ra,dec for all objects + + + Right Ascension from the Mean of RA from individual detections if it exists. This is for information only + + + Declination from the Mean of Dec from individual detections if it exists. This is for information only + + + Combination of the measurement errors in the Right Ascension from the mean detections if they exist + + + Combination of the measurement errors in the Declination measurements if they exist + + + standard deviation of Right Ascension measuments in detections if they exist + + + standard deviation of Declination measurements in detections if they exist + + + Cartesian x from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't - - Cartesian y + + Cartesian y from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't - - Cartesian z + + Cartesian z from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't - - ecliptic longitude - - - ecliptic latitude - - - galactic longitude - - - galactic latitude - - + + ecliptic longitude from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't + + + ecliptic latitude from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't + + + galactic longitude from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't + + + galactic latitude from stack Ra, Dec WENEVER a stack detection exists, and Mean Ra, Dec (mean of one?) when it doesn't + + + number of independent StackObjectRows this object has. 0 if not detected in the stack, 1 if there is a primary, 2 or more if it is in an overlap region + + + which filter bands it is detected in (not forced) or if it was masked in a band. + + total number of detection measurements in all filters - + number of measures in g filter - - mean g psf magnitude - - - estimated error of g psf magnitude mean - - - mean g kron mag - - - estimated error of g kron mean mag - - - 20 pct g magnitude - - - 80 pct g magnitude - - + number of measures in r filter - - mean r psf magnitude - - - estimated error of r psf magnitude mean - - - mean r kron mag - - - estimated error of r mean kron mag - - - 20 pct r magnitude - - - 80 pct r magnitude - - + number of measures in i filter - - mean psf i magnitude - - - estimated error of i psf magitude mean - - - mean i kron mag - - - estimated error of i kron mag mean - - - 20 pct i magniutde - - - 80 pct i magnitude - - - number of measures in z filter - - - mean z psf magnitude - - - estimated error of z psf magnitude mean - - - mean z kron mag - - - estimated error of z mean kron mag - - - 20 pct z magnitude - - - 80 pct z magnitude - - + + number of measures in z filter + + number of measures in y filter - - mean y psf magnitude - - - estimated error of y psf magnitude mean - - - mean y kron mag - - - estimated error of y kron mag mean - - - 20 pct y magnitude - - - 80 pct y magnitude - - - number of measures in the wide (w) solar system filter - - - mean w psf magnitude - - - estimated error of w psf magnitude mean - - - mean w kron mag - - - estimated error of w kron mag mean - - - 20 pct w magnitude - - - 80 pct w magnitude - - - g stack psf flux - - - estimated error in g stack psf flux - - - g kron magnitude from stack frames - - - error from g stack kron mag - - - flags for g stack - - - ODM detection identifier - - - r stack psf flux - - - estimated error in r stack psf flux - - - r kron magnitude from stack frames - - - error from r stack kron mag - - - flags for r stack - - - ODM detection identifier - - - i stack psf flux - - - estimated error in i stack psf flux - - - i kron magnitude from stack frames - - - error from i stack kron mag - - - flags for i stack - - - ODM detection identifier - - - z stack psf flux - - - estimated error in z stack psf flux - - - z kron magnitude from stack frames - - - error from z stack kron mag - - - flags for z stack - - - ODM detection identifier - - - y band stack PSF flux - - - y band stack psf flux err - - - y kron magnitude from stack frames - - - y band stack kron mag err - - - flags for y stack - - - ODM detection identifier - - - qf perfect - - - star/galaxy separator - true if any one color is - - - Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation - - - Random number between 0 and 1 - - - Data release - - - batch id + + + +
+ + VOTable description of PSPS table MeanObject + + + ODM object identifier index + + + galactic coordinates ODM object identifier + + + maximum psf QfPerfect from best available g band detections + + + mean g psf AB magnitude from best available measurements (best available as currently defined by ipp "waterfall logic" + + + estimated error of best available g-band psf AB magnitudes + + + number of best available detections included in mean psfMag + + + standard deviation of best available measurements - compared with MeanErr this is one measure of true variability + + + minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability + + + maxiumum best available PSFMag + + + mean g kron AB magnitude from best available measurements + + + estimated error of mean g kron AB magnitude + + + number of best available detections included in mean Kron Mag + + + standard deviation of best available measurements + + + mean g aperture AB magnitude from best available measurements + + + estimated error of mean g aperture AB magnitude + + + number of best available detections included in mean Ap Mag + + + standard deviation of best available measurements + + + flags for g mean + + + maximum psf QfPerfect from best available r band detections + + + mean r psf AB magnitude from best available measurements (best available as currently defined by ipp "waterfall logic" + + + estimated error of best available r-band psf AB magnitudes + + + number of best available detections included in mean psfMag + + + standard deviation of best available measurements - compared with MeanErr this is one measure of true variability + + + minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability + + + maxiumum best available PSFMag + + + mean r kron AB magnitude from best available measurements + + + estimated error of mean r kron AB magnitude + + + number of best available detections included in mean Kron Mag + + + standard deviation of best available measurements + + + mean r aperture AB magnitude from best available measurements + + + estimated error of mean r aperture AB magnitude + + + number of best available detections included in mean Ap Mag + + + standard deviation of best available measurements + + + flags for r mean + + + maximum psf QfPerfect from best available i band detections + + + mean i psf AB magnitude from best available measurements (best available as currently defined by ipp "waterfall logic" + + + estimated error of best available i-band psf AB magnitudes + + + number of best available detections included in mean psfMag + + + standard deviation of best available measurements - compared with MeanErr this is one measure of true variability + + + minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability + + + maxiumum best available PSFMag + + + mean i kron AB magnitude from best available measurements + + + estimated error of mean i kron AB magnitude + + + number of best available detections included in mean Kron Mag + + + standard deviation of best available measurements + + + mean i aperture AB magnitude from best available measurements + + + estimated error of mean i aperture AB magnitude + + + number of best available detections included in mean Ap Mag + + + standard deviation of best available measurements + + + flags for i mean + + + maximum psf QfPerfect from best available z band detections + + + mean z psf AB magnitude from best available measurements (best available as currently defined by ipp "waterfall logic" + + + estimated error of best available z-band psf AB magnitudes + + + number of best available detections included in mean psfMag + + + standard deviation of best available measurements - compared with MeanErr this is one measure of true variability + + + minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability + + + maxiumum best available PSFMag + + + mean z kron AB magnitude from best available measurements + + + estimated error of mean z kron AB magnitude + + + number of best available detections included in mean Kron Mag + + + standard deviation of best available measurements + + + mean z aperture AB magnitude from best available measurements + + + estimated error of mean z aperture AB magnitude + + + number of best available detections included in mean Ap Mag + + + standard deviation of best available measurements + + + flags for z mean + + + maximum psf QfPerfect from best available y band detections + + + mean y psf AB magnitude from best available measurements (best available as currently defined by ipp "waterfall logic" + + + estimated error of best available y-band psf AB magnitudes + + + number of best available detections included in mean psfMag + + + standard deviation of best available measurements - compared with MeanErr this is one measure of true variability + + + minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability + + + maxiumum best available PSFMag + + + mean y kron AB magnitude from best available measurements + + + estimated error of mean y kron AB magnitude + + + number of best available detections included in mean Kron Mag + + + standard deviation of best available measurements + + + mean y aperture AB magnitude from best available measurements + + + estimated error of mean y aperture AB magnitude + + + number of best available detections included in mean Ap Mag + + + standard deviation of best available measurements + + + flags for y mean Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.P2.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.P2.vot (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.P2.vot (revision 36826) @@ -6,149 +6,149 @@ VOTable description of PSPS table FrameMeta - + unique exposure/frame identifier. - + frame name provided by camera software - + survey flag identifier - + filter identifier - + camera identifier - + camera configuration identifier - + telescope identifier - + IPP software analysis release - + IPP phase 1 MD5 Checksum - + IPP phase 2 MD5 Checksum - + IPP phase 3 MD5 Checksum - + number of valid OTA/CCD images in this exposure - + global photometric scatter - + number of photometric reference sources - + exposure start time in MJD - + exposure time - + airmass at mid-exposure - + RA of telescope boresight - + DEC of telescope boresight - + name of astrometric projection in RA - + name of astrometric projection in DEC - + RA corresponding to reference pixel - + DEC corresponding to reference pixel - + reference pixel value for RA - + reference pixel value for DEC - + scale factor for RA - + scale factor for DEC - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + polynomial order of astrometry fit - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + calibration modification number - + batchID - + Data release @@ -160,206 +160,224 @@ VOTable description of PSPS table ImageMeta - + unique ID for each image, hashed from frameID and ccdID (ALEX) - + frame identifier, also known as exposure number - + ccd cell identifier in the focal plane - + photometry reduction code identifier - + filter ID - + detector bias level - + scatter in bias level - + mean sky level - + scatter in mean sky level - + number of detections on CCD - + saturation magnitude level - + 95% completion level in mag - + astrometric scatter for chip - + photometric scatter for chip - + number of astrometric reference sources - + number of photometric reference sources - + chip dimension in x - + chip dimension in y - + PSF model identifier - + model psf full width at half maximum at chip center - + model PSF parameters at chip center - + model PSF parameters at chip center - + model PSF parameters at chip center - - average PSF full width at half maximum from moments - - - average PSF parameters from moments - - - average PSF parameters from moments - - + + moment XX + + + moment XY + + + moment YY + + + moment M3C + + + moment M3S + + + moment M4C + + + moment M4S + + + moment R1 + + + moment RH + + corrected aperture residual - + scatter of aperture corrections - + identifier for actual CCD chip - + Q/A flags for this OTA - + identifier of detrend image 1 - + identifier of detrend image 2 - + identifier of detrend image 3 - + identifier of detrend image 4 - + identifier of detrend image 5 - + identifier of detrend image 6 - + identifier of detrend image 7 - + identifier of detrend image 8 - + local derived photometric zero point - + name of astrometric projection in RA - + name of astrometric projection in DEC - + RA corresponding to reference pixel - + DEC corresponding to reference pixel - + reference pixel value for RA - + reference pixel value for DEC - + scale factor for RA - + scale factor for DEC - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + polynomial order of astrometry fit - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - - polynomial coefficients for the astrometric fit - - + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + + polynomial coefficients for the astrometric fit + + calibration modification number - + Data release @@ -371,194 +389,168 @@ VOTable description of PSPS table Detection - + ODM object identifier - + + galactic coordinates ODM object identifier + + ODM detection identifier - + object id generated by IPP - + detection id generated by IPP - + filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... - + survey flag identifier - + image id hash of frame identifier & OTA/CCD number - - Time of mid observation - - + + random detection Id + + + identifier to dvo spacial region + + + Time of mid observation in Mean Julian Days + + measured x on CCD from PSF fit - + measured y on CCD from PSF fit - + estimated error in x - + estimated error in y - - PSF instrumental flux - - + + plate scale at this location (from DVO) + + + position angle (sky-to-chip) at this location (from DVO) + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zeropoint + + + estimated extinction due to non-photometric conditions + + + exposure time (informational) + + + airmass (informational) + + + PSF instrumental flux (in analog-to-digital units/sec) or adu/sec + + estimated error in flux - - peak count in source pixel - - + model PSF width in major axis - + model PSF width in minor axis - - model PSF orientation angle - - + + model PSF orientation angle degrees East of North + + + model PSF core parameter [k in 1 + kz + z^1.66)] + + + PSF coverage factor + + + PSF-weighted fraction of pixels totally unmasked + + + reduced-chisq of the psf model fit. (ndof?, npix?) + + PSF likelihood - - PSF qf - - + moment XX - + moment XY - - momeny YY - - + + moment YY + + + first radial moment + + + half radial moment + + + 3rd moment cos(theta) + + + 3rd moment sin(theta) + + + 4th moment cos(theta) + + + 4th moment sin(theta) + + Aperture Flux - + Aperture Flux error - + + Aperture fill factor + + Kron flux - + Kron flux error - + Kron radius - + Kron radius error - - psf QF Perfect - - - PSF Chi sq - - + + PSF sky level at source + + + estimated error in sky + + flag indicating provenance information - - PSF sky level at source - - - estimated error in sky - - - star/galaxy separator - - - indicates whether this detection/orphan is still a detection/orphan - - - date object association assigned - - - modification number in the O-D association history - - + Data release when this detection was originally taken. Recalibrations do not affect this value. - - - - -
- - VOTable description of PSPS table SkinnyObject - - - ODM object identifier index - - - IPP object number - - - projection cell identifier at discovery time - - - Data release to propagate to the object - - - surveyID </coluumn> - - - - -
- - VOTable description of PSPS table DetectionCalib - - - ODM object identifier index - - - ODM detection identifier index - - - IPP object number - - - detection ID generated by IPP - - - filter identifier - - - survey identifier - - - right ascension - - - declination - - - right ascension error - - - declination error - - - zero point offset - - - zerop point offset error - - - exposure time - - - airmass - - - Data release to propagate to the object Index: /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.ST.vot =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.ST.vot (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/config/tables.ST.vot (revision 36826) @@ -5,108 +5,108 @@
VOTable description of PSPS table StackMeta - - + + stack identifier - + + batchID + + + survey flag identifier + + + filter identifier + + + stack type identifier + + skycell region identifier - - survey flag identifier - - + photometry code numerical id - - filter identifier - - - stack type identifier - - + saturation magnitude level - + analysis version index - + exposure time (unit = s) - + number of P2 images contributing to this cell - + 95% completion level in mag - + astrometric scatter for chip - + photometric scatter for chip - + number of astrometric reference sources - + number of photometric reference sources - + PSF model identifier - + mean PSF FWHM in the stack - + maximum PSF FWHM in the stack - + local derived photometric zero point - + local derived photometric color term - + name of astrometric projection in RA - + name of astrometric projection in DEC - + RA corresponding to reference pixel - + DEC corresponding to reference pixel - + reference pixel value for RA - + reference pixel value for DEC - + scale factor for RA - + scale factor for DEC - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - - elements of rotation/Dcale matrix - - + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + + elements of rotation/scale matrix + + calibration modification number - - batchID - - + Data release @@ -115,151 +115,712 @@
- - VOTable description of PSPS table StackDetection +
+ VOTable description of PSPS table StackObject - + ODM object identifier - + + galactic coordinates ODM object identifier + + + survey flag identifier + + + single float giving PrimaryCell.SkyCell identifier + + + random stack id + + + identifies which of up to 5 possible stackdetections this row refers to: primary, up to 3 Secondary, 1 best + + + identifies if this row is a primary stack detection. True about 80 percent of time + + + identifies if this row is a best detection or not. Also set if only a primary detection exists. + + + identifier to dvo spatial region + + + Data release when this table was loaded. + + + IPP object identifier + + + ipp identifier + + ODM detection identifier - + + stack identifier + + + measured x on skycell from PSF fit + + + measured y on skycell from PSF fit + + + estimated error in x + + + estimated error in y + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + exposure time (informational) + + + PSF width in major axis + + + PSF width in minor axis + + + PSF orientation angle + + + PSF core parameter [k in (1 + kz * z^1.66)] + + + PSF likelihood + + + PSF coverage factor + + + PSF-weighted fraction of arcsec totally unmasked + + + PSF-weighted fraction of arcsec to. + + + moment XX + + + moment XY + + + moment YY + + + moment R1 + + + moment RH + + + PSF instrumental flux + + + estimated error in instrumental flux + + + Aperture flux + + + Aperture flux error + + + Aperture fill factor - redundant with psfQF, keep? + + + Kron flux + + + Kron flux error + + + Kron radius + + + g psf AB magnitude from stack + + + estimated error of g psf AB magnitude from stack + + + g Aperture AB magnitude from stack + + + estimated error of g Ap AB magnitude from stack + + + g kron AB magnitude from stack + + + estimated error of g kron AB magnitude from stack + + + PSF sky level at source (adu) + + + estimated error in sky + + + indicator of strange propeties + + + number of frames contributing to source + + IPP object identifier - - detection ID generated by IPP - - - filter identifier - - - stack type identifier - - - survey flag identifier - - - identifies best stack detection for Stacks overlapping the same region of the sky. - - + + ipp identifier + + + ODM detection identifier + + stack identifier - - skycell identifier - - - projection cell identifier - - - measured x on CCD from PSF fit - - - measured y on CCD from PSF fit - - + + measured x on skycell from PSF fit + + + measured y on skycell from PSF fit + + estimated error in x - + estimated error in y - + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + exposure time (informational) + + + PSF width in major axis + + + PSF width in minor axis + + + PSF orientation angle + + + PSF core parameter [k in (1 + kz * z^1.66)] + + + PSF likelihood + + + PSF coverage factor + + + PSF-weighted fraction of arcsec totally unmasked + + + PSF-weighted fraction of arcsec to. + + + moment XX + + + moment XY + + + moment YY + + + moment R1 + + + moment RH + + PSF instrumental flux - + estimated error in instrumental flux - - ratio of peak flux to total flux - - + + Aperture flux + + + Aperture flux error + + + Aperture fill factor - redundant with psfQF, keep? + + + Kron flux + + + Kron flux error + + + Kron radius + + + psf AB mag from stack + + + estimated error of psf mag from stack + + + aperture AB mag from stack + + + estimated error of aperture mag from stack + + + kron AB mag from stack + + + estimated error of kron mag from stack + + PSF sky level at source (adu) - + estimated error in sky - - star/galaxy separator - - + + indicator of strange propeties + + + number of frames contributing to source + + + IPP object identifier + + + ipp identifier + + + ODM detection identifier + + + stack identifier + + + measured x on skycell from PSF fit + + + measured y on skycell from PSF fit + + + estimated error in x + + + estimated error in y + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + exposure time (informational) + + PSF width in major axis - + PSF width in minor axis - + PSF orientation angle - + + PSF core parameter [k in (1 + kz * z^1.66)] + + PSF likelihood - - PSF Q factor - - - PSF QF Perfect + + PSF coverage factor + + + PSF-weighted fraction of arcsec totally unmasked - - psf chi sq - - - moment XX (unit = arcsec) - - - moment XY (unit = arcsec) - - - moment YY (unit = arcsec) - - - moment M3C (unit = arcsec) - - - moment M3S (unit = arcsec) - - - moment M4C (unit = arcsec) - - - moment M4S (unit = arcsec) - - - moment R1 (unit = arcsec) - - - moment RH (unit = arcsec) - - - Aperture flux (unit = adu/s) - - - Aperture flux error (unit = adu/s) - - - Kron flux (unit = adu/s) - - - Kron flux error (unit = adu/s) - - - Kron radius (unit = arcsecs) - - + + PSF-weighted fraction of arcsec to. + + + moment XX + + + moment XY + + + moment YY + + + moment R1 + + + moment RH + + + PSF instrumental flux + + + estimated error in instrumental flux + + + Aperture flux + + + Aperture flux error + + + Aperture fill factor - redundant with psfQF, keep? + + + Kron flux + + + Kron flux error + + + Kron radius + + + psf AB mag from stack + + + estimated error of psf mag from stack + + + aperture AB mag from stack + + + estimated error of aperture mag from stack + + + kron AB mag from stack + + + estimated error of kron mag from stack + + + PSF sky level at source (adu) + + + estimated error in sky + + indicator of strange propeties - + number of frames contributing to source - - indicates whether this detection/orphan is still a detection/orphan - - - date object association assigned - - - modification number in the O-D association history - - - Data release when this detection was originally taken. - + + IPP object identifier + + + ipp identifier + + + ODM detection identifier + + + stack identifier + + + measured x on skycell from PSF fit + + + measured y on skycell from PSF fit + + + estimated error in x + + + estimated error in y + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + exposure time (informational) + + + PSF width in major axis + + + PSF width in minor axis + + + PSF orientation angle + + + PSF core parameter [k in (1 + kz * z^1.66)] + + + PSF likelihood + + + PSF coverage factor + + + PSF-weighted fraction of arcsec totally unmasked + + + PSF-weighted fraction of arcsec to. + + + moment XX + + + moment XY + + + moment YY + + + moment R1 + + + moment RH + + + PSF instrumental flux + + + estimated error in instrumental flux + + + Aperture flux + + + Aperture flux error + + + Aperture fill factor - redundant with psfQF, keep? + + + Kron flux + + + Kron flux error + + + Kron radius + + + psf AB mag from stack + + + estimated error of psf mag from stack + + + aperture AB mag from stack + + + estimated error of aperture mag from stack + + + kron AB mag from stack + + + estimated error of kron mag from stack + + + PSF sky level at source (adu) + + + estimated error in sky + + + indicator of strange propeties + + + number of frames contributing to source + + + IPP object identifier + + + ipp identifier + + + ODM detection identifier + + + stack identifier + + + measured x on skycell from PSF fit + + + measured y on skycell from PSF fit + + + estimated error in x + + + estimated error in y + + + right ascension + + + declination + + + right ascension error + + + declination error + + + zero point offset + + + exposure time (informational) + + + PSF width in major axis + + + PSF width in minor axis + + + PSF orientation angle + + + PSF core parameter [k in (1 + kz * z^1.66)] + + + PSF likelihood + + + PSF coverage factor + + + PSF-weighted fraction of arcsec totally unmasked + + + PSF-weighted fraction of arcsec to. + + + moment XX + + + moment XY + + + moment YY + + + moment R1 + + + moment RH + + + PSF instrumental flux + + + estimated error in instrumental flux + + + Aperture flux + + + Aperture flux error + + + Aperture fill factor - redundant with psfQF, keep? + + + Kron flux + + + Kron flux error + + + Kron radius + + + psf AB mag from stack + + + estimated error of psf mag from stack + + + aperture AB mag from stack + + + estimated error of aperture mag from stack + + + kron AB mag from stack + + + estimated error of kron mag from stack + + + PSF sky level at source (adu) + + + estimated error in sky + + + indicator of strange propeties + + + number of frames contributing to source + @@ -269,462 +830,633 @@ VOTable description of PSPS table StackApFlx - + ODM object identifier - + + galactic coordinates ODM object identifier + + + survey flag identifier + + + single float giving PrimaryCell.SkyCell identifier + + + random stack id + + + identifies which of up to 5 possible stackdetections thisrow refers to: primary, up to 3 Secondary, 1 best + + + identifies if this row is a primary stack detection. True about 80 percent of time + + + identifies if this row is a best detection or not. Also set if only a primary detection exists. + + + identifier to dvo spatial region + + + Data release when this table was loaded. + + ODM detection identifier - + + stack identifier + + IPP object identifier - - detection ID generated by IPP - - - filter identifier - - - stack type identifier - - - survey flag identifier - - - identifies best stack detection for Stacks overlapping the same region of the sky. - - + + ipp identifier + + + Flux inside r = 3 + + + estimated error in flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error in flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is gflxR5 + + + estimated standard deviation in gflxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error in c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error in c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error in c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + ODM detection identifier + + stack identifier - - isophotal flux - - - estimated error in isophotal flux - - - isophotal Major Axis - - - estimated error in isophotal Major Axis - - - isophotal Minor Axis - - - estimated error in isophotal Minor Axis - - - isophotal major axis gradient - - - isophotal minor axis gradient - - - isophotal position angle - - - estimated error in isophotal position angle - - - isophotal position angle gradient - - - Petrosian radius - - - estimated error inPetrosian radius - - - Petrosian Flux - - - estimated error in petFlux - - - Petrosian radius at 50% light - - - estimated error inPetrosian radius at 50% light - - - Petrosian radius at 90% light - - - estimated error in Petrosian radius at 90% light - - - Petrosian fit coverage factor - - - Flux inside r = 1 - - - estimated error is flxR1 - - - estimated standard deviation in flxR1 - - - estimated filling factor at R1 - - - Flux inside r = 2 - - - estimated error is flxR2 - - - estimated standard deviation in flxR2 - - - estimated filling factor at R1 - - + + IPP object identifier + + + ipp identifier + + Flux inside r = 3 - - estimated error is flxR3 - - + + estimated error in flxR3 + + estimated standard deviation in flxR3 - + estimated filling factor at R3 - + Flux inside r = 4 - - estimated error is flxR4 - - + + estimated error in flxR4 + + estimated standard deviation in flxR4 - + estimated filling factor at R4 - + Flux inside r = 5 - - estimated error is flxR5 - - - estimated standard deviation in flxR5 - - + + estimated error is gflxR5 + + + estimated standard deviation in gflxR5 + + estimated filling factor at R5 - - Flux inside r = 6 - - - estimated error is flxR6 - - - estimated standard deviation in flxR6 - - - estimated filling factor at R6 - - - Flux inside r = 7 - - - estimated error is flxR7 - - - estimated standard deviation in flxR7 - - - estimated filling factor at R7 - - - Flux inside r = 8 - - - estimated error is flxR8 - - - estimated standard deviation in flxR8 - - - estimated filling factor at R8 - - - Flux inside r = 9 - - - estimated error is flxR9 - - - estimated standard deviation in flxR9 - - - estimated filling factor at R9 - - - Flux inside r = 10 - - - estimated error is flxR10 - - - estimated standard deviation in flxR10 - - - estimated filling factor at R10 - - - Flux inside r = 1 - - - estimated error is c1flxR1 - - - estimated standard deviation in c1flxR1 - - - estimated filling factor at R1 - - - Flux inside r = 2 - - - estimated error is c1flxR2 - - - estimated standard deviation in c1flxR2 - - - estimated filling factor at R1 - - + Flux inside r = 3 - + estimated error is c1flxR3 - + estimated standard deviation in c1flxR3 - + estimated filling factor at R3 - + Flux inside r = 4 - + estimated error is c1flxR4 - + estimated standard deviation in c1flxR4 - + estimated filling factor at R4 - + Flux inside r = 5 - + estimated error is c1flxR5 - + estimated standard deviation in c1flxR5 - + estimated filling factor at R5 - - Flux inside r = 6 - - - estimated error is c1flxR6 - - - estimated standard deviation in c1flxR6 - - - estimated filling factor at R6 - - - Flux inside r = 7 - - - estimated error is c1flxR7 - - - estimated standard deviation in c1flxR7 - - - estimated filling factor at R7 - - - Flux inside r = 8 - - - estimated error is c1flxR8 - - - estimated standard deviation in c1flxR8 - - - estimated filling factor at R8 - - - Flux inside r = 9 - - - estimated error is c1flxR9 - - - estimated standard deviation in c1flxR9 - - - estimated filling factor at R9 - - - Flux inside r = 10 - - - estimated error is c1flxR10 - - - estimated standard deviation in c1flxR10 - - - estimated filling factor at R10 - - - Flux inside r = 1 - - - estimated error is c2flxR1 - - - estimated standard deviation in c2flxR1 - - - estimated filling factor at R1 - - - Flux inside r = 2 - - - estimated error is c2flxR2 - - - estimated standard deviation in c2flxR2 - - - estimated filling factor at R1 - - + Flux inside r = 3 - + estimated error is c2flxR3 - + estimated standard deviation in c2flxR3 - + estimated filling factor at R3 - + Flux inside r = 4 - + estimated error is c2flxR4 - + estimated standard deviation in c2flxR4 - + estimated filling factor at R4 - + Flux inside r = 5 - + estimated error is c2flxR5 - + estimated standard deviation in c2flxR5 - + estimated filling factor at R5 - - Flux inside r = 6 - - - estimated error is c2flxR6 - - - estimated standard deviation in c2flxR6 - - - estimated filling factor at R6 - - - Flux inside r = 7 - - - estimated error is c2flxR7 - - - estimated standard deviation in c2flxR7 - - - estimated filling factor at R7 - - - Flux inside r = 8 - - - estimated error is c2flxR8 - - - estimated standard deviation in c2flxR8 - - - estimated filling factor at R8 - - - Flux inside r = 9 - - - estimated error is c2flxR9 - - - estimated standard deviation in c2flxR9 - - - estimated filling factor at R9 - - - Flux inside r = 10 - - - estimated error is c2flxR10 - - - estimated standard deviation in c2flxR10 - - - estimated filling factor at R10 - - - Abraham concentration index - - - Abraham asymmetry index - - - indicates whether this detection/orphan is still a detection/orphan - - - Data release when this detection was taken + + ODM detection identifier + + + stack identifier + + + IPP object identifier + + + ipp identifier + + + Flux inside r = 3 + + + estimated error in flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error in flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is gflxR5 + + + estimated standard deviation in gflxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + ODM detection identifier + + + stack identifier + + + IPP object identifier + + + ipp identifier + + + Flux inside r = 3 + + + estimated error in flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error in flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is gflxR5 + + + estimated standard deviation in gflxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + ODM detection identifier + + + stack identifier + + + IPP object identifier + + + ipp identifier + + + Flux inside r = 3 + + + estimated error in flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error in flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is gflxR5 + + + estimated standard deviation in gflxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 @@ -735,447 +1467,2736 @@ VOTable description of PSPS table StackModelFit - + ODM object identifier - + + galactic coordinates ODM object identifier + + + survey flag identifier + + + single float giving PrimaryCell.SkyCell identifier + + + random stack id + + + identifies which of up to 5 possible stackdetections thisrow refers to: primary, up to 3 Secondary, 1 best + + + identifies if this row is a primary stack detection. True about 80 percent of time + + + identifies if this row is a best detection or not. Also set if only a primary detection exists. + + + identifier to dvo spatial region + + + Data release when this table was loaded. + + + concentration index + + + asymmetry index + + + clumpiness factor + + ODM detection identifier - + + stack identifier + + IPP object identifier - - detection ID generated by IPP - - - filter identifier - - - stack type identifier - - - survey flag identifier - - - identifies best stack detection for Stacks overlapping the same region of the sky. - - + + ipp detect identifier + + + deVaucouleurs radius + + + estimated error in deVaucouleurs radius + + + deVaucouleurs magntiude + + + estimated error in deV_mag + + + deVaucoulerus axis ratio + + + estimated error in deVaucoulerus axis ratio + + + estmated phi of deVaucouleurs axis + + + estmated error of phi of deVaucouleurs axis + + + Offset in RA of deVaucouleurs fit from PSF RA + + + Offset in DEC of deVaucouleurs fit from PSF DEC + + + estimated error in ra offset + + + estimated error in dec offset + + + deVaucouleurs fit coverage factor + + + deVaucouleurs fit likelihood factor + + + Exponential fit radius + + + estimated error in Exponential fit radius + + + Exponential fit flux + + + estimated error in expFlux + + + Exponential fit axis ratio + + + estimated error in Exponential fit axis ratio + + + estmated phi of Exponential axis + + + estmated error of phi of Exponential axis + + + Offset in RA of Exponential fit from PSF RA + + + Offset in DEC of Exponential fit from PSF DEC + + + estimated error in raExpOff + + + estimated error in decExpOff + + + Exponential fit coverage factor + + + Exponential fit likelihood factor + + + Sersic radius + + + estimated error in Sersic radius + + + Sersic Flux + + + estimated error in serFlux + + + Sersic axis ratio + + + estimated error in Sersic axis ratio + + + Sersic index + + + estimated error in Sersic index + + + estmated phi of Sersic axis + + + estmated error of phi of Sersic axis + + + Offset in RA of Sersic fit from PSF RA + + + Offset in DEC of Sersic fit from PSF DEC + + + estimated error in raSerOff + + + estimated error in decSerOff + + + Sersic fit coverage factor + + + Sersic fit likelihood factor + + + Petrosian radius + + + estimated error inPetrosian radius + + + Petrosian Flux + + + estimated error in petFlux + + + Petrosian radius at 50% light + + + estimated error inPetrosian radius at 50% light + + + Petrosian radius at 90% light + + + estimated error in Petrosian radius at 90% light + + + Petrosian fit coverage factor + + + Flux inside r = 1 + + + estimated error is flxR1 + + + estimated standard deviation in flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is flxR2 + + + estimated standard deviation in flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is flxR7 + + + estimated standard deviation in flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is flxR8 + + + estimated standard deviation in flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is flxR9 + + + estimated standard deviation in flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is flxR10 + + + estimated standard deviation in flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c1flxR1 + + + estimated standard deviation in c1flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c1flxR2 + + + estimated standard deviation in c1flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c1flxR6 + + + estimated standard deviation in c1flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c1flxR7 + + + estimated standard deviation in c1flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c1flxR8 + + + estimated standard deviation in c1flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c1flxR9 + + + estimated standard deviation in c1flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c1flxR10 + + + estimated standard deviation in c1flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c2flxR1 + + + estimated standard deviation in c2flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c2flxR2 + + + estimated standard deviation in c2flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c2flxR6 + + + estimated standard deviation in c2flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c2flxR7 + + + estimated standard deviation in c2flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c2flxR8 + + + estimated standard deviation in c2flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c2flxR9 + + + estimated standard deviation in c2flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c2flxR10 + + + estimated standard deviation in c2flxR10 + + + estimated filling factor at R10 + + + concentration index + + + asymmetry index + + + clumpiness factor + + + ODM detection identifier + + stack identifier - + + IPP object identifier + + + ipp detect identifier + + deVaucouleurs radius - + estimated error in deVaucouleurs radius - + deVaucouleurs magntiude - + estimated error in deV_mag - + deVaucoulerus axis ratio - + estimated error in deVaucoulerus axis ratio - + estmated phi of deVaucouleurs axis - + estmated error of phi of deVaucouleurs axis - + Offset in RA of deVaucouleurs fit from PSF RA - + Offset in DEC of deVaucouleurs fit from PSF DEC - + estimated error in ra offset - + estimated error in dec offset - + deVaucouleurs fit coverage factor - + deVaucouleurs fit likelihood factor - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - - covariance for deVaucouleurs fit - - + Exponential fit radius - + estimated error in Exponential fit radius - + Exponential fit flux - + estimated error in expFlux - + Exponential fit axis ratio - + estimated error in Exponential fit axis ratio - + estmated phi of Exponential axis - + estmated error of phi of Exponential axis - + Offset in RA of Exponential fit from PSF RA - + Offset in DEC of Exponential fit from PSF DEC - + estimated error in raExpOff - + estimated error in decExpOff - + Exponential fit coverage factor - + Exponential fit likelihood factor - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - - covariance for Exponential fit - - + Sersic radius - + estimated error in Sersic radius - + Sersic Flux - + estimated error in serFlux - + Sersic axis ratio - + estimated error in Sersic axis ratio - + Sersic index - + estimated error in Sersic index - + estmated phi of Sersic axis - + estmated error of phi of Sersic axis - + Offset in RA of Sersic fit from PSF RA - + Offset in DEC of Sersic fit from PSF DEC - + estimated error in raSerOff - + estimated error in decSerOff - + Sersic fit coverage factor - + Sersic fit likelihood factor - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - covariance for Sersic fit - - - indicates whether this detection/orphan is still a detection/orphan - - - Data release when this detection was taken + + Petrosian radius + + + estimated error inPetrosian radius + + + Petrosian Flux + + + estimated error in petFlux + + + Petrosian radius at 50% light + + + estimated error inPetrosian radius at 50% light + + + Petrosian radius at 90% light + + + estimated error in Petrosian radius at 90% light + + + Petrosian fit coverage factor + + + Flux inside r = 1 + + + estimated error is flxR1 + + + estimated standard deviation in flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is flxR2 + + + estimated standard deviation in flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is flxR7 + + + estimated standard deviation in flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is flxR8 + + + estimated standard deviation in flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is flxR9 + + + estimated standard deviation in flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is flxR10 + + + estimated standard deviation in flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c1flxR1 + + + estimated standard deviation in c1flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c1flxR2 + + + estimated standard deviation in c1flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c1flxR6 + + + estimated standard deviation in c1flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c1flxR7 + + + estimated standard deviation in c1flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c1flxR8 + + + estimated standard deviation in c1flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c1flxR9 + + + estimated standard deviation in c1flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c1flxR10 + + + estimated standard deviation in c1flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c2flxR1 + + + estimated standard deviation in c2flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c2flxR2 + + + estimated standard deviation in c2flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c2flxR6 + + + estimated standard deviation in c2flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c2flxR7 + + + estimated standard deviation in c2flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c2flxR8 + + + estimated standard deviation in c2flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c2flxR9 + + + estimated standard deviation in c2flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c2flxR10 + + + estimated standard deviation in c2flxR10 + + + estimated filling factor at R10 + + + concentration index + + + asymmetry index + + + clumpiness factor + + + ODM detection identifier + + + stack identifier + + + IPP object identifier + + + ipp detect identifier + + + deVaucouleurs radius + + + estimated error in deVaucouleurs radius + + + deVaucouleurs magntiude + + + estimated error in deV_mag + + + deVaucoulerus axis ratio + + + estimated error in deVaucoulerus axis ratio + + + estmated phi of deVaucouleurs axis + + + estmated error of phi of deVaucouleurs axis + + + Offset in RA of deVaucouleurs fit from PSF RA + + + Offset in DEC of deVaucouleurs fit from PSF DEC + + + estimated error in ra offset + + + estimated error in dec offset + + + deVaucouleurs fit coverage factor + + + deVaucouleurs fit likelihood factor + + + Exponential fit radius + + + estimated error in Exponential fit radius + + + Exponential fit flux + + + estimated error in expFlux + + + Exponential fit axis ratio + + + estimated error in Exponential fit axis ratio + + + estmated phi of Exponential axis + + + estmated error of phi of Exponential axis + + + Offset in RA of Exponential fit from PSF RA + + + Offset in DEC of Exponential fit from PSF DEC + + + estimated error in raExpOff + + + estimated error in decExpOff + + + Exponential fit coverage factor + + + Exponential fit likelihood factor + + + Sersic radius + + + estimated error in Sersic radius + + + Sersic Flux + + + estimated error in serFlux + + + Sersic axis ratio + + + estimated error in Sersic axis ratio + + + Sersic index + + + estimated error in Sersic index + + + estmated phi of Sersic axis + + + estmated error of phi of Sersic axis + + + Offset in RA of Sersic fit from PSF RA + + + Offset in DEC of Sersic fit from PSF DEC + + + estimated error in raSerOff + + + estimated error in decSerOff + + + Sersic fit coverage factor + + + Sersic fit likelihood factor + + + Petrosian radius + + + estimated error inPetrosian radius + + + Petrosian Flux + + + estimated error in petFlux + + + Petrosian radius at 50% light + + + estimated error inPetrosian radius at 50% light + + + Petrosian radius at 90% light + + + estimated error in Petrosian radius at 90% light + + + Petrosian fit coverage factor + + + Flux inside r = 1 + + + estimated error is flxR1 + + + estimated standard deviation in flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is flxR2 + + + estimated standard deviation in flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is flxR7 + + + estimated standard deviation in flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is flxR8 + + + estimated standard deviation in flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is flxR9 + + + estimated standard deviation in flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is flxR10 + + + estimated standard deviation in flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c1flxR1 + + + estimated standard deviation in c1flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c1flxR2 + + + estimated standard deviation in c1flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c1flxR6 + + + estimated standard deviation in c1flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c1flxR7 + + + estimated standard deviation in c1flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c1flxR8 + + + estimated standard deviation in c1flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c1flxR9 + + + estimated standard deviation in c1flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c1flxR10 + + + estimated standard deviation in c1flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c2flxR1 + + + estimated standard deviation in c2flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c2flxR2 + + + estimated standard deviation in c2flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c2flxR6 + + + estimated standard deviation in c2flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c2flxR7 + + + estimated standard deviation in c2flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c2flxR8 + + + estimated standard deviation in c2flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c2flxR9 + + + estimated standard deviation in c2flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c2flxR10 + + + estimated standard deviation in c2flxR10 + + + estimated filling factor at R10 + + + concentration index + + + asymmetry index + + + clumpiness factor + + + ODM detection identifier + + + stack identifier + + + IPP object identifier + + + ipp detect identifier + + + deVaucouleurs radius + + + estimated error in deVaucouleurs radius + + + deVaucouleurs magntiude + + + estimated error in deV_mag + + + deVaucoulerus axis ratio + + + estimated error in deVaucoulerus axis ratio + + + estmated phi of deVaucouleurs axis + + + estmated error of phi of deVaucouleurs axis + + + Offset in RA of deVaucouleurs fit from PSF RA + + + Offset in DEC of deVaucouleurs fit from PSF DEC + + + estimated error in ra offset + + + estimated error in dec offset + + + deVaucouleurs fit coverage factor + + + deVaucouleurs fit likelihood factor + + + Exponential fit radius + + + estimated error in Exponential fit radius + + + Exponential fit flux + + + estimated error in expFlux + + + Exponential fit axis ratio + + + estimated error in Exponential fit axis ratio + + + estmated phi of Exponential axis + + + estmated error of phi of Exponential axis + + + Offset in RA of Exponential fit from PSF RA + + + Offset in DEC of Exponential fit from PSF DEC + + + estimated error in raExpOff + + + estimated error in decExpOff + + + Exponential fit coverage factor + + + Exponential fit likelihood factor + + + Sersic radius + + + estimated error in Sersic radius + + + Sersic Flux + + + estimated error in serFlux + + + Sersic axis ratio + + + estimated error in Sersic axis ratio + + + Sersic index + + + estimated error in Sersic index + + + estmated phi of Sersic axis + + + estmated error of phi of Sersic axis + + + Offset in RA of Sersic fit from PSF RA + + + Offset in DEC of Sersic fit from PSF DEC + + + estimated error in raSerOff + + + estimated error in decSerOff + + + Sersic fit coverage factor + + + Sersic fit likelihood factor + + + Petrosian radius + + + estimated error inPetrosian radius + + + Petrosian Flux + + + estimated error in petFlux + + + Petrosian radius at 50% light + + + estimated error inPetrosian radius at 50% light + + + Petrosian radius at 90% light + + + estimated error in Petrosian radius at 90% light + + + Petrosian fit coverage factor + + + Flux inside r = 1 + + + estimated error is flxR1 + + + estimated standard deviation in flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is flxR2 + + + estimated standard deviation in flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is flxR7 + + + estimated standard deviation in flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is flxR8 + + + estimated standard deviation in flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is flxR9 + + + estimated standard deviation in flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is flxR10 + + + estimated standard deviation in flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c1flxR1 + + + estimated standard deviation in c1flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c1flxR2 + + + estimated standard deviation in c1flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c1flxR6 + + + estimated standard deviation in c1flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c1flxR7 + + + estimated standard deviation in c1flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c1flxR8 + + + estimated standard deviation in c1flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c1flxR9 + + + estimated standard deviation in c1flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c1flxR10 + + + estimated standard deviation in c1flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c2flxR1 + + + estimated standard deviation in c2flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c2flxR2 + + + estimated standard deviation in c2flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c2flxR6 + + + estimated standard deviation in c2flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c2flxR7 + + + estimated standard deviation in c2flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c2flxR8 + + + estimated standard deviation in c2flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c2flxR9 + + + estimated standard deviation in c2flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c2flxR10 + + + estimated standard deviation in c2flxR10 + + + estimated filling factor at R10 + + + concentration index + + + asymmetry index + + + clumpiness factor + + + ODM detection identifier + + + stack identifier + + + IPP object identifier + + + ipp detect identifier + + + deVaucouleurs radius + + + estimated error in deVaucouleurs radius + + + deVaucouleurs magntiude + + + estimated error in deV_mag + + + deVaucoulerus axis ratio + + + estimated error in deVaucoulerus axis ratio + + + estmated phi of deVaucouleurs axis + + + estmated error of phi of deVaucouleurs axis + + + Offset in RA of deVaucouleurs fit from PSF RA + + + Offset in DEC of deVaucouleurs fit from PSF DEC + + + estimated error in ra offset + + + estimated error in dec offset + + + deVaucouleurs fit coverage factor + + + deVaucouleurs fit likelihood factor + + + Exponential fit radius + + + estimated error in Exponential fit radius + + + Exponential fit flux + + + estimated error in expFlux + + + Exponential fit axis ratio + + + estimated error in Exponential fit axis ratio + + + estmated phi of Exponential axis + + + estmated error of phi of Exponential axis + + + Offset in RA of Exponential fit from PSF RA + + + Offset in DEC of Exponential fit from PSF DEC + + + estimated error in raExpOff + + + estimated error in decExpOff + + + Exponential fit coverage factor + + + Exponential fit likelihood factor + + + Sersic radius + + + estimated error in Sersic radius + + + Sersic Flux + + + estimated error in serFlux + + + Sersic axis ratio + + + estimated error in Sersic axis ratio + + + Sersic index + + + estimated error in Sersic index + + + estmated phi of Sersic axis + + + estmated error of phi of Sersic axis + + + Offset in RA of Sersic fit from PSF RA + + + Offset in DEC of Sersic fit from PSF DEC + + + estimated error in raSerOff + + + estimated error in decSerOff + + + Sersic fit coverage factor + + + Sersic fit likelihood factor + + + Petrosian radius + + + estimated error inPetrosian radius + + + Petrosian Flux + + + estimated error in petFlux + + + Petrosian radius at 50% light + + + estimated error inPetrosian radius at 50% light + + + Petrosian radius at 90% light + + + estimated error in Petrosian radius at 90% light + + + Petrosian fit coverage factor + + + Flux inside r = 1 + + + estimated error is flxR1 + + + estimated standard deviation in flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is flxR2 + + + estimated standard deviation in flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is flxR3 + + + estimated standard deviation in flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is flxR4 + + + estimated standard deviation in flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is flxR5 + + + estimated standard deviation in flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is flxR6 + + + estimated standard deviation in flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is flxR7 + + + estimated standard deviation in flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is flxR8 + + + estimated standard deviation in flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is flxR9 + + + estimated standard deviation in flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is flxR10 + + + estimated standard deviation in flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c1flxR1 + + + estimated standard deviation in c1flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c1flxR2 + + + estimated standard deviation in c1flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c1flxR3 + + + estimated standard deviation in c1flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c1flxR4 + + + estimated standard deviation in c1flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c1flxR5 + + + estimated standard deviation in c1flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c1flxR6 + + + estimated standard deviation in c1flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c1flxR7 + + + estimated standard deviation in c1flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c1flxR8 + + + estimated standard deviation in c1flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c1flxR9 + + + estimated standard deviation in c1flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c1flxR10 + + + estimated standard deviation in c1flxR10 + + + estimated filling factor at R10 + + + Flux inside r = 1 + + + estimated error is c2flxR1 + + + estimated standard deviation in c2flxR1 + + + estimated filling factor at R1 + + + Flux inside r = 2 + + + estimated error is c2flxR2 + + + estimated standard deviation in c2flxR2 + + + estimated filling factor at R1 + + + Flux inside r = 3 + + + estimated error is c2flxR3 + + + estimated standard deviation in c2flxR3 + + + estimated filling factor at R3 + + + Flux inside r = 4 + + + estimated error is c2flxR4 + + + estimated standard deviation in c2flxR4 + + + estimated filling factor at R4 + + + Flux inside r = 5 + + + estimated error is c2flxR5 + + + estimated standard deviation in c2flxR5 + + + estimated filling factor at R5 + + + Flux inside r = 6 + + + estimated error is c2flxR6 + + + estimated standard deviation in c2flxR6 + + + estimated filling factor at R6 + + + Flux inside r = 7 + + + estimated error is c2flxR7 + + + estimated standard deviation in c2flxR7 + + + estimated filling factor at R7 + + + Flux inside r = 8 + + + estimated error is c2flxR8 + + + estimated standard deviation in c2flxR8 + + + estimated filling factor at R8 + + + Flux inside r = 9 + + + estimated error is c2flxR9 + + + estimated standard deviation in c2flxR9 + + + estimated filling factor at R9 + + + Flux inside r = 10 + + + estimated error is c2flxR10 + + + estimated standard deviation in c2flxR10 + + + estimated filling factor at R10 - +
@@ -1183,83 +4204,9 @@ VOTable description of PSPS table StackToImage - + stack identifier - + hashed exposure-ccdID identifier - - - - - - - VOTable description of PSPS table SkinnyObject - - - ODM object identifier index - - - IPP object number - - - projection cell identifier at discovery time - - - Data release to propagate to the object - - - surveyID </coluumn> - - - - -
- - VOTable description of PSPS table StackDetectionCalib - - - ODM object identifier index - - - ODM detection identifier index - - - IPP object number - - - detection ID generated by IPP - - - filter identifier - - - survey identifier - - - right ascension - - - declination - - - right ascension error - - - declination error - - - zero point offset - - - zerop point offset error - - - exposure time - - - airmass - - - Data release to propagate to the object Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/Makefile.am (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/Makefile.am (revision 36826) @@ -11,4 +11,6 @@ datastoreRemover.py \ detectionbatch.py \ + diffbatch.py \ + diffobjectbatch.py \ dvodetections.py \ dvoobjects.py \ @@ -16,6 +18,9 @@ dxlayer.py \ fits.py \ + forcedobjectbatch.py \ + forcedwarpbatch.py \ gpc1db.py \ initbatch.py \ + insert.dvodb.info.py \ ipptopspsdb.py \ ipptopsps.py \ Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/batch.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/batch.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/batch.py (revision 36826) @@ -440,4 +440,94 @@ return True + ''' + Imports IPP tables from FITS file + + Accepts a regular expression filter so not all tables need to be imported + ''' + def importIppSTTables(self, columns="*", filter=""): + + self.logger.infoPair("Importing ST tables with filter", filter) + + # print "trying to read ", self.fits.getPath() + + # ST has 5 cmf files - all with the same table names. To counter this we do it like this. + + filters = ["g","r","i","z","y"] + count = 0 + for f in filters: + skip = 0 + if (f == "g"): + if self.gstackID > 0: + fileName = self.gfits.getPath() + else: + skip = 1 + if (f == "r"): + if self.rstackID > 0: + fileName = self.rfits.getPath() + else: + skip = 1 + if (f == "i"): + if self.istackID > 0: + fileName = self.ifits.getPath() + else: + skip = 1 + if (f == "z"): + if self.zstackID > 0: + fileName = self.zfits.getPath() + else: + skip = 1 + if (f == "y"): + if self.ystackID > 0: + fileName = self.yfits.getPath() + else: + skip = 1 + + if skip == 1: + self.logger.infoPair("no files for filter" , f) + else: + self.logger.infoPair("using filename:",fileName) + + try: + tables = stilts.treads(fileName) + except: + self.logger.errorPair("STILTS could not import from", fileName) + return False + + #count = 0 + for table in tables: + + # print "import smf table ", table + match = re.match(filter, table.name) + + if not match: continue + self.logger.infoPair("Reading IPP table", f + table.name) + table = stilts.tpipe(table, cmd='addcol table_index $0') + + table = stilts.tpipe(table, cmd='explodeall') + + # print "read smf table ", table + + # drop any previous tables before import + self.scratchDb.dropTable(f + table.name) + + # IPP FITS files are littered with infinities, so remove these + self.logger.debug("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + # print "cleaned up values ", table + + try: + table.write(self.scratchDb.url + '#' + f + table.name) + # XXX this one is not needed + # self.scratchDb.killLastConnectionID() + count = count + 1 + except: + self.logger.exception("Problem writing table '" + f + table.name + "' to the database") + + self.logger.infoPair("Done. Imported", "%d tables" % count) + self.indexIppTables() + + return True ''' @@ -467,6 +557,7 @@ # replace nulls and empty fields with weird PSPS -999 pseudo-null _table = stilts.tpipe(_table, cmd='replaceval "" -999 *') - - match = re.match(regex, table) + self.logger.infoPair('regex', regex) + self.logger.infoPair('table', table) + match = re.search(regex, table) newTableName = match.group(1) @@ -551,14 +642,22 @@ # for P2/ST, this reads the detection tables from the CMF/SMF file(s) # for OB, this imports object data from DVO - if not self.importIppTables(): - if self.batchType == "OB": - self.logger.errorPair("skipping this batch", "no dvo tables for this region") - self.ippToPspsDb.updateProcessed(self.batchID, 1) - return True - - self.logger.errorPair("Aborting this batch", "could not import IPP tables") - self.ippToPspsDb.updateProcessed(self.batchID, -1) - raise - + if self.batchType == "ST": + self.logger.infoPair("did we get here","?") + if not self.importIppSTTables(): + self.logger.errorPair("skipping this batch", "could not import ST IPP tables") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + raise + self.logger.infoPair("we did the ST","yes") + else: + # if not ST we do this + if not self.importIppTables(): + if self.batchType == "OB": + self.logger.errorPair("skipping this batch", "no dvo tables for this region") + self.ippToPspsDb.updateProcessed(self.batchID, 1) + return True + self.logger.errorPair("Aborting this batch", "could not import IPP tables") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + raise + self.logger.infoPair("populatePspsTables","ok") if not self.populatePspsTables(): if self.skipBatch: Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/cleanup.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/cleanup.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/cleanup.py (revision 36826) @@ -46,4 +46,8 @@ self.clean("ST") self.clean("OB") + self.clean("DF") + self.clean("DO") + self.clean("FW") + self.clean("FO") self.checkClientStatus() Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/detectionbatch.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/detectionbatch.py (revision 36826) @@ -257,7 +257,13 @@ ,psfWidMinor \ ,psfTheta \ - ,momentFwhm \ - ,momentWidMajor \ - ,momentWidMinor \ + ,momentXX \ + ,momentXY \ + ,momentYY \ + ,momentM3C \ + ,momentM3S \ + ,momentM4C \ + ,momentM4S \ + ,momentR1 \ + ,momentRH \ ,apResid \ ,dapResid \ @@ -319,7 +325,13 @@ ," + self.safeDictionaryAccess(header, 'FWHM_MIN') + " \ ," + self.safeDictionaryAccess(header, 'ANGLE') + " \ - , ( " + self.safeDictionaryAccess(header, 'IQ_FW1') + " + " + self.safeDictionaryAccess(header, 'IQ_FW2') + " ) / 2.0 \ - ," + self.safeDictionaryAccess(header, 'IQ_FW1') + " \ - ," + self.safeDictionaryAccess(header, 'IQ_FW2') + " \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ + ,-999 \ ," + self.safeDictionaryAccess(header, 'APMIFIT') + " \ ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ @@ -398,4 +410,5 @@ sql = "INSERT IGNORE INTO " + pspsTableName + " ( \ ippDetectID \ + ,randomDetID \ ,filterID \ ,surveyID \ @@ -407,8 +420,8 @@ ,psfFlux \ ,psfFluxErr \ - ,peakADU \ - ,psfWidMajor \ - ,psfWidMinor \ + ,psfMajorFWHM \ + ,psfMinorFWHM \ ,psfTheta \ + ,psfCore \ ,psfLikelihood \ ,psfQf \ @@ -416,4 +429,10 @@ ,momentXY \ ,momentYY \ + ,momentR1 \ + ,momentRH \ + ,momentM3C \ + ,momentM3S \ + ,momentM4C \ + ,momentM4S \ ,apFlux \ ,kronFlux \ @@ -424,12 +443,11 @@ ,sky \ ,skyErr \ - ,sgSep \ - ,activeFlag \ - ,assocDate \ - ,historyModNum \ + ,pltScale \ + ,posAngle \ ,dataRelease \ ) \ SELECT \ IPP_IDET \ + , FLOOR(RAND()*9223372036854775807) \ , " + str(self.filterID) + "\ , " + str(self.surveyID) + " \ @@ -441,8 +459,8 @@ ,PSF_INST_FLUX / " + self.header['EXPTIME'] + " \ ,PSF_INST_FLUX_SIG / " + self.header['EXPTIME'] + " \ - ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + self.header['EXPTIME'] + " \ ,PSF_MAJOR \ ,PSF_MINOR \ ,PSF_THETA \ + ,-999 \ ,psfLikelihood(EXT_NSIGMA) \ ,PSF_QF \ @@ -450,4 +468,10 @@ ,MOMENTS_XY \ ,MOMENTS_YY \ + ,MOMENTS_R1 \ + ,MOMENTS_RH \ + ,MOMENTS_M3C \ + ,MOMENTS_M3S \ + ,MOMENTS_M4C \ + ,MOMENTS_M4S \ ,POW(10.0, -0.4 *AP_MAG) / " + self.header['EXPTIME'] + " \ ,KRON_FLUX / " + self.header['EXPTIME'] + " \ @@ -458,8 +482,6 @@ ,SKY / " + self.header['EXPTIME'] + " \ ,SKY_SIGMA / " + self.header['EXPTIME'] + " \ - ,EXT_NSIGMA \ - , 0 \ - , '" + self.dateStr + "' \ - , 0 \ + ,PLTSCALE \ + ,POSANGLE \ , " + str(self.skychunk.dataRelease) + "\ FROM " + ippTableName @@ -496,5 +518,5 @@ # we don't delete these anymore results['NULLINSTFLUX'] = 0; - results['NULLPEAKADU'] = 0; + ''' @@ -531,10 +553,10 @@ def populateDetectionCalibTableUpdateInsert(self, ota): - tableName = "DetectionCalib_" + ota + tableName = "Detection_" + ota # drop then re-create table - self.scratchDb.dropTable(tableName) - sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" - try: self.scratchDb.execute(sql) - except: pass + #self.scratchDb.dropTable(tableName) + #sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" + #try: self.scratchDb.execute(sql) + #except: pass # insert all detections into table @@ -555,5 +577,5 @@ ,surveyID \ FROM Detection_" + ota - self.scratchDb.execute(sql) +# self.scratchDb.execute(sql) # insert calibration information from dvoDetections into the Table @@ -564,5 +586,4 @@ a.decErr = b.decErr, \ a.zp = b.zp, \ - a.zpErr = b.zpErr, \ a.expTime = b.expTime, \ a.airMass = b.airMass \ @@ -577,66 +598,33 @@ # target table name: - tableName = "DetectionCalib_" + ota + tableName = "Detection_" + ota # drop then re-create table - self.scratchDb.dropTable(tableName) - sql = "CREATE TABLE " + tableName + " LIKE DetectionCalib" - try: self.scratchDb.execute(sql) - except: pass + #self.scratchDb.dropTable(tableName) + #sql = "CREATE TABLE " + tableName + " LIKE Detection" + #try: self.scratchDb.execute(sql) + #except: pass externID = self.imageIDs[ota] imageID = self.scratchDb.getImageIDFromExternID(externID) - self.logger.infoPair("obtained","imageID") - - # check for & create output directory first - datadumpDir = "/tmp/datadump" - try: - statinfo = os.stat(datadumpDir) - # check on the stat results? - except: - print "making the data dump directory ", datadumpDir - os.mkdir(datadumpDir) - os.chmod(datadumpDir, 0777) - statinfo = os.stat(datadumpDir) - - dumpFile = datadumpDir + "/genetest.xx.dat" - files = glob.glob(dumpFile) - if len(files) > 0: - os.unlink(dumpFile) + self.logger.infoPair("obtained imageID",imageID) + self.logger.infoPair("obtained externID",externID) # insert all detections into table - sql = "SELECT \ - a.objID, \ - a.detectID, \ - a.ippObjID, \ - a.ippDetectID, \ - a.filterID, \ - a.surveyID, \ - b.ra, \ - b.dec_, \ - b.raErr, \ - b.decErr, \ - b.zp, \ - b.zpErr, \ - b.expTime, \ - b.airMass, \ - " + str(self.skychunk.dataRelease) + " \ - FROM \ - Detection_" + ota + " as a \ - JOIN " + self.scratchDb.dvoDetectionTable + " as b \ - ON (a.objID = b.objID AND a.detectID = b.detectID) \ - WHERE b.imageID = " + str(imageID) + \ - " INTO OUTFILE '" + dumpFile + "'" + sql = "Update Detection_" + ota + " AS a, "+ self.scratchDb.dvoDetectionTable + " as b \ + set a.ra=b.ra, \ + a.dec = b.dec_, \ + a.raErr = b.raErr, \ + a.decErr = b.decErr, \ + a.zp = b.zp, \ + a.expTime = b.expTime, \ + a.airMass = b.airMass \ + WHERE (a.objID = b.objID AND a.detectID = b.detectID) \ + AND b.imageID = " + str(imageID) \ + try: self.scratchDb.execute(sql) except: - self.logger.info("failed to select data for detectionCalib") - self.logger.infoPair("sql: ", sql) - raise - - sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName - try: self.scratchDb.execute(sql) - except: - self.logger.info("failed to load data from infile for detectionCalib") + self.logger.info("failed to update data for detectionCalib") self.logger.infoPair("sql: ", sql) raise @@ -751,5 +739,5 @@ # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of # the column in PSPS - self.scratchDb.execute("ALTER TABLE DetectionCalib CHANGE dec_ `dec` double") + self.scratchDb.execute("ALTER TABLE Detection CHANGE dec_ `dec` double") return True @@ -818,23 +806,22 @@ return False - self.logger.info("populate stuff "); + #self.logger.info("populate stuff "); # populate remainder of tables self.populateDetectionTable(chipname, results) - self.logger.info("successful populate "); + #self.logger.info("successful populate "); # now add DVO IDs self.updateDvoIDs("Detection_" + chipname, self.imageIDs[chipname]) - self.logger.info("updated dvoids") + #self.logger.info("updated dvoids") results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + chipname, "objID") #self.logger.info("deleted nulls") self.updateImageID("Detection_" + chipname, x, y) - self.logger.info("updateImageId") + #self.logger.info("updateImageId") rowCount = self.scratchDb.getRowCount("Detection_" + chipname) - self.logger.info("got row count") - self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", + #self.logger.info("got row count") + self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |", chipname, results['ORIGINALTOTAL'], results['SATDET'], results['NULLINSTFLUX'], - results['NULLPEAKADU'], results['NULLOBJID'], rowCount) @@ -842,5 +829,4 @@ self.totalSatDet = self.totalSatDet + results['SATDET'] self.totalNulIInstFlux = self.totalNulIInstFlux + results['NULLINSTFLUX'] - self.totalNullPeakFlux = self.totalNullPeakFlux + results['NULLPEAKADU'] self.totalNullObjID = self.totalNullObjID + results['NULLOBJID'] self.totalDetections = self.totalDetections + rowCount @@ -867,11 +853,8 @@ self.scratchDb.execute(sql) - self.logger.info("updated imagedata") - self.populateSkinnyObjectTable(chipname) - self.logger.info("updated skinnyobject") - #self.populateObjectCalColorTable(chipname) - #self.logger.info("updated objectcalcolor") + #self.logger.info("updated imagedata") + self.populateDetectionCalibTable(chipname) - self.logger.info("updated detectioncalibtable") + #self.logger.info("updated detectioncalibtable") # add these to list of tables to export later @@ -882,11 +865,5 @@ self.logger.info("export Detection") - self.tablesToExport.append("SkinnyObject_" + chipname) - self.logger.info("export Skinny") - - #self.tablesToExport.append("ObjectCalColor_" + chipname) - self.tablesToExport.append("DetectionCalib_" + chipname) - self.logger.info("export DetectionCalib") - + tables.append("Detection_" + chipname) self.logger.info("updated detectioncalibtable") @@ -910,7 +887,7 @@ results = {} self.totalOriginal = self.totalSatDet = self.totalNulIInstFlux = self.totalNullPeakFlux = self.totalNullObjID = self.totalDetections = 0 - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") - self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL peak ADU | NULL obj ID | Remainder |") - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL obj ID | Remainder |") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") for x in range(self.startX, self.endX): for y in range(self.startY, self.endY): @@ -937,5 +914,5 @@ # print totals - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", self.totalOriginal, @@ -945,5 +922,5 @@ self.totalNullObjID, self.totalDetections) - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") # if we only have one table export, i.e. FrameMeta, then get out of here (skip batch, but do not abort) @@ -961,4 +938,7 @@ sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) self.scratchDb.execute(sql) + #to make it stop + #test = self.scratchDb.getRowCount("Object") + #return False return True @@ -1040,5 +1020,5 @@ # XXX EAM NOTE : this is fragile : requires PS1_V4 - columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ" + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PSF_INST_FLUX PSF_INST_FLUX_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY MOMENTS_R1 MOMENTS_RH MOMENTS_M3C MOMENTS_M3S MOMENTS_M4C MOMENTS_M4S AP_MAG KRON_FLUX KRON_FLUX_ERR FLAGS FLAGS2 SKY SKY_SIGMA EXT_NSIGMA PSF_QF_PERFECT PSF_CHISQ POSANGLE PLTSCALE" return super(DetectionBatch, self).importIppTables(columns, regex) Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/gpc1db.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/gpc1db.py (revision 36826) @@ -87,5 +87,5 @@ stage = "skycal" - sql = "SELECT DISTINCT stack_id, radeg, decdeg FROM skycalRun \ + sql = "SELECT DISTINCT sky_id, radeg, decdeg FROM skycalRun \ JOIN addRun ON(skycalRun.skycal_id = addRun.stage_id) \ JOIN minidvodbRun USING(minidvodb_name) \ @@ -107,5 +107,15 @@ AND mjd_obs >= (to_days('" + tstart + "')-678941) \ AND mjd_obs <= (to_days('" + tend + "') - 678941) " - self.logger.infoPair("sql",sql) + # self.logger.infoPair("sql",sql) + elif batchType == "DF": + + stage = "diff" + sql = "SELECT stuff from stufftable" + + elif batchType == "FW": + + stage = "forcedwarp" + sql = "SELECT differentStuff from Stufftable" + try: # XXX EAM : test output @@ -388,5 +398,5 @@ # print "staring stack stage cmf" - + #self.logger.infoPair("in stackstagecmf - sql is: ", sql) # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id try: @@ -489,2 +499,47 @@ + ''' + grabs the stackID for a particular skyId, filter, and dvodb + the mapping of stack_id from sky_id is not unique unless + joined to the dvodb + ''' + def getStackIDFromSkyIDAndFilter(self,dvoDb,skyID,filter): + + self.logger.infoPair("finding stack_ids from gpc1 for sky_id ", skyID) + + sql = "SELECT DISTINCT stack_id from skycalRun \ + JOIN addRun ON(skycalRun.skycal_id = addRun.stage_id) \ + JOIN minidvodbRun USING(minidvodb_name) \ + JOIN minidvodbProcessed USING(minidvodb_id) \ + JOIN mergedvodbRun USING(minidvodb_id) \ + JOIN mergedvodbProcessed USING(merge_id) \ + JOIN stackRun USING (stack_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND minidvodbProcessed.fault = 0 \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbProcessed.fault = 0 \ + AND addRun.stage = 'skycal' \ + AND addRun.state = 'full' \ + AND filter = '" + filter + "' \ + AND sky_id = %d" % skyID + +# self.logger.infoPair("sql",sql) + + try: + rs = self.executeQuery(sql) + + except: + self.logger.exception("can't query for stackid in DVO") + self.logger.infoPair("failed sql:", sql) + return 0 + + rs.first() + try: + stackid = rs.getInt(1) + self.logger.infoPair("found stack_ID", stackid) + except: + stackid = -999 + self.logger.infoPair("using stack_ID",stackid) + + return stackid Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/insert.dvodb.info.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/insert.dvodb.info.py (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/insert.dvodb.info.py (revision 36826) @@ -0,0 +1,150 @@ +#!/usr/bin/env jython + +# +# The main loading program for ippToPsps. See usage below. +# + +import time +import sys +import os +import math +import logging.config + +from ipptopsps import IppToPsps +from gpc1db import Gpc1Db +from datastore import Datastore +from dvoobjects import DvoObjects +from batch import Batch +from scratchdb import ScratchDb + +''' +Queue class +''' +class Queue(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Queue, self).__init__(argv) + + # print "inited ipptopsps" + + # create various objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + # print "connect to gpc1" + + self.datastore = Datastore(self.logger, self.skychunk, self.ippToPspsDb) + + print "made datastore" + + # connect to scratch database + self.scratchDb = ScratchDb(self.logger, self.config) + + print "connect to scratchdb" + + try: + print "why doesn't this work" + self.dvoObjects = DvoObjects(self.logger, self.config, self.skychunk, self.ippToPspsDb, self.scratchDb) + except: + self.exitProgram("Unable to create instance of DvoObject") + raise + + print "got objects" + + self.skychunk.printAll() + + if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + ''' + Main processing loop. + Tiled area is defined in 'skychunk' and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing + by one or more loading clients that may be running on any host + ''' + def run(self): + print "here" + # this outer while loop simply waits a few minutes then starts queuing again (as more stuff may appear in DVO over time) + while True: + + # self.ippToPspsDb.removeAllBoxes() + print "remove boxes" + # queue up batches that are processed but not loaded to datastore + for batchType in self.skychunk.batchTypes: + self.logger.infoPair("BatchType",batchType) + #self.logger.infoTitle("Previous failed datastore loads") + #self.publishAnyUnpublishedBatches(batchType) + + # get totals for whole area to check if there is anything to do + processedIDs = self.ippToPspsDb.getProcessedIDs(batchType) + print "got processed ids" + consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType) + print "got here" + # for object batches, get full list of stuff from Dvo + # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo + #for object-like batches, info comes from dvo + if (batchType == "OB" or batchType =="DO" or batchType =="FO"): + self.dvoObjects.setSkyArea( + self.skychunk.minRa, + self.skychunk.maxRa, + self.skychunk.minDec, + self.skychunk.maxDec) + rows = self.dvoObjects.allPopulatedRegionInfo + self.dvoObjects.printSummary() + + # for other batches, info comes from gpc1 database + else: + rows = self.gpc1Db.getItemsInThisDVODbForThisStage( + self.skychunk.dvoLabel, + batchType, + self.skychunk.minRa, + self.skychunk.maxRa, + self.skychunk.minDec, + self.skychunk.maxDec, + self.skychunk.trange_start, + self.skychunk.trange_end) + + # EAM TEST I/O + self.logger.infoPair("received rows from db:", len(rows)) + + # first report total stuff + allIDs = [] + for row in rows: + try: allIDs.append(row[0]) + except: pass + + ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs)) + self.logger.infoPair("All unprocessed items", "%d" % len(ids)) + + # now trim full list down to only pending stuff + pending = [[], [], []] + for row in rows: + try: + findId = row[0] + except: continue + if findId in allIDs: pending.append([row[0], row[1], row[2]]) + + # store all items in temp table + self.logger.infoPair("Inserting all pending items into", "ippToPsps database") + self.ippToPspsDb.storeAllItemsInDvodb(pending, self.skychunk.dvoLabel, batchType) + + self.logger.info("+-------------+-------------+-------------+-------------+") + + self.checkClientStatus() + if not self.waitForPollTime(): break + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Queue, self).printUsage("[]") + + +''' +Start of program. +''' +try: + queue = Queue(sys.argv) + queue.run() + queue.exitProgram("completed") +except: pass + Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/ipptopspsdb.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/ipptopspsdb.py (revision 36826) @@ -1116,4 +1116,8 @@ ,queue_ST \ ,queue_OB \ + ,queue_DF \ + ,queue_DO \ + ,queue_FW \ + ,queue_FO \ ,parallel \ ,P2_smf_version \ @@ -1162,13 +1166,17 @@ if rs.getInt(20) == 1: self.skychunk.batchTypes.append("ST") if rs.getInt(21) == 1: self.skychunk.batchTypes.append("OB") + if rs.getInt(22) == 1: self.skychunk.batchTypes.append("DF") + if rs.getInt(23) == 1: self.skychunk.batchTypes.append("DO") + if rs.getInt(24) == 1: self.skychunk.batchTypes.append("FW") + if rs.getInt(25) == 1: self.skychunk.batchTypes.append("FO") self.skychunk.force = True # TODO self.skychunk.parallel = False # TODO - if rs.getInt(22) == 1: self.skychunk.parallel = True - self.skychunk.P2_smf_version = rs.getString(23) - self.skychunk.ST_cmf_version = rs.getString(24) - self.skychunk.trange_start = rs.getString(25) - self.skychunk.trange_end = rs.getString(26) + if rs.getInt(26) == 1: self.skychunk.parallel = True + self.skychunk.P2_smf_version = rs.getString(27) + self.skychunk.ST_cmf_version = rs.getString(28) + self.skychunk.trange_start = rs.getString(29) + self.skychunk.trange_end = rs.getString(30) if self.skychunk.parallel: print "USING parallel" @@ -1643,4 +1651,32 @@ # self.logger.infoPair("Items written to alt Db", "%d" % count) + + ''' + Creates a temporary table and shoves it full of stage_ids and ra/dec coords + ''' + def storeAllItemsInDvodb(self, rows, dvo_db, batchType): + + #try: + # self.execute("DROP TABLE all_pending") + #except: + # pass + + #self.execute("CREATE TEMPORARY TABLE all_pending (stage_id bigint(20), ra_bore float, dec_bore float)") + for row in rows: + + try: + uniq = dvo_db + "."+ batchType + "." + str(row[0]) + sql = "INSERT INTO dvodb (dvo_db, batch_type, stage_id, ra_center, dec_center, uniq_key) \ + VALUES ( '" + dvo_db + "', '" + batchType + "'," + str(row[0]) + "," + str(row[1]) + " \ + , " + str(row[2]) + ", '"+uniq+"' ) " + print "sql 1: ", sql + self.execute(sql) + except: continue + + #count = self.getRowCount("all_pending") + #self.logger.infoPair("Items written to Db", "%d" % count) + + + ''' Gets all items in the all_pending temporary table within the bounds of this box Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/loader.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/loader.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/loader.py (revision 36826) @@ -182,5 +182,5 @@ # XXXX EAM : in parallel model, this should happen for all P2 & Stack batches # - if (batchType == 'P2' or batchType == 'ST') and smfsPerGB > 30: + if (batchType == 'P2' or batchType == 'ST' or batchType == 'DF' or batchType == 'FW') and smfsPerGB > 30: if not self.dvoDetections.sync(): self.logger.errorPair("Could not sync DVO with MySQL", "skipping") @@ -234,24 +234,25 @@ # loop round IDs of all items to be processed self.ippToPspsDb.lockBatchTable() - self.logger.infoPair("heather:","lockbatchtable") + #self.logger.infoPair("heather:","lockbatchtable") unattemptedCount = 0 for id in ids: - self.logger.infoPair("heather:","in ids") - self.logger.infoPair("heather:id",str(id)) + #self.logger.infoPair("heather:","in ids") + #self.logger.infoPair("heather:id",str(id)) batchID = self.ippToPspsDb.createNewBatch(batchType, id) - self.logger.infoPair("heather:","creatednewbatch") - self.logger.infoPair("heather:batchId", str(batchID)) + #self.logger.infoPair("heather:","creatednewbatch") + #self.logger.infoPair("heather:batchId", str(batchID)) if batchID < 0: unattemptedCount += 1 continue - self.logger.infoPair("heather:","passed logic") + #self.logger.infoPair("heather:","passed logic") self.ippToPspsDb.unlockTables() - self.logger.infoPair("heather:","unlock table") + #self.logger.infoPair("heather:","unlock table") self.ippToPspsDb.deletePendingItem(batchType, id) - self.logger.infoPair("heather:","deleted pending item") + #self.logger.infoPair("heather:","deleted pending item") # catch any raised exceptions in batch constructors try: if batchType == "P2": + self.logger.infoPair("defining P2 batch for cam_id:",id) batch = DetectionBatch(self.logger, self.config, @@ -264,4 +265,5 @@ useFullTables) elif batchType == "ST": + self.logger.infoPair("defining ST batch for sky_id:",id) batch = StackBatch(self.logger, self.config, @@ -285,4 +287,48 @@ useFullTables) + elif batchType == "DF": + batch = DiffBatch(self.logger, + self.config, + self.skychunk, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + elif batchType == "DO": + batch = DiffObjectBatch(self.logger, + self.config, + self.skychunk, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + elif batchType == "FW": + batch = ForcedWarpBatch(self.logger, + self.config, + self.skychunk, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + elif batchType == "FO": + batch = ForcedObjectBatch(self.logger, + self.config, + self.skychunk, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + batch.run() Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/metrics.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/metrics.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/metrics.py (revision 36826) @@ -48,4 +48,8 @@ rateST = self.printStats("ST") rateOB = self.printStats("OB") + rateDF = self.printStats("DF") + rateDO = self.printStats("DO") + rateFW = self.printStats("FW") + rateFO = self.printStats("FO") self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") @@ -66,4 +70,8 @@ pendSTProcessed = self.printTableRow(stages, "ST") pendOBProcessed = self.printTableRow(stages, "OB") + pendDFProcessed = self.printTableRow(stages, "DF") + pendDOProcessed = self.printTableRow(stages, "DO") + pendFWProcessed = self.printTableRow(stages, "FW") + pendFOProcessed = self.printTableRow(stages, "FO") self.writeTableSeparator(colCount) @@ -75,12 +83,23 @@ try: self.logger.infoPair("Estimated time for OBs", "%.1f hours" % (pendOBProcessed / rateOB)) except: pass + try: self.logger.infoPair("Estimated time for DFs", "%.1f hours" % (pendDFProcessed / rateDF)) + except: pass + try: self.logger.infoPair("Estimated time for DOs", "%.1f hours" % (pendDOProcessed / rateDO)) + except: pass + try: self.logger.infoPair("Estimated time for FWs", "%.1f hours" % (pendFWProcessed / rateFW)) + except: pass + try: self.logger.infoPair("Estimated time for FOs", "%.1f hours" % (pendFOProcessed / rateFO)) + except: pass self.checkClientStatus() - self.logger.infoPair("here","yay") + self.logger.infoPair("creating density plots","") # create a density plot of pending stuff for czartool self.plot.createDensityPlot("P2", True) self.plot.createDensityPlot("ST", True) self.plot.createDensityPlot("OB", True) - self.logger.infoPair("here","yay2") + self.plot.createDensityPlot("DF", True) + self.plot.createDensityPlot("DO", True) + self.plot.createDensityPlot("FW", True) + self.plot.createDensityPlot("FO", True) # some clean-up before going to sleep self.ippToPspsDb.purgeDeadClients() Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/objectbatch.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/objectbatch.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/objectbatch.py (revision 36826) @@ -91,5 +91,6 @@ # dec is reserved in MySQL, so STILTS replaces if with dec_, which PSPS doesn't like. so, force it back again using `` - self.scratchDb.execute("ALTER TABLE Object CHANGE dec_ `dec` double") + # self.scratchDb.execute("ALTER TABLE MeanObject CHANGE dec_ `dec` double") + # self.scratchDb.execute("ALTER TABLE ObjectThin CHANGE dec_ `dec` double") return True @@ -107,5 +108,5 @@ Inserts stuff for all mags ''' - def insertMags(self, cpsTable): + def updateMeanObjectFromCps(self, cpsTable): # list of all filters PSPS is interested in @@ -129,40 +130,28 @@ # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags - sql = "UPDATE Object JOIN \ - " + cpsTable + " AS cps ON (cps.row = (Object.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + sql = "UPDATE MeanObject JOIN \ + " + cpsTable + " AS cps ON (cps.row = (MeanObject.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ SET \ - n" + filter[1] + " = NCODE \ - ," + filter[1] + "MeanPSFMag = MAG \ - ," + filter[1] + "MeanPSFMagErr = MAG_ERR \ - ," + filter[1] + "MeanKronMag = MAG_KRON \ - ," + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \ - ," + filter[1] + "StackPSFMag = STACK_PSF_MAG \ - ," + filter[1] + "StackPSFMagErr = STACK_PSF_MAG_ERR \ - ," + filter[1] + "StackKronMag = STACK_KRON_MAG \ - ," + filter[1] + "StackKronMagErr = STACK_KRON_MAG_ERR \ - ," + filter[1] + "20pct = MAG_20/1000 \ - ," + filter[1] + "80pct = MAG_80/1000 \ - ," + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) \ - ," + filter[1] + "StackDetectID = STACK_DETECT_ID" - -# old method for setting the Stack (PSF,Kron) Mags: -# ," + filter[1] + "StackPSFMag = -2.5 * log10(FLUX_PSF) + 8.9 \ -# ," + filter[1] + "StackPSFMagErr = FLUX_PSF_ERR / FLUX_PSF \ -# ," + filter[1] + "StackKronMag = -2.5 * log10(FLUX_KRON) + 8.9 \ -# ," + filter[1] + "StackKronMagErr = FLUX_KRON_ERR / FLUX_KRON \ - - #self.logger.info(sql) - self.scratchDb.execute(sql) - - self.logger.infoPair("Calculating nDetections from", "n[filters]") - for filter in filters: - # now do a sum of n[filters], but do not include the ones with -999 - sql = "UPDATE Object \ - SET nDetections = nDetections + n" + filter[1] + " \ - WHERE n" + filter[1] + " != -999" - self.scratchDb.execute(sql) - - - + MeanObject." + filter[1] + "QfPerfect = -999 \ + ,MeanObject." + filter[1] + "MeanPSFMag = MAG \ + ,MeanObject." + filter[1] + "MeanPSFMagErr = MAG_ERR \ + ,MeanObject." + filter[1] + "MeanKronMag = MAG_KRON \ + ,MeanObject." + filter[1] + "MeanKronMagErr = MAG_KRON_ERR \ + ,MeanObject." + filter[1] + "nIncPSFMag = -999 \ + ,MeanObject." + filter[1] + "MeanPSFMagStd = -999 \ + ,MeanObject." + filter[1] + "MinPSFMag = -999 \ + ,MeanObject." + filter[1] + "MaxPSFMag = -999 \ + ,MeanObject." + filter[1] + "nIncKronMag = -999 \ + ,MeanObject." + filter[1] + "MeanKronMag = -999 \ + ,MeanObject." + filter[1] + "MeanKronMagStd = -999 \ + ,MeanObject." + filter[1] + "MeanApMag = MAG_AP \ + ,MeanObject." + filter[1] + "MeanApMagErr = MAG_ERR \ + ,MeanObject." + filter[1] + "MeanApMagStd = MAG_STDEV \ + ,MeanObject." + filter[1] + "nIncApMag = -999 \ + ,MeanObject." + filter[1] + "Flags = (0x7fff & FLAGS) | ((FLAGS >> 11) & 0x2000) " + + + self.logger.info(sql) + self.scratchDb.execute(sql) # now set to null all MeanMagErr values > 0.5 (cut set by Gene, 2012-04-12) @@ -171,51 +160,135 @@ for filter in filters: - sql = "UPDATE Object \ + sql = "UPDATE MeanObject \ SET " + filter[1] + "MeanPSFMagErr = null \ WHERE " + filter[1] + "MeanPSFMagErr > " + str(cut) self.scratchDb.execute(sql) - - self.logger.infoPair("Setting to NULL all x20pct =", "32.767" ) + ''' + Inserts stuff for all mags + ''' + def updateObjectThinFromCps(self, cpsTable): + + # list of all filters PSPS is interested in + interestedFilters = ['g', 'r', 'i', 'z', 'y'] + + filters = self.scratchDb.getOrderedListOfFiltersFromPhotcodesTable(interestedFilters) + + # get a count of the available filters + filterCount = self.scratchDb.getCountOfFiltersFromPhotcodesTable() + # filterCount = len(filters) + + self.logger.infoPair("Available filters in Photcodes", filters) + + # the 'code' now defines the order in the cps file that the mags are listed for a given filter + self.logger.infoPair("Adding magnitudes from", "cps table") for filter in filters: - sql = "UPDATE Object \ - SET " + filter[1] + "20pct = null \ - WHERE " + filter[1] + "20pct > 32.766" - self.scratchDb.execute(sql) - self.logger.infoPair("Setting to NULL all x80pct =", "32.767" ) + + filterID = self.scratchDb.getFilterID(filter[1]) + + # NOTE: Manipulation of FLAGS from cpsTable is to move ID_SECF_OBJ_EXT flag (0x01000000) + # from bit 24 to bit 13 so that it fits into the SMALLINT Object.Flags + + sql = "UPDATE ObjectThin JOIN \ + " + cpsTable + " AS cps ON (cps.row = (ObjectThin.row* " + str(filterCount) + ")-(" + str(filterCount) + " - " + str(filter[0]) + ")) \ + SET \ + ObjectThin.n" + filter[1] + " = NCODE " + self.logger.info(sql) + self.scratchDb.execute(sql) + + self.logger.infoPair("Calculating nDetections from", "n[filters]") for filter in filters: - sql = "UPDATE Object \ - SET " + filter[1] + "80pct = null \ - WHERE " + filter[1] + "80pct > 32.766" - self.scratchDb.execute(sql) - - - - - ''' - Inserts colors - ''' - def updateColors(self): - - self.logger.infoPair("Calculating", "colors") - sql = "UPDATE Object \ - SET \ - grMeanColor = (gMeanMag - rMeanMag) \ - ,riMeanColor = (rMeanMag - iMeanMag) \ - ,izMeanColor = (iMeanMag - zMeanMag) \ - ,zyMeanColor = (zMeanMag - yMeanMag) \ - ,grMeanColorErr = SQRT(POW(gMeanMagErr,2) + POW(rMeanMagErr,2)) \ - ,riMeanColorErr = SQRT(POW(rMeanMagErr,2) + POW(iMeanMagErr,2)) \ - ,izMeanColorErr = SQRT(POW(iMeanMagErr,2) + POW(zMeanMagErr,2)) \ - ,zyMeanColorErr = SQRT(POW(zMeanMagErr,2) + POW(yMeanMagErr,2)) \ - " - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't calculate colors", sql) + # now do a sum of n[filters], but do not include the ones with -999 + sql = "UPDATE ObjectThin \ + SET nDetections = nDetections + n" + filter[1] + " \ + WHERE n" + filter[1] + " != -999" + self.scratchDb.execute(sql) + + + + ''' + give objectName to objectThin + ''' + def updateObjName(self): + ##this is a 4 part update: + self.logger.infoPair("updating objName", "using stacks") + ## if we have a stackra and dec, use stackra and dec , dec >=0 + + sql = "update ObjectThin set objName = concat('PS1.2 J', \ + lpad(floor(raStack/15.),2,'0'), \ + lpad((raStack/15.-(floor(raStack/15.)))*60. ,2,'0'), \ + lpad(format(((raStack/15.-(floor(raStack/15.)))*60.-floor((raStack/15.-(floor(raStack/15.)))*60.))*60.,2),4,'0'), \ + '+', \ + lpad(floor(decStack),2,'0'), \ + lpad((decStack-(floor(decStack)))*60. ,2,'0'), \ + lpad(format(((decStack-(floor(decStack)))*60.-floor((decStack-(floor(decStack)))*60.))*60.,2),4,'0') \ + ) where raStack > -999 and decStack > -999 and decStack >= 0" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql", sql) + return False + + + ## if we have a stackra and dec, use stackra and dec, dec < 0 + + sql = "update ObjectThin set objName = concat('PS1.2 J', \ + lpad(floor(raStack/15.),2,'0'), \ + lpad((raStack/15.-(floor(raStack/15.)))*60. ,2,'0'), \ + lpad(format(((raStack/15.-(floor(raStack/15.)))*60.-floor((raStack/15.-(floor(raStack/15.)))*60.))*60.,2),4,'0'), \ + '-', \ + lpad(floor(decStack),2,'0'), \ + lpad((decStack-(floor(decStack)))*60. ,2,'0'), \ + lpad(format(((decStack-(floor(decStack)))*60.-floor((decStack-(floor(decStack)))*60.))*60.,2),4,'0') \ + ) where raStack > -999 and decStack > -999 and decStack <= 0" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql", sql) + return False + self.logger.infoPair("updating objName", "using means") + ## use mean ra and dec (dec >= 0) + sql = "update ObjectThin set objName = concat('PS1.2 J', \ + lpad(floor(raMean/15.),2,'0'), \ + lpad((raMean/15.-(floor(raMean/15.)))*60. ,2,'0'), \ + lpad(format(((raMean/15.-(floor(raMean/15.)))*60.-floor((raMean/15.-(floor(raMean/15.)))*60.))*60.,2),4,'0'), \ + '+', \ + lpad(floor(decMean),2,'0'), \ + lpad((decMean-(floor(decMean)))*60. ,2,'0'), \ + lpad(format(((decMean-(floor(decMean)))*60.-floor((decMean-(floor(decMean)))*60.))*60.,2),4,'0') \ + ) where raMean > -999 and decMean > -999 and decMean >= 0" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql", sql) + return False + + ## use mean ra and dec (dec < 0) + + sql = "update ObjectThin set objName = concat('PS1.2 J', \ + lpad(floor(raMean/15.),2,'0'), \ + lpad((raMean/15.-(floor(raMean/15.)))*60. ,2,'0'), \ + lpad(format(((raMean/15.-(floor(raMean/15.)))*60.-floor((raMean/15.-(floor(raMean/15.)))*60.))*60.,2),4,'0'), \ + '-', \ + lpad(floor(decMean),2,'0'), \ + lpad((decMean-(floor(decMean)))*60. ,2,'0'), \ + lpad(format(((decMean-(floor(decMean)))*60.-floor((decMean-(floor(decMean)))*60.))*60.,2),4,'0') \ + ) where raMean > -999 and decMean > -999 and decMean < 0" + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("failed sql", sql) + return False + + + + + + ''' Populates the Object table ''' - def populateObjectTable(self): + def populateObjectThinTable(self): cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") @@ -242,42 +315,62 @@ - self.logger.infoPair("Populating", "Object s") + self.logger.infoPair("Populating", "ThinObject") self.logger.infoPair("Inserting objects from", "cpt file") # note `` around dec here, as this is a reserved word in MySQL - sql = "INSERT IGNORE INTO Object (\ + sql = "INSERT IGNORE INTO ObjectThin (\ objID \ + ,gcobjID \ ,ippObjID \ + ,surveyID \ + ,skyCellID \ + ,randomID \ + ,batchID \ + ,dvoRegionID \ + ,dataRelease \ ,objInfoFlag \ ,qualityFlag \ - ,surveyID \ - ,ra \ - ,`dec` \ - ,raErr \ - ,decErr \ + ,consistencyFlag \ + ,raStack \ + ,decStack \ + ,raStackErr \ + ,decStackErr \ + ,raMean \ + ,decMean \ + ,raMeanErr \ + ,decMeanErr \ + ,raMeanStd \ + ,decMeanStd \ + ,nStackObjectRows \ + ,nStackDetections \ ,nDetections \ - ,qfPerfect \ - ,sgSep \ - ,dataRelease \ - ,Random \ - ,batchID \ - ) \ + ) \ SELECT \ EXT_ID \ + , -999 \ ,CAT_ID*1000000000 + OBJ_ID \ + ," + str(self.surveyID) + " \ + , -999 \ + , FLOOR(RAND()*9223372036854775807) \ + , " + str(self.batchID) + "\ + , -999 \ + , " + str(self.skychunk.dataRelease) + "\ ,FLAGS \ ,FLAGS >> 24 & 0xFF \ - ," + str(self.surveyID) + " \ + , 0 \ + , -999 \ + , -999 \ + , -999 \ + , -999 \ ,RA \ ,DEC_ \ ,RA_ERR \ ,DEC_ERR \ + , -999 \ + , -999 \ ,0 \ - ,PSF_QF_PERF \ - ,STARGAL_SEP \ - , " + str(self.skychunk.dataRelease) + "\ - , RAND() \ - , " + str(self.batchID) + "\ - FROM " + cptTableName + ,0 \ + ,0 \ + FROM " + cptTableName try: @@ -289,11 +382,16 @@ # add row count columns so we can perform joins to get colors self.logger.infoPair("Adding 'row' columns to", "Object and cps tables") - self.scratchDb.addRowCountColumn("Object", "row") + self.scratchDb.addRowCountColumn("ObjectThin", "row") self.scratchDb.addRowCountColumn(cpsTableName, "row") - self.insertMags(cpsTableName) - -#don't do this as we removed those columns.. -# self.updateColors() +## self.insertMeanMags(cpsTableName) +## this is the new version + self.logger.infoPair("update objName for ","objectThin") + self.updateObjName() + + + self.logger.infoPair("update ObjectThin from ","cps table") + + self.updateObjectThinFromCps(cpsTableName) #objects can have out of range ra dec in dvo - need to find and kill them at the end @@ -312,5 +410,9 @@ #count out of range - sql = "SELECT count(*) FROM Object where Object.dec > " + str(decMax) + " or Object.dec < " + str(decMin) + " or Object.ra > " + str(raMax) + " or Object.ra < " + str(raMin) + sql = "SELECT count(*) FROM ObjectThin where \ + ObjectThin.decMean > " + str(decMax) + " \ + or ObjectThin.decMean < " + str(decMin) + " \ + or ObjectThin.raMean > " + str(raMax) + " \ + or ObjectThin.raMean < " + str(raMin) rs = self.scratchDb.executeQuery(sql) @@ -321,5 +423,9 @@ - sql = "DELETE FROM Object where Object.dec > " + str(decMax) + " or Object.dec < " + str(decMin) + " or Object.ra > " + str(raMax) + " or Object.ra < " + str(raMin) + sql = "DELETE FROM ObjectThin where \ + ObjectThin.decMean > " + str(decMax) + " or \ + ObjectThin.decMean < " + str(decMin) + " or \ + ObjectThin.raMean > " + str(raMax) + " or \ + ObjectThin.raMean < " + str(raMin) self.logger.infoPair("Deleting", str(nToDelete) + " objects outside of ra/dec range") @@ -327,14 +433,15 @@ self.scratchDb.execute(sql) except: - self.logger.errorPair("Couldn't cull outsiders from Object table", sql) - return False - - self.logger.infoPair("Dropping row column from", "Object table") - self.scratchDb.dropColumn("Object", "row") - self.logger.infoPair("Purging from scratch Db", self.region + " region") - - self.dvoObjects.purgeRegion(self.region) - - self.setMinMaxObjID(["Object"]) + self.logger.errorPair("Couldn't cull outsiders from ObjectThin table", sql) + return False + + self.logger.infoPair("Dropping row column from", "ObjectThin table") + self.scratchDb.dropColumn("ObjectThin", "row") + ##self.logger.infoPair("Purging from scratch Db", self.region + " region") + + ##Don't do this till after MeanObject + ##self.dvoObjects.purgeRegion(self.region) + + self.setMinMaxObjID(["ObjectThin"]) return True @@ -343,23 +450,33 @@ Populates the ObjectCalColor table ''' - def populateObjectCalColorTable(self): - - self.logger.infoPair("Populating", "ObjectCalColor") - - sql = "INSERT INTO ObjectCalColor ( \ + def populateMeanObjectTable(self): + cptTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cpt") + cpsTableName = self.scratchDb.getDbFriendlyTableName(self.region + ".cps") + self.logger.infoPair("Populating MeanObject from ", "ObjectThin") + + sql = "INSERT INTO MeanObject ( \ objID \ - ,ippObjID \ - ,dataRelease \ + ,gcobjID \ ) \ SELECT \ objID \ - ,ippObjID \ - ,dataRelease \ - FROM Object" - try: - self.scratchDb.execute(sql) - except: - self.logger.errorPair("Couldn't populate ObjectCalColor table", sql) - return False + ,gcobjID \ + FROM ObjectThin" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Couldn't populate MeanObject table", sql) + return False + + self.logger.infoPair("Adding 'row' columns to", "MeanObject table") + self.scratchDb.addRowCountColumn("MeanObject", "row") + + + ##self.scratchDb.addRowCountColumn(cpsTableName, "row") + self.logger.infoPair("update MeanObjects from ","cps table") + self.updateMeanObjectFromCps(cpsTableName) + + self.logger.infoPair("Dropping row column from", "MeanObject table") + self.scratchDb.dropColumn("MeanObject", "row") return True @@ -370,10 +487,11 @@ def populatePspsTables(self): - if not self.populateObjectTable(): return False + if not self.populateObjectThinTable(): return False #if not self.populateObjectCalColorTable(): return False + if not self.populateMeanObjectTable(): return False # now remove the objID duplicates. We could not do this before as cpt/cps tables relate by row number - self.logger.infoPair("Forcing uniqueness on", "objID in Object table") - rowCountBefore = self.scratchDb.getRowCount("Object") + self.logger.infoPair("Forcing uniqueness on", "objID in ObjectThin table") + rowCountBefore = self.scratchDb.getRowCount("ObjectThin") # XXX EAM : note that in mysql versions later than 5.1, this fails @@ -386,11 +504,35 @@ self.scratchDb.execute("set session old_alter_table=1") - self.scratchDb.execute("ALTER IGNORE TABLE Object ADD UNIQUE INDEX(objID)") + self.scratchDb.execute("ALTER IGNORE TABLE ObjectThin ADD UNIQUE INDEX(objID)") if self.scratchDb.version > 5.1: self.scratchDb.execute("set session old_alter_table=0") - rowCountAfter = self.scratchDb.getRowCount("Object") + rowCountAfter = self.scratchDb.getRowCount("ObjectThin") self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + self.logger.infoPair("Forcing uniqueness on", "objID in MeanObject table") + rowCountBefore = self.scratchDb.getRowCount("MeanObject") + + # XXX EAM : note that in mysql versions later than 5.1, this fails + # unless the following is called first: + # set session old_alter_table=1 + # follow the command with + # set session old_alter_table=0 + # OF COURSE, this fails for mysql version < 5.5... + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=1") + + self.scratchDb.execute("ALTER IGNORE TABLE MeanObject ADD UNIQUE INDEX(objID)") + if self.scratchDb.version > 5.1: + self.scratchDb.execute("set session old_alter_table=0") + + rowCountAfter = self.scratchDb.getRowCount("MeanObject") + self.logger.infoPair("Number of duplicated objIDs removed", "%d out of %d" % ((rowCountBefore - rowCountAfter), rowCountBefore)) + + self.dvoObjects.purgeRegion(self.region) + + #this is abuse of something but this is how I get the object batches to crash to further investigate them + +# rowCountAfter = self.scratchDb.getRowCount("Object") return True - +# return False Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/odm.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/odm.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/odm.py (revision 36826) @@ -31,5 +31,5 @@ tempFile = tempfile.NamedTemporaryFile(mode='w+b') - + self.logger.infoPair("checkOdmStatus.pl -b " + batch + " -f " + tempFile.name, "") p = Popen("checkOdmStatus.pl -b " + batch + " -f " + tempFile.name, shell=True, Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/plotter.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/plotter.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/plotter.py (revision 36826) @@ -28,4 +28,8 @@ self.plot.createDensityPlot("ST") self.plot.createDensityPlot("OB") + self.plot.createDensityPlot("DF") + self.plot.createDensityPlot("DO") + self.plot.createDensityPlot("FW") + self.plot.createDensityPlot("FO") ''' Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/pollOdm.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/pollOdm.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/pollOdm.py (revision 36826) @@ -42,5 +42,5 @@ else: self.stages = [self.STAGE] - if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB'] + if self.BATCHTYPE == "all": self.batchTypes = ['IN', 'P2', 'ST', 'OB', 'DF', 'DO', 'FW', 'FO'] else: self.batchTypes = [self.BATCHTYPE] @@ -126,5 +126,5 @@ ''' def printUsage(self): - super(PollOdm, self).printUsage(" [hours]") + super(PollOdm, self).printUsage(" [hours]") Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/queue.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/queue.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/queue.py (revision 36826) @@ -82,5 +82,6 @@ # for object batches, get full list of stuff from Dvo # XXX : need to fix 'dvo.setSkyArea' or this may not return the full list for a parallel dvo - if batchType == "OB": + #for object-like batches, info comes from dvo + if (batchType == "OB" or batchType =="DO" or batchType =="FO"): self.dvoObjects.setSkyArea( self.skychunk.minRa, Index: /branches/eam_branches/ipp-20140423/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/jython/stackbatch.py (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/jython/stackbatch.py (revision 36826) @@ -4,4 +4,5 @@ import glob import sys + import stilts @@ -37,5 +38,5 @@ ippToPspsDb, scratchDb, - stackID, + skyID, batchID, useFullTables): @@ -48,24 +49,128 @@ ippToPspsDb, scratchDb, - stackID, + skyID, batchID, "ST", - gpc1Db.getStackStageCmf(skychunk.dvoLabel, stackID), + "", + #gpc1Db.getStackStageCmf(skychunk.dvoLabel, stackID), useFullTables) # self.printline = 0 # # self.testprint() - +# self.logger.infoPair("did I get", "here"); self.stackType = "DEEP_STACK" # TODO - + ## = stuff I commented out for pv2 stacks + + #heather tasks + + # get a list of stackIDs + self.logger.infoPair ("getting stackID for: ","g.00000") + gstackID = gpc1Db.getStackIDFromSkyIDAndFilter( skychunk.dvoLabel, skyID, "g.00000") + self.logger.infoPair ("getting stackID for: ","r.00000") + rstackID = gpc1Db.getStackIDFromSkyIDAndFilter( skychunk.dvoLabel, skyID, "r.00000") + self.logger.infoPair ("getting stackID for: ","i.00000") + istackID = gpc1Db.getStackIDFromSkyIDAndFilter( skychunk.dvoLabel, skyID, "i.00000") + self.logger.infoPair ("getting stackID for: ","z.00000") + zstackID = gpc1Db.getStackIDFromSkyIDAndFilter( skychunk.dvoLabel, skyID, "z.00000") + self.logger.infoPair ("getting stackID for: ","y.00000") + ystackID = gpc1Db.getStackIDFromSkyIDAndFilter( skychunk.dvoLabel, skyID, "y.00000") + # self.logger.infoPair("g stackID ",gstackID) + # self.logger.infoPair("r stackID ",rstackID) + # self.logger.infoPair("i stackID ",istackID) + # self.logger.infoPair("z stackID ",zstackID) + # self.logger.infoPair("y stackID ",ystackID) + # raise + # from each stackget fits name from getStackStageCmf + if gstackID > 0: + self.logger.infoPair ("finding fits file for: ","g.00000") + gfits = gpc1Db.getStackStageCmf(skychunk.dvoLabel, gstackID) + self.gfits = gfits + if rstackID > 0: + self.logger.infoPair ("finding fits file for: ","r.00000") + rfits = gpc1Db.getStackStageCmf(skychunk.dvoLabel, rstackID) + self.rfits = rfits + if istackID > 0: + self.logger.infoPair ("finding fits file for: ","i.00000") + ifits = gpc1Db.getStackStageCmf(skychunk.dvoLabel, istackID) + self.ifits = ifits + if zstackID > 0: + self.logger.infoPair ("finding fits file for: ","z.00000") + zfits = gpc1Db.getStackStageCmf(skychunk.dvoLabel, zstackID) + self.zfits = zfits + if ystackID > 0: + self.logger.infoPair ("finding fits file for: ","y.00000") + yfits = gpc1Db.getStackStageCmf(skychunk.dvoLabel, ystackID) + self.yfits = yfits + + #from each of those get self.gheader, self.rheader, etc. + if gstackID > 0: + if gfits: + gheader = gfits.getPrimaryHeader() + if not gheader: + self.logger.errorPair("could not read FITS PHU for ",gstackID) + raise + self.gheader = gheader + if rstackID > 0: + if rfits: + rheader = rfits.getPrimaryHeader() + if not rheader: + self.logger.errorPair("could not read FITS PHU for ",rstackID) + raise + self.rheader = rheader + if istackID > 0: + if ifits: + iheader = ifits.getPrimaryHeader() + if not iheader: + self.logger.errorPair("could not read FITS PHU for ",istackID) + raise + self.iheader = iheader + if zstackID > 0: + if zfits: + zheader = zfits.getPrimaryHeader() + if not zheader: + self.logger.errorPair("could not read FITS PHU for ",zstackID) + raise + self.zheader = zheader + if ystackID > 0: + if yfits: + yheader = yfits.getPrimaryHeader() + if not yheader: + self.logger.errorPair("could not read FITS PHU for ",ystackID) + raise + self.yheader = yheader # get stack meta data - meta = self.gpc1Db.getStackStageMeta(self.id) + self.gstackID = gstackID + self.rstackID = rstackID + self.istackID = istackID + self.zstackID = zstackID + self.ystackID = ystackID + + metastackid = -999 + if gstackID > 0: + metastackid = gstackID + if rstackID > 0: + metastackid = rstackID + if istackID > 0: + metastackid = istackID + if zstackID > 0: + metastackid = zstackID + if ystackID > 0: + metastackid = ystackID + #there is probably a better way to do this + if metastackid < 0: + self.logger.errorPair("can't find any stack ids...?? ","??") + raise + + meta = self.gpc1Db.getStackStageMeta(metastackid) + ## this should be ok - it takes a stack_id and returns filter/skycell etc. + + if not meta: self.logger.errorPair("Could not get stack", "metadata") raise - self.filter = meta[0]; - self.filter = self.filter[0:1] - self.filterID = self.scratchDb.getFilterID(self.filter) + #self.filter = meta[0]; + #self.filter = self.filter[0:1] + #self.filterID = self.scratchDb.getFilterID(self.filter) self.skycell = meta[1]; @@ -75,5 +180,5 @@ # we need 1133.081 for skycell, but store as an int, so instead store 1133081 # projection cell is just the 1133 bit - # this is how it should be done + # This Is how it should be done #self.projectioncell = self.skycell[8:12] #self.skycell = self.skycell[8:] @@ -82,5 +187,5 @@ # TODO this is the wrong way, but is consistent with SkyCell table in IN data - self.skycell = self.skycell[8:12] + self.skycell = self.skycell[8:] self.projectioncell = self.skycell @@ -94,37 +199,71 @@ #self.expTime = gpc1Db.getStackExpTime(self.id) - self.expTime = self.header['EXPTIME'] - + # delete PSPS tables - self.scratchDb.dropTable("StackMeta") - self.scratchDb.dropTable("StackDetection") + self.logger.infoPair("dropping table:","gStackMeta") + self.scratchDb.dropTable("gStackMeta") + self.logger.infoPair("dropping table:","rStackMeta") + self.scratchDb.dropTable("rStackMeta") + self.logger.infoPair("dropping table:","iStackMeta") + self.scratchDb.dropTable("iStackMeta") + self.logger.infoPair("dropping table:","zStackMeta") + self.scratchDb.dropTable("zStackMeta") + self.logger.infoPair("dropping table:","yStackMeta") + self.scratchDb.dropTable("yStackMeta") + self.logger.infoPair("dropping table:","StackObject") + + self.scratchDb.dropTable("StackObject") + self.logger.infoPair("dropping table:","StackModelFit") + self.scratchDb.dropTable("StackModelFit") + self.logger.infoPair("dropping table:","StackApFlx") + self.scratchDb.dropTable("StackApFlx") + self.logger.infoPair("dropping table:","StackToImage") + self.scratchDb.dropTable("StackToImage") - self.scratchDb.dropTable("SkinnyObject") - #self.scratchDb.dropTable("ObjectCalColor") - self.scratchDb.dropTable("StackDetectionCalib") - - # self.testprint() + # self.testprint() + + # for PV2 we will have (grizy) of stack meta and skychip + # delete IPP tables - self.scratchDb.dropTable("SkyChip_psf") - self.scratchDb.dropTable("SkyChip_xrad") - self.scratchDb.dropTable("SkyChip_xfit") - self.scratchDb.dropTable("SkyChip_xsrc") + self.scratchDb.dropTable("gSkyChip_psf") + self.scratchDb.dropTable("gSkyChip_xrad") + self.scratchDb.dropTable("gSkyChip_xfit") + self.scratchDb.dropTable("gSkyChip_xsrc") + self.scratchDb.dropTable("rSkyChip_psf") + self.scratchDb.dropTable("rSkyChip_xrad") + self.scratchDb.dropTable("rSkyChip_xfit") + self.scratchDb.dropTable("rSkyChip_xsrc") + self.scratchDb.dropTable("iSkyChip_psf") + self.scratchDb.dropTable("iSkyChip_xrad") + self.scratchDb.dropTable("iSkyChip_xfit") + self.scratchDb.dropTable("iSkyChip_xsrc") + self.scratchDb.dropTable("zSkyChip_psf") + self.scratchDb.dropTable("zSkyChip_xrad") + self.scratchDb.dropTable("zSkyChip_xfit") + self.scratchDb.dropTable("zSkyChip_xsrc") + self.scratchDb.dropTable("ySkyChip_psf") + self.scratchDb.dropTable("ySkyChip_xrad") + self.scratchDb.dropTable("ySkyChip_xfit") + self.scratchDb.dropTable("ySkyChip_xsrc") # create an output filename, which is {stackID}.FITS + self.logger.infoPair( "self.id", self.id) + self.logger.infoPair ("self.localOutPath",self.localOutPath) self.outputFitsFile = "%08d.FITS" % self.id self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) - + self.logger.infoPair("self.outputFitsFile",self.outputFitsFile) + # set some constants self.historyModNum = "0" # insert what we know about this stack batch into the stack table - self.ippToPspsDb.insertStackMeta(self.batchID, self.filter, self.stackType) + self.ippToPspsDb.insertStackMeta(self.batchID, "x" , self.stackType) # insert sourceID/imageID combo so DVO can look it up - if not self.useFullTables: - self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID']) + ##if not self.useFullTables: + ## self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID']) # self.testprint() @@ -135,5 +274,5 @@ self.logger.infoPair("Skycell", "%s" % self.skycell) self.logger.infoPair("Projectioncell", "%s" % self.projectioncell) - self.logger.infoPair("Filter", "%s" % self.filter) + ##self.logger.infoPair("Filter", "%s" % self.filter) ''' @@ -142,5 +281,10 @@ def updateStackMetaID(self, table): - sql = "UPDATE " + table + " SET stackMetaID=" + str(self.id) + sql = "UPDATE " + table + " SET gstackMetaID=" + str(self.gstackID) + " \ + , rstackMetaID=" + str(self.rstackID) + " \ + , istackMetaID=" + str(self.istackID) + " \ + , zstackMetaID=" + str(self.zstackID) + " \ + , ystackMetaID=" + str(self.ystackID) + self.scratchDb.execute(sql) @@ -161,55 +305,56 @@ Updates aperture fluxes for StackApFkx table ''' - def updateApFlxs(self, prefix, psfCondition): + def updateModelFitFlxs(self, prefix, psfCondition, tablename, filter): #properly -999 these to start with: - - sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \ - "+prefix+"flxR1=-999, \ - "+prefix+"flxR1Err=-999, \ - "+prefix+"flxR1Std=-999, \ - "+prefix+"flxR1Fill=-999, \ - "+prefix+"flxR2=-999, \ - "+prefix+"flxR2Err=-999, \ - "+prefix+"flxR2Std=-999, \ - "+prefix+"flxR2Fill=-999, \ - "+prefix+"flxR3=-999, \ - "+prefix+"flxR3Err=-999, \ - "+prefix+"flxR3Std=-999, \ - "+prefix+"flxR3Fill=-999, \ - "+prefix+"flxR4=-999, \ - "+prefix+"flxR4Err=-999, \ - "+prefix+"flxR4Std=-999, \ - "+prefix+"flxR4Fill=-999, \ - "+prefix+"flxR5=-999, \ - "+prefix+"flxR5Err=-999, \ - "+prefix+"flxR5Std=-999, \ - "+prefix+"flxR5Fill=-999, \ - "+prefix+"flxR6=-999, \ - "+prefix+"flxR6Err=-999, \ - "+prefix+"flxR6Std=-999, \ - "+prefix+"flxR6Fill=-999, \ - "+prefix+"flxR7=-999, \ - "+prefix+"flxR7Err=-999, \ - "+prefix+"flxR7Std=-999, \ - "+prefix+"flxR7Fill=-999, \ - "+prefix+"flxR8=-999, \ - "+prefix+"flxR8Err=-999, \ - "+prefix+"flxR8Std=-999, \ - "+prefix+"flxR8Fill=-999, \ - "+prefix+"flxR9=-999, \ - "+prefix+"flxR9Err=-999, \ - "+prefix+"flxR9Std=-999, \ - "+prefix+"flxR9Fill=-999, \ - "+prefix+"flxR10=-999, \ - "+prefix+"flxR10Err=-999, \ - "+prefix+"flxR10Std=-999, \ - "+prefix+"flxR10Fill=-999 \ - WHERE a.ippDetectID=b.IPP_IDET " + f = filter + if tablename == 'StackModelFit': + sql = "UPDATE " + tablename + " AS a, "+f+"SkyChip_xrad AS b SET \ + "+f+prefix+"flxR1=-999, \ + "+f+prefix+"flxR1Err=-999, \ + "+f+prefix+"flxR1Std=-999, \ + "+f+prefix+"flxR1Fill=-999, \ + "+f+prefix+"flxR2=-999, \ + "+f+prefix+"flxR2Err=-999, \ + "+f+prefix+"flxR2Std=-999, \ + "+f+prefix+"flxR2Fill=-999, \ + "+f+prefix+"flxR3=-999, \ + "+f+prefix+"flxR3Err=-999, \ + "+f+prefix+"flxR3Std=-999, \ + "+f+prefix+"flxR3Fill=-999, \ + "+f+prefix+"flxR4=-999, \ + "+f+prefix+"flxR4Err=-999, \ + "+f+prefix+"flxR4Std=-999, \ + "+f+prefix+"flxR4Fill=-999, \ + "+f+prefix+"flxR5=-999, \ + "+f+prefix+"flxR5Err=-999, \ + "+f+prefix+"flxR5Std=-999, \ + "+f+prefix+"flxR5Fill=-999, \ + "+f+prefix+"flxR6=-999, \ + "+f+prefix+"flxR6Err=-999, \ + "+f+prefix+"flxR6Std=-999, \ + "+f+prefix+"flxR6Fill=-999, \ + "+f+prefix+"flxR7=-999, \ + "+f+prefix+"flxR7Err=-999, \ + "+f+prefix+"flxR7Std=-999, \ + "+f+prefix+"flxR7Fill=-999, \ + "+f+prefix+"flxR8=-999, \ + "+f+prefix+"flxR8Err=-999, \ + "+f+prefix+"flxR8Std=-999, \ + "+f+prefix+"flxR8Fill=-999, \ + "+f+prefix+"flxR9=-999, \ + "+f+prefix+"flxR9Err=-999, \ + "+f+prefix+"flxR9Std=-999, \ + "+f+prefix+"flxR9Fill=-999, \ + "+f+prefix+"flxR10=-999, \ + "+f+prefix+"flxR10Err=-999, \ + "+f+prefix+"flxR10Std=-999, \ + "+f+prefix+"flxR10Fill=-999 \ + WHERE a."+f+"ippDetectID=b.IPP_IDET " #self.logger.infoPair("sql", sql) - self.scratchDb.execute(sql) + self.scratchDb.execute(sql) #we have variable numbers of these columns. find out which are in use - sql = "DESCRIBE SkyChip_xrad 'aper_fill_%'" + sql = "DESCRIBE "+f+"SkyChip_xrad 'aper_fill_%'" rs = self.scratchDb.executeQuery(sql) rs.first() @@ -225,155 +370,73 @@ numbers = [] + min = 1 + max = 10 + if tablename == 'StackApFlx': + min = 3 + max = 5 for column in columns: number = re.sub("APER_FILL_","",column) - if (int(number) >= 1 and int(number) <=10): + if (int(number) >= min and int(number) <=max): numbers.append(number) self.logger.infoPair("Aperture Numbers",number) #generate the sql to do the necessary ops on the columns - sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET " + sql = "UPDATE " + tablename + " AS a, "+f+"SkyChip_xrad AS b SET " cnt =0 + exptime = self.safeDictionaryAccess(self.header, 'EXPTIME') for number in numbers: if (cnt>0): sql=sql+" , " - sql=sql+prefix+"flxR"+number+"=b.APER_FLUX_"+number +" / " + str(self.expTime) + ", " - sql=sql+prefix+"flxR"+number+"Err=b.APER_FLUX_ERR_"+number +" / " + str(self.expTime) + ", " - sql=sql+prefix+"flxR"+number+"Std=b.APER_FLUX_STDEV_"+number +" / " + str(self.expTime) + ", " - sql=sql+prefix+"flxR"+number+"Fill=b.APER_FILL_"+number + " " + sql=sql+f+prefix+"flxR"+number+"=b.APER_FLUX_"+number +" / " + str(exptime) + ", " + sql=sql+f+prefix+"flxR"+number+"Err=b.APER_FLUX_ERR_"+number +" / " + str(exptime) + ", " + sql=sql+f+prefix+"flxR"+number+"Std=b.APER_FLUX_STDEV_"+number +" / " + str(exptime) + ", " + sql=sql+f+prefix+"flxR"+number+"Fill=b.APER_FILL_"+number + " " cnt=cnt+1 - sql=sql+"WHERE a.ippDetectID=b.IPP_IDET AND "+psfCondition - self.scratchDb.execute(sql) - - #self.logger.infoPair("sql",sql) - - #sql = "UPDATE StackApFlx AS a, SkyChip_xrad AS b SET \ - #"+prefix+"flxR1=b.APER_FLUX_1 / " + str(self.expTime) + ", \ - #"+prefix+"flxR1Err=b.APER_FLUX_ERR_1 / " + str(self.expTime) + ", \ - #"+prefix+"flxR1Std=b.APER_FLUX_STDEV_1 / " + str(self.expTime) + ", \ - #"+prefix+"flxR1Fill=b.APER_FILL_1, \ - #"+prefix+"flxR2=b.APER_FLUX_2 / " + str(self.expTime) + ", \ - ##"+prefix+"flxR2Err=b.APER_FLUX_ERR_2 / " + str(self.expTime) + ", \ - #"+prefix+"flxR2Std=b.APER_FLUX_STDEV_2 / " + str(self.expTime) + ", \ - #"+prefix+"flxR2Fill=b.APER_FILL_2, \ - #"+prefix+"flxR3=b.APER_FLUX_3 / " + str(self.expTime) + ", \ - #"+prefix+"flxR3Err=b.APER_FLUX_ERR_3 / " + str(self.expTime) + ", \ - #"+prefix+"flxR3Std=b.APER_FLUX_STDEV_3 / " + str(self.expTime) + ", \ - #"+prefix+"flxR3Fill=b.APER_FILL_3, \ - #"+prefix+"flxR4=b.APER_FLUX_4 / " + str(self.expTime) + ", \ - #"+prefix+"flxR4Err=b.APER_FLUX_ERR_4 / " + str(self.expTime) + ", \ - #"+prefix+"flxR4Std=b.APER_FLUX_STDEV_4 / " + str(self.expTime) + ", \ - #"+prefix+"flxR4Fill=b.APER_FILL_4, \ - #"+prefix+"flxR5=b.APER_FLUX_5 / " + str(self.expTime) + ", \ - #"+prefix+"flxR5Err=b.APER_FLUX_ERR_5 / " + str(self.expTime) + ", \ - #"+prefix+"flxR5Std=b.APER_FLUX_STDEV_5 / " + str(self.expTime) + ", \ - #"+prefix+"flxR5Fill=b.APER_FILL_5, \ - #"+prefix+"flxR6=b.APER_FLUX_6 / " + str(self.expTime) + ", \ - #"+prefix+"flxR6Err=b.APER_FLUX_ERR_6 / " + str(self.expTime) + ", \ - #"+prefix+"flxR6Std=b.APER_FLUX_STDEV_6 / " + str(self.expTime) + ", \ - #"+prefix+"flxR6Fill=b.APER_FILL_6, \ - #"+prefix+"flxR7=b.APER_FLUX_7 / " + str(self.expTime) + ", \ - #"+prefix+"flxR7Err=b.APER_FLUX_ERR_7 / " + str(self.expTime) + ", \ - #"+prefix+"flxR7Std=b.APER_FLUX_STDEV_7 / " + str(self.expTime) + ", \ - #"+prefix+"flxR7Fill=b.APER_FILL_7, \ - #"+prefix+"flxR8=b.APER_FLUX_8 / " + str(self.expTime) + ", \ - #"+prefix+"flxR8Err=b.APER_FLUX_ERR_8 / " + str(self.expTime) + ", \ - #"+prefix+"flxR8Std=b.APER_FLUX_STDEV_8 / " + str(self.expTime) + ", \ - #"+prefix+"flxR8Fill=b.APER_FILL_8, \ - #"+prefix+"flxR9=b.APER_FLUX_9 / " + str(self.expTime) + ", \ - #"+prefix+"flxR9Err=b.APER_FLUX_ERR_9 / " + str(self.expTime) + ", \ - #"+prefix+"flxR9Std=b.APER_FLUX_STDEV_9 / " + str(self.expTime) + ", \ - #"+prefix+"flxR9Fill=b.APER_FILL_9, \ - #"+prefix+"flxR10=b.APER_FLUX_10 / " + str(self.expTime) + ", \ - #"+prefix+"flxR10Err=b.APER_FLUX_ERR_10 / " + str(self.expTime) + ", \ - #"+prefix+"flxR10Std=b.APER_FLUX_STDEV_10 / " + str(self.expTime) + ", \ - #"+prefix+"flxR10Fill=b.APER_FILL_10 \ - #WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition - - #self.scratchDb.execute(sql) + sql=sql+"WHERE a."+f+"ippDetectID=b.IPP_IDET AND "+psfCondition + self.logger.infoPair("sql",sql) + self.scratchDb.execute(sql) + ''' Updates parameters for a particular model in the StackModelFit table ''' - def updateModelFit(self, model, ippModelType): - - if model == "ser": - modelLong = "sersic" - else: - modelLong = model; - - #does it have the EXT_COVAR in there? - - sql = "DESCRIBE SkyChip_xfit 'EXT_COVAR_%'" + def updateModelFit(self, model, ippModelType, filter, exptime): + f = filter + + # idk what to do about PET + + sql = "DESCRIBE "+f+"SkyChip_xfit 'EXT_COVAR_%'" rs = self.scratchDb.executeQuery(sql) rs.first() hasResult = rs.next() - if hasResult: - print "EXT_COVARs exist" - else: - print "No EXT_COVARS here" - - ext_covar_sql = "" - - if (hasResult): - ext_covar_sql = ", "+modelLong+"Covar11=b.EXT_COVAR_00_00, \ - "+modelLong+"Covar12=b.EXT_COVAR_00_01, \ - "+modelLong+"Covar13=b.EXT_COVAR_00_02, \ - "+modelLong+"Covar14=b.EXT_COVAR_00_03, \ - "+modelLong+"Covar15=b.EXT_COVAR_00_04, \ - "+modelLong+"Covar16=b.EXT_COVAR_00_05, \ - "+modelLong+"Covar17=b.EXT_COVAR_00_06, \ - "+modelLong+"Covar22=b.EXT_COVAR_01_01, \ - "+modelLong+"Covar23=b.EXT_COVAR_01_02, \ - "+modelLong+"Covar24=b.EXT_COVAR_01_03, \ - "+modelLong+"Covar25=b.EXT_COVAR_01_04, \ - "+modelLong+"Covar26=b.EXT_COVAR_01_05, \ - "+modelLong+"Covar27=b.EXT_COVAR_01_06, \ - "+modelLong+"Covar33=b.EXT_COVAR_02_02, \ - "+modelLong+"Covar34=b.EXT_COVAR_02_03, \ - "+modelLong+"Covar35=b.EXT_COVAR_02_04, \ - "+modelLong+"Covar36=b.EXT_COVAR_02_05, \ - "+modelLong+"Covar37=b.EXT_COVAR_02_06, \ - "+modelLong+"Covar44=b.EXT_COVAR_03_03, \ - "+modelLong+"Covar45=b.EXT_COVAR_03_04, \ - "+modelLong+"Covar46=b.EXT_COVAR_03_05, \ - "+modelLong+"Covar47=b.EXT_COVAR_03_06, \ - "+modelLong+"Covar55=b.EXT_COVAR_04_04, \ - "+modelLong+"Covar56=b.EXT_COVAR_04_05, \ - "+modelLong+"Covar57=b.EXT_COVAR_04_06, \ - "+modelLong+"Covar66=b.EXT_COVAR_05_05, \ - "+modelLong+"Covar67=b.EXT_COVAR_05_06, \ - "+modelLong+"Covar77=b.EXT_COVAR_06_06" - - - - - sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ - "+model+"Radius=b.EXT_WIDTH_MAJ, \ - "+model+"Flux=POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " , \ - "+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(self.expTime) + " / 1.085736, \ - "+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ - "+model+"Phi=b.EXT_THETA \ - "+ext_covar_sql+" \ - WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" + + + + sql = "UPDATE StackModelFit AS a, " + f + "SkyChip_xfit AS b SET \ + "+f+model+"Radius=b.EXT_WIDTH_MAJ, \ + "+f+model+"Flux=POW(10, -0.4 * b.EXT_INST_MAG) / " + str(exptime) + " , \ + "+f+model+"FluxErr=ABS(b.EXT_INST_MAG_SIG) * POW(10, -0.4 * b.EXT_INST_MAG) / " + str(exptime) + " / 1.085736, \ + "+f+model+"Ab=b.EXT_WIDTH_MAJ/b.EXT_WIDTH_MIN, \ + "+f+model+"Phi=b.EXT_THETA \ + \ + WHERE a."+f+"ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" print sql self.scratchDb.execute(sql) + + # sersic fit has an extra parameter - if (ippModelType == "PS_MODEL_SERSIC" and hasResult > 0): - sql = "UPDATE StackModelFit AS a, SkyChip_xfit AS b SET \ - "+modelLong+"Covar18=b.EXT_COVAR_00_07, \ - "+modelLong+"Covar28=b.EXT_COVAR_01_07, \ - "+modelLong+"Covar38=b.EXT_COVAR_02_07, \ - "+modelLong+"Covar48=b.EXT_COVAR_03_07, \ - "+modelLong+"Covar58=b.EXT_COVAR_04_07, \ - "+modelLong+"Covar68=b.EXT_COVAR_05_07, \ - "+modelLong+"Covar78=b.EXT_COVAR_06_07, \ - "+modelLong+"Covar88=b.EXT_COVAR_07_07, \ - "+"serNu=b.EXT_PAR_07, \ - "+"serNuErr=SQRT(EXT_COVAR_07_07) \ - WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - + if (ippModelType == "PS_MODEL_SERSIC"): + extra_sql = "" + if hasResult: + extra_sql = ","+f+"serNuErr=SQRT(EXT_COVAR_07_07)" + sql = "UPDATE StackModelFit AS a, "+ f+ "SkyChip_xfit AS b SET \ + "+f+"serNu=b.EXT_PAR_07 \ + "+extra_sql+" \ + WHERE a."+f+"ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" +# "+f+"serNuErr=SQRT(EXT_COVAR_07_07) \ + self.logger.infoPair("sql",sql) self.scratchDb.execute(sql) @@ -382,7 +445,34 @@ Populates the StackMeta table, mainly from dictionary values found in IPP FITS header ''' - def populateStackMeta(self): + def populateStackMeta(self,filterName): self.logger.infoPair("Procesing table", "StackMeta") + + # this should work : set header into (grizy)header and run from there + if filterName == 'g.00000': + f = "g" + self.header = self.gheader + tablename = 'gStackMeta' + stackmetaid = self.gstackID + if filterName == 'r.00000': + f = "r" + tablename = 'rStackMeta' + self.header = self.rheader + stackmetaid = self.rstackID + if filterName == 'i.00000': + f = "i" + tablename = 'iStackMeta' + self.header = self.iheader + stackmetaid = self.istackID + if filterName == 'z.00000': + f = "z" + tablename = 'zStackMeta' + self.header = self.zheader + stackmetaid = self.zstackID + if filterName == 'y.00000': + f = "y" + tablename = 'yStackMeta' + self.header = self.yheader + stackmetaid = self.ystackID self.fwhm_maj = self.safeDictionaryAccess(self.header, 'FWHM_MAJ') @@ -396,7 +486,17 @@ # print "psfmodel = ", self.psfmodel - sql = "INSERT INTO StackMeta (\ + #make a table + + filterid = self.scratchDb.getFilterID(f) + self.logger.infoPair("filterid is ",str(filterid)) + + sql = "CREATE TABLE " + tablename + " LIKE StackMeta" + try: self.scratchDb.execute(sql) + except: pass + + sql = "INSERT INTO " + tablename + " (\ stackMetaID \ - ,skycellID \ + ,filterID \ + ,skyCellID \ ,photoCalID \ ,photoZero \ @@ -418,9 +518,10 @@ ,pc002002 \ ) VALUES ( \ - " + str(self.id) + " \ + " + str(stackmetaid) + " \ + , " + str(filterid) + " \ ," + self.skycell + " \ ," + str(self.scratchDb.getPhotoCalID(self.header['IMAGEID'])) + " \ ," + self.header['FPA.ZP'] + " \ - ," + self.expTime + " \ + ," + self.header['EXPTIME'] + " \ ,'" + self.psfmodel + "' \ ," + str(self.fwhm_maj) + " \ @@ -440,137 +541,158 @@ )" self.scratchDb.execute(sql) - self.scratchDb.updateAllRows("StackMeta", "batchID", str(self.batchID)) - self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) - self.scratchDb.updateFilterID("StackMeta", self.filter) - self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.skychunk.dataRelease)) - self.updateStackTypeID("StackMeta") - - ''' - Populates the StackDetection table - ''' - def populateStackDetection(self): - - self.logger.infoPair("Procesing table", "StackDetection") - - # insert all the detections - sql = "INSERT INTO StackDetection (\ - ippDetectID \ - ,skyCellID \ - ,projectionCellID \ - ,xPos \ - ,yPos \ - ,xPosErr \ - ,yPosErr \ - ,psfFlux \ - ,psfFluxErr \ - ,peakFlux \ - ,sky \ - ,skyErr \ - ,sgSep \ - ,psfWidMajor \ - ,psfWidMinor \ - ,psfTheta \ - ,psfLikelihood \ - ,infoFlag \ - ,psfQf \ - ,psfQfPerfect \ - ,psfChiSq \ - ,momentXX \ - ,momentXY \ - ,momentYY \ - ,momentM3C \ - ,momentM3S \ - ,momentM4C \ - ,momentM4S \ - ,momentR1 \ - ,momentRH \ - ,apFlux \ - ,apFluxErr \ - ,kronFlux \ - ,kronFluxErr \ - ,kronRad \ - ,nFrames \ - ,assocDate \ - ,historyModNum \ - ) \ - SELECT \ - IPP_IDET \ - ," + self.skycell + " \ - ," + self.projectioncell + " \ - ,X_PSF \ - ,Y_PSF \ - ,X_PSF_SIG \ - ,Y_PSF_SIG \ - ,PSF_INST_FLUX / " + str(self.expTime) + " \ - ,PSF_INST_FLUX_SIG / " + str(self.expTime) + " \ - ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / " + str(self.expTime) + " \ - ,SKY / " + str(self.expTime) + " \ - ,SKY_SIGMA / " + str(self.expTime) + " \ - ,EXT_NSIGMA \ - ,PSF_MAJOR \ - ,PSF_MINOR \ - ,PSF_THETA \ - ,psfLikelihood(EXT_NSIGMA) \ - , FLAGS2 << 32 | FLAGS \ - ,PSF_QF \ - ,PSF_QF_PERFECT \ - ,PSF_CHISQ / PSF_NDOF \ - ,MOMENTS_XX \ - ,MOMENTS_XY \ - ,MOMENTS_YY \ - ,MOMENTS_M3C \ - ,MOMENTS_M3S \ - ,MOMENTS_M4C \ - ,MOMENTS_M4S \ - ,MOMENTS_R1 \ - ,MOMENTS_RH \ - ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " \ - ,NULL \ - ,KRON_FLUX / " + str(self.expTime) + " \ - ,KRON_FLUX_ERR / " + str(self.expTime) + " \ - ,MOMENTS_R1 * 2.5 \ - ,N_FRAMES \ - , '" + self.dateStr + "' \ - ," + self.historyModNum + " \ - FROM SkyChip_psf" - -# these were used above (left over from before PSF_INST_FLUX, PSF_INST_FLUX_SIG were available? -# ,POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + " -# ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / " + str(self.expTime) + ")) / 1.085736) -# ,POW(10.0, -0.4*AP_MAG) / " + str(self.expTime) + " - -# I need a way to choose the sql above based on the cmf version: V3 (LAP.PV1) does not have AP_FLUX -# ,AP_FLUX / " + str(self.expTime) + " - - # print "sql: ", sql - # response = raw_input("ready to insert stack det ") - - self.scratchDb.execute(sql) - - #it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid + + + self.scratchDb.updateAllRows(tablename, "batchID", str(self.batchID)) + self.scratchDb.updateAllRows(tablename, "surveyID", str(self.surveyID)) + # self.scratchDb.updateFilterID(tablename, self.scratchDb.getFilterID(filterName)) + self.scratchDb.updateAllRows(tablename, "dataRelease", str(self.skychunk.dataRelease)) + self.updateStackTypeID(tablename) + + ''' + Populates the StackObject table + ''' + def populateStackObject(self): + + self.logger.infoPair("Procesing table", "StackObject") + + self.logger.infoPair("inserting non-filter dependent items into", "StackObject") + + #this is dvo crap + #sql = "INSERT INTO StackObject(\ + # objID, \ + # gcobjID, \ + # ippObjID, \ + # surveyID, \ + # skyCellID, \ + # randomStackObjID, \ + # stackDetectRowID, \ + # dvoRegionID, \ + # dataRelease) SELECT \ + # -999, + # -999, + # -999, + # , + + + + + self.logger.infoPair("inserting filter dependent cmf items into", "StackObject") + filters =['g','r','i','z','y'] + for f in filters: + skip = 0 + if (f == "g"): + self.logger.infoPair("gstackID = ",self.gstackID) + if self.gstackID > 0: + stackid = self.gstackID + exptime = self.gheader['EXPTIME'] + else: + skip = 1 + if (f == "r"): + self.logger.infoPair("rstackID = ",self.rstackID) + if self.rstackID > 0: + stackid = self.rstackID + exptime = self.rheader['EXPTIME'] + else: + skip = 1 + if (f == "i"): + self.logger.infoPair("istackID = ",self.istackID) + if self.istackID > 0: + stackid = self.istackID + exptime = self.iheader['EXPTIME'] + else: + skip = 1 + if (f == "z"): + self.logger.infoPair("zstackID = ",self.zstackID) + if self.zstackID > 0: + stackid = self.zstackID + exptime = self.zheader['EXPTIME'] + else: + skip = 1 + if (f == "y"): + self.logger.infoPair("ystackID = ",self.ystackID) + if self.ystackID > 0: + stackid = self.ystackID + exptime = self.yheader['EXPTIME'] + else: + skip = 1 + + if skip == 1: + self.logger.infoPair("nothing for filter" , f) + else: + self.logger.infoPair("filter",f) + + + + # insert all the detections + sql = "UPDATE StackObject as a , " + f + "SkyChip_psf as b set \ + a."+f+"xPos = b.X_PSF \ + ,a."+f+"yPos = b.Y_PSF \ + ,a."+f+"xPosErr = b.X_PSF_SIG \ + ,a."+f+"yPosErr = b.Y_PSF_SIG \ + ,a."+f+"PSFFlux = b.PSF_INST_FLUX / " + str(exptime) + " \ + ,a."+f+"PSFFluxErr = b.PSF_INST_FLUX_SIG / " + str(exptime) + " \ + ,a."+f+"sky = b.SKY / " + str(exptime) + " \ + ,a."+f+"skyErr = b.SKY_SIGMA / " + str(exptime) + " \ + ,a."+f+"psfMajorFWHM = b.PSF_MAJOR \ + ,a."+f+"psfMinorFWHM = b.PSF_MINOR \ + ,a."+f+"psfTheta = b.PSF_THETA \ + ,a."+f+"psfCore = NULL \ + ,a."+f+"psfLikelihood = psfLikelihood(b.EXT_NSIGMA) \ + ,a."+f+"infoFlag = b.FLAGS2 << 32 | FLAGS \ + ,a."+f+"psfQf = b.PSF_QF \ + ,a."+f+"psfQfPerfect = b.PSF_QF_PERFECT \ + ,a."+f+"psfChiSq = b.PSF_CHISQ / b.PSF_NDOF \ + ,a."+f+"momentXX = b.MOMENTS_XX \ + ,a."+f+"momentXY = b.MOMENTS_XY \ + ,a."+f+"momentYY = b.MOMENTS_YY \ + ,a."+f+"momentR1 = b.MOMENTS_R1 \ + ,a."+f+"momentRH = b.MOMENTS_RH \ + ,a."+f+"ApFlux = POW(10.0, -0.4*b.AP_MAG) / " + str(exptime) + " \ + ,a."+f+"ApFluxErr = NULL \ + ,a."+f+"ApFillFac = NULL \ + ,a."+f+"KronFlux = b.KRON_FLUX / " + str(exptime) + " \ + ,a."+f+"KronFluxErr = b.KRON_FLUX_ERR / " + str(exptime) + " \ + ,a."+f+"KronRad = b.MOMENTS_R1 * 2.5 \ + ,a."+f+"PSFMag = b.PSF_INST_MAG \ + ,a."+f+"PSFMagErr = b.PSF_INST_MAG_SIG \ + ,a."+f+"ApMag = b.AP_MAG \ + ,a."+f+"ApMagErr = NULL \ + ,a."+f+"KronMag = NULL \ + ,a."+f+"KronMagErr = NULL \ + ,a."+f+"nFrames = b. N_FRAMES \ + WHERE a." + f + "ippDetectID=b.IPP_IDET " + self.logger.infoPair('sql', sql) + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + #it is possible to drop some detections from dvo (that are present in the cmf). when that happens we get a 0 for objid #we drop those... - sql="DELETE FROM StackDetection where objID = 0" - self.scratchDb.execute(sql) - self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0") - - # XXX EAM : this seems quite inefficient : these commands use updates to set - # fields which are constant across all rows. Why not just include these in - # the original insert above? - - self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) - - self.scratchDb.updateFilterID("StackDetection", self.filter) - - self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.skychunk.dataRelease)) - #set primary F to 0 - self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") - - self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") - - self.updateStackMetaID("StackDetection") - - self.updateStackTypeID("StackDetection") - - self.updateDvoIDsAndFlags("StackDetection") + + self.logger.infoPair("Deleting", "entries with StackObject.objID = 0") + sql="DELETE FROM StackObject where objID = 0" + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair('failed sql',sql) + return + + try: + self.scratchDb.updateAllRows("StackObject", "surveyID", str(self.surveyID)) + except: + self.logger.errorPair('failed update surveyID on','StackObject') + raise +# self.scratchDb.updateFilterID("StackObject", f+".00000") + + self.scratchDb.updateAllRows("StackObject", "dataRelease", str(self.skychunk.dataRelease)) + try: + self.updateStackMetaID("StackObject") + except: + self.logger.errorPair("failed update stack meta id on ", "StackObject") + raise + ##self.updateStackTypeID("StackObject") + + #self.updateDvoIDsAndFlags("StackObject") #this now updates primary F as well # response = raw_input("updated dvo ") @@ -585,5 +707,5 @@ self.scratchDb.execute("set session old_alter_table=1") - sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)" + sql = "ALTER IGNORE TABLE StackObject ADD PRIMARY KEY (objID)" if self.scratchDb.version > 5.1: @@ -595,7 +717,7 @@ ### XXX this should not be a DEEP_STACK only case... if False and self.stackType == "DEEP_STACK": - + ## need to do for all filters? #if deep stack and instFlux = null and err not null - sql = "UPDATE StackDetection AS a, SkyChip_psf AS b \ + sql = "UPDATE StackObject AS a, SkyChip_psf AS b \ SET psfFlux = 2*b.PSF_INST_FLUX_SIG / " + str(self.expTime) + " \ WHERE psfFlux IS NULL \ @@ -605,28 +727,61 @@ self.scratchDb.execute(sql) # response = raw_input("add psf flux ") - + + #filter dependent # NOTE : Flux limits : in the current PSPS schema, negative fluxes # cause problems for sql queries which work in mags as SQL cannot do # something like (f < 0.0) ? -999 : -2.5*log10(f) # as a result, the negative fluxes here result in floating point errors - sql = "UPDATE StackDetection SET psfFlux = 1e20 WHERE psfFlux <= 0.0" - self.scratchDb.execute(sql) - - sql = "UPDATE StackDetection SET apFlux = 1e20 WHERE apFlux <= 0.0" - self.scratchDb.execute(sql) - - sql = "UPDATE StackDetection SET kronFlux = 1e20 WHERE kronFlux <= 0.0" + sql = "UPDATE StackObject SET gPSFFlux = 1e20 WHERE gPSFFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET gApFlux = 1e20 WHERE gApFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET gKronFlux = 1e20 WHERE gKronFlux <= 0.0" + self.scratchDb.execute(sql) + sql = "UPDATE StackObject SET rPSFFlux = 1e20 WHERE rPSFFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET rApFlux = 1e20 WHERE rApFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET rKronFlux = 1e20 WHERE rKronFlux <= 0.0" + self.scratchDb.execute(sql) + sql = "UPDATE StackObject SET iPSFFlux = 1e20 WHERE iPSFFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET iApFlux = 1e20 WHERE iApFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET iKronFlux = 1e20 WHERE iKronFlux <= 0.0" + self.scratchDb.execute(sql) + sql = "UPDATE StackObject SET zPSFFlux = 1e20 WHERE zPSFFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET zApFlux = 1e20 WHERE zApFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET zKronFlux = 1e20 WHERE zKronFlux <= 0.0" + self.scratchDb.execute(sql) + sql = "UPDATE StackObject SET yPSFFlux = 1e20 WHERE yPSFFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET yApFlux = 1e20 WHERE yApFlux <= 0.0" + self.scratchDb.execute(sql) + + sql = "UPDATE StackObject SET yKronFlux = 1e20 WHERE yKronFlux <= 0.0" self.scratchDb.execute(sql) #leave null instflux in - #self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") - - self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") + #self.scratchDb.reportAndDeleteRowsWithNULLS("StackObject", "instFlux") + + self.scratchDb.reportAndDeleteRowsWithNULLS("StackObject", "objID") # response = raw_input("delete nulls ") - sql="DELETE FROM StackDetection where objID = 0" - - self.scratchDb.execute(sql) - self.logger.infoPair("Deleting", "entries with StackDetection.objID = 0") + sql="DELETE FROM StackObject where objID = 0" + + self.scratchDb.execute(sql) + self.logger.infoPair("Deleting", "entries with StackObject.objID = 0") # response = raw_input("deleted objID is 0 ") @@ -636,78 +791,46 @@ ''' def populateStackApFlx(self): - self.logger.infoPair("getting","imageID") - imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) - self.logger.infoPair("obtained","imageID") - - self.logger.infoPair("Procesing", "StackApFlx") - - sql = "INSERT INTO StackApFlx \ - (ippDetectID ) \ - SELECT \ - DISTINCT IPP_IDET \ - FROM SkyChip_xrad" - - try: - self.scratchDb.execute(sql) - except: return - - # TODO temporarily loading 1st convolved fluxes into unconvolved fields - - #This assumes that there are 3 different fluxs, the first is unconv, and the 2nd and 3rd are convolved. If there are more than 3 fluxes it will break. - # If there are more than 3 types of fluxes, we should do a count to find out how many there are and adjust this accordingly (make it more robust) - - self.logger.infoPair("Adding fluxes", "un-convolved") - self.updateApFlxs("", "((b.table_index -1) % 3 = 0)") - self.logger.infoPair("Adding fluxes", "1st convolved") - self.updateApFlxs("c1", "((b.table_index -1) % 3 = 1)") - self.logger.infoPair("Adding fluxes", "2nd convolved") - self.updateApFlxs("c2", "((b.table_index -1) % 3 = 2)") - - self.logger.infoPair("Adding", "petrosians for extended sources") - sql = "UPDATE StackApFlx AS a, SkyChip_xsrc AS b SET \ - petRadius=b.PETRO_RADIUS \ - ,petRadiusErr=b.PETRO_RADIUS_ERR \ - ,petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.expTime) + " \ - ,petFluxErr=-999 \ - ,petR50=b.PETRO_RADIUS_50 \ - ,petR50Err=b.PETRO_RADIUS_50_ERR \ - ,petR90=b.PETRO_RADIUS_90 \ - ,petR90Err=b.PETRO_RADIUS_90_ERR \ - ,petCf=b.PETRO_FILL \ - WHERE a.ippDetectID=b.IPP_IDET" - self.scratchDb.execute(sql) - - self.logger.infoPair("Adding", "petFluxErr for non nulls") - sql = "UPDATE StackApFlx as a, SkyChip_xsrc AS b SET \ - petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.expTime) + " / 1.085736 \ - WHERE a.ippDetectID=b.IPP_IDET \ - AND (b.PETRO_MAG_ERR is not NULL AND b.PETRO_MAG is not NULL AND PETRO_MAG_ERR < 1000.0)" - # added this because when the petromagerr is crazy large (10^29 it out of ranges the flux err calc) - self.logger.infoPair("sql", sql) - self.scratchDb.execute(sql) - - - - self.logger.infoPair("adding"," PrimaryF to stackApFlx") - sql = "UPDATE StackApFlx AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ - a.primaryF = ((b.flags & 0x10000) >> 16 ) \ - WHERE a.ippDetectID = b.ippDetectID \ - AND b.imageID = " + str(imageID) - - print "update dvo sql: ", sql - # response = raw_input("update dvo...") - self.scratchDb.execute(sql) - - - - self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) - self.scratchDb.updateFilterID("StackApFlx", self.filter) - self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.skychunk.dataRelease)) - self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") - self.updateStackMetaID("StackApFlx") - self.updateStackTypeID("StackApFlx") - self.updateDvoIDs("StackApFlx") + filter = ['g','r','i','z','y'] + + for f in filter: + skip = 0 + if (f == 'g'): + if self.gstackID > 0: + self.header = self.gheader + else: + skip = 1 + if (f == 'r'): + if self.rstackID > 0: + self.header = self.rheader + else: + skip = 1 + if (f == 'i'): + if self.istackID > 0: + self.header = self.iheader + else: + skip = 1 + if (f == 'z'): + if self.zstackID > 0: + self.header = self.zheader + else: + skip = 1 + if (f == 'y'): + if self.ystackID > 0: + self.header = self.yheader + else: + skip =1 + + if skip == 0: + self.logger.infoPair("Adding fluxes", "un-convolved") + self.updateModelFitFlxs("", "((b.table_index -1) % 3 = 0)", "StackApFlx", f) + self.logger.infoPair("Adding fluxes", "1st convolved") + self.updateModelFitFlxs("c1", "((b.table_index -1) % 3 = 1)","StackApFlx",f) + self.logger.infoPair("Adding fluxes", "2nd convolved") + self.updateModelFitFlxs("c2", "((b.table_index -1) % 3 = 2)","StackApFlx",f) + + self.deleteDetectionsNotInStackObject("StackApFlx",f) + self.scratchDb.reportAndDeleteRowsWithNULLS("StackApFlx", "objID") - self.deleteDetectionsNotInStackDetection("StackApFlx") + @@ -715,5 +838,23 @@ Populates the StackModelFit table ''' - def populateStackModelFit(self): + def populateStackModelFit(self, filter): + # for each filter + f =filter + if (filter == 'g'): + self.header = self.gheader + + if (filter == 'r'): + self.header = self.rheader + if (filter == 'i'): + self.header = self.iheader + if (filter == 'z'): + self.header = self.zheader + if (filter == 'y'): + self.header = self.yheader + + + + + self.logger.infoPair("getting","imageID") imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) @@ -723,23 +864,51 @@ # insert all the detections - sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit" - try: - self.scratchDb.execute(sql) - except: - return # populate model parameters self.logger.infoPair("Adding model fit", "deVaucouleurs") - self.updateModelFit("deV", "PS_MODEL_DEV") + self.updateModelFit("deV", "PS_MODEL_DEV", filter, self.header['EXPTIME']) self.logger.infoPair("Adding model fit", "exponential") - self.updateModelFit("exp", "PS_MODEL_EXP") + self.updateModelFit("deV", "PS_MODEL_EXP", filter, self.header['EXPTIME']) self.logger.infoPair("Adding model fit", "sersic") - self.updateModelFit("ser", "PS_MODEL_SERSIC") + self.updateModelFit("deV", "PS_MODEL_SERSIC", filter, self.header['EXPTIME']) + + + self.logger.infoPair("Adding fluxes", "un-convolved") + self.updateModelFitFlxs("", "((b.table_index -1) % 3 = 0)","StackModelFit",f) + self.logger.infoPair("Adding fluxes", "1st convolved") + self.updateModelFitFlxs("c1", "((b.table_index -1) % 3 = 1)","StackModelFit",f) + self.logger.infoPair("Adding fluxes", "2nd convolved") + self.updateModelFitFlxs("c2", "((b.table_index -1) % 3 = 2)","StackModelFit",f) + + + self.logger.infoPair("Adding", "petrosians for extended sources") + sql = "UPDATE StackModelFit as a, "+f+"SkyChip_xsrc AS b SET \ + "+f+"petRadius=b.PETRO_RADIUS \ + ,"+f+"petRadiusErr=b.PETRO_RADIUS_ERR \ + ,"+f+"petFlux=POW(10.0, -0.4 * b.PETRO_MAG) /" + str(self.header['EXPTIME']) + " \ + ,"+f+"petFluxErr=-999 \ + ,"+f+"petR50=b.PETRO_RADIUS_50 \ + ,"+f+"petR50Err=b.PETRO_RADIUS_50_ERR \ + ,"+f+"petR90=b.PETRO_RADIUS_90 \ + ,"+f+"petR90Err=b.PETRO_RADIUS_90_ERR \ + ,"+f+"petCf=b.PETRO_FILL \ + WHERE a."+f+"ippDetectID=b.IPP_IDET" + self.scratchDb.execute(sql) + + self.logger.infoPair("Adding", "petFluxErr for non nulls") + sql = "UPDATE StackModelFit as a, "+f+"SkyChip_xsrc AS b SET \ + "+f+"petFluxErr=ABS(b.PETRO_MAG_ERR) * POW(10.0, (-0.4*b.PETRO_MAG)) / " + str(self.header['EXPTIME']) + " / 1.085736 \ + WHERE a."+f+"ippDetectID=b.IPP_IDET \ + AND (b.PETRO_MAG_ERR is not NULL AND b.PETRO_MAG is not NULL AND PETRO_MAG_ERR < 1000.0)" + # added this because when the petromagerr is crazy large (10^29 it out of ranges the flux err calc) + self.logger.infoPair("sql", sql) + self.scratchDb.execute(sql) + + self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) - self.scratchDb.updateFilterID("StackModelFit", self.filter) self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.skychunk.dataRelease)) - self.logger.infoPair("adding"," PrimaryF to stackApFlx") + self.logger.infoPair("not adding"," PrimaryF to stackApFlx") sql = "UPDATE StackModelFit AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ a.primaryF = ((b.flags & 0x10000) >> 16 ) \ @@ -747,24 +916,23 @@ AND b.imageID = " + str(imageID) - print "update dvo sql: ", sql - # response = raw_input("update dvo...") - try: - self.scratchDb.execute(sql) - except: - return - - self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") - self.updateStackMetaID("StackModelFit") - self.updateStackTypeID("StackModelFit") - self.updateDvoIDs("StackModelFit") +# try: +# self.scratchDb.execute(sql) +# except: +# return + +## self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") + ##self.updateStackMetaID("StackModelFit") +## self.updateStackTypeID("StackModelFit") + self.scratchDb.reportAndDeleteRowsWithNULLS("StackModelFit", "objID") - self.deleteDetectionsNotInStackDetection("StackModelFit") - - ''' - Reports and deletes detections in this table that are not in StackDetection - ''' - def deleteDetectionsNotInStackDetection(self, table): - - sql = "SELECT COUNT(*) FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)" + self.deleteDetectionsNotInStackObject("StackModelFit", f) + + ''' + Reports and deletes detections in this table that are not in StackObject + ''' + def deleteDetectionsNotInStackObject(self, table, filter): + f= filter + + sql = "SELECT COUNT(*) FROM " + table + " WHERE "+f+"ippDetectID NOT IN (SELECT "+f+"ippDetectID FROM StackObject)" rs = self.scratchDb.executeQuery(sql) rs.first() @@ -773,7 +941,7 @@ if nMissing < 1: return - sql = "DELETE FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)" - self.scratchDb.execute(sql) - self.logger.infoPair("Detections not in StackDetection", "%d deleted from %s" % (nMissing, table)) + sql = "DELETE FROM " + table + " WHERE "+f+"ippDetectID NOT IN (SELECT "+f+"ippDetectID FROM StackObject)" + self.scratchDb.execute(sql) + self.logger.infoPair("Detections not in StackObject", "%d deleted from %s" % (nMissing, table)) @@ -784,167 +952,142 @@ self.logger.infoPair("Procesing table", "StackToImage") - - imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.id) - - for imageID in imageIDs: - sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ + filters = ['g','r','i','z','y'] + for f in filters: + + if f == 'g': + stackmetaid = self.gstackID + if f == 'r': + stackmetaid = self.rstackID + if f == 'i': + stackmetaid = self.istackID + if f == 'z': + stackmetaid = self.zstackID + if f == 'y': + stackmetaid = self.ystackID + if stackmetaid > 0: + + imageIDs = self.gpc1Db.getImageIDsForThisStackID(stackmetaid) + + for imageID in imageIDs: + sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ VALUES (\ - " + str(self.id) + ", " + imageID + ")" + " + str(stackmetaid) + ", " + imageID + ")" + self.scratchDb.execute(sql) + + # now update StackMeta with correct number of inputs + sql = "UPDATE "+f+"StackMeta SET nP2Images = (SELECT COUNT(distinct(floor(imageID/100))) FROM StackToImage where stackMetaID = "+str(stackmetaid)+")" + self.scratchDb.execute(sql) + + ''' + Generates the randomStackIds + ''' + def populateRandomsAndSkyCellID(self): + + self.logger.infoPair("Generating Randoms","for StackObject") + sql = "update StackObject set randomStackObjID = FLOOR(RAND()*9223372036854775807) \ + ,skyCellID = " + str(self.skycell)+ " \ + ,surveyID = " + str(self.surveyID) + try: self.scratchDb.execute(sql) - - # now update StackMeta with correct number of inputs - sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(distinct(floor(imageID/100))) FROM StackToImage)" - self.scratchDb.execute(sql) - - ''' - Populates the SkinnyObject table - ''' - def populateSkinnyObject(self): - - self.logger.infoPair("Procesing table", "SkinnyObject") - - sql = "INSERT INTO SkinnyObject (\ - objID \ - ,ippObjID \ - ,projectionCellID \ - ) \ - SELECT \ - DISTINCT objID \ - ,ippObjID \ - ,projectionCellID \ - FROM StackDetection" - self.scratchDb.execute(sql) - - self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) - self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.skychunk.dataRelease)) - - ''' - Populates the StackDetectionCalib table - XXX this can probably get a big speed increase by using 'SELECT () INTO OUTFILE '/tmp/name' and then - calling the load data infile '/tmp/name' into table; - ''' - def populateStackDetectionCalibInsertUpdate(self): - self.logger.infoPair("Processing table", "StackDetectionCalib") - tableName = "StackDetectionCalib" - # insert all detections into table - sql = "INSERT INTO " + tableName + " ( \ - objID \ - ,stackDetectID \ - ,ippObjID \ - ,ippDetectID \ - ,filterID \ - ,surveyID \ - ) \ - SELECT \ - objID \ - ,stackDetectID \ - ,ippObjID \ - ,ippDetectID \ - ,filterID \ - ,surveyID \ - FROM StackDetection" - self.scratchDb.execute(sql) - - imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) - self.logger.infoPair("obtained","imageID") - - # insert calibration information from dvoDetections into the Table - sql = "UPDATE " + tableName + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b \ - SET a.ra = b.ra, \ - a.dec = b.dec_, \ - a.raErr = b.raErr, \ - a.decErr = b.decErr, \ - a.zp = b.zp, \ - a.zpErr = b.zpErr, \ - a.expTime = b.expTime, \ - a.airMass = b.airMass \ - WHERE a.stackDetectID = b.detectID \ - AND b.imageID = " + str(imageID) - self.scratchDb.execute(sql) + except: + self.logger.infoPair("failed sql", sql) + + ''' + copies randomStackIds from StackObject to tableName + ''' + def copyRandomsAndSkyCellIDFromStackObject(self, tableName): + + self.logger.infoPair("copying Randoms from StackObject to ", tableName) + sql = "update " + tableName + " join StackObject using (objID) \ + set " + tableName + ".randomStackObjID = StackObject.randomStackObjID \ + , " + tableName + ".skyCellID = StackObject.skyCellID \ + , " + tableName + ".surveyID = StackObject.surveyID" + self.logger.infoPair("sql", sql) + try: + self.scratchDb.execute(sql) + except: + self.logger.infoPair("failed sql", sql) + + + + def populateStackIPPDetectIDFromDvo(self, tableName, all): + + #tableName = "StackObject" + self.logger.infoPair("Processing table", tableName) + + filters = ['g','r','i','z','y'] + + for f in filters: + skip = 0 + #first: do we skip this filter + if f == 'g': + if self.gstackID < 0: + skip = 1 + else: + self.header=self.gheader + stackmetaid = self.gstackID + elif f == 'r': + if self.rstackID < 0: + skip = 1 + else: + self.header = self.rheader + stackmetaid = self.rstackID + elif f == 'i': + if self.istackID < 0: + skip = 1 + else: + self.header = self.iheader + stackmetaid = self.istackID + elif f == 'z': + if self.zstackID < 0: + skip = 1 + else: + self.header = self.zheader + stackmetaid = self.zstackID + elif f == 'y': + if self.ystackID < 0: + skip = 1 + else: + self.header = self.yheader + stackmetaid = self.ystackID + if skip == 0: + imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) + self.logger.infoPair("obtained","imageID") + addsql = "" + if (all == 1): + addsql = "a." + f + "ra = b.ra, \ + a." + f + "dec = b.dec_, \ + a." + f + "raErr = b.raErr, \ + a." + f + "decErr = b.decErr, \ + a." + f + "zp = b.zp, \ + a." + f + "expTime = b.expTime," + + + sql = "UPDATE " + tableName + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b \ + SET a." + f + "ippDetectID = b.ippDetectID, \ + " + addsql + " \ + a." + f + "stackDetectID = b.detectID, \ + a." + f + "ippObjID = b.ippObjID, \ + a." + f + "stackMetaID = " + str(stackmetaid) + " \ + WHERE a.objID = b.objID AND b.imageID = " + str(imageID) + self.logger.infoPair('sql',sql) + self.scratchDb.execute(sql) + else: + self.logger.infoPair("skipping ingest of dvo for filter",f) + #out of the loops - give dataRelase to all self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) - - def populateStackDetectionCalib(self): - - tableName = "StackDetectionCalib" - self.logger.infoPair("Processing table", tableName) - - imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) - self.logger.infoPair("obtained","imageID") - - # check for & create output directory first - datadumpDir = "/tmp/datadump" - try: - statinfo = os.stat(datadumpDir) - # check on the stat results? - except: - print "making the data dump directory ", datadumpDir - os.mkdir(datadumpDir) - os.chmod(datadumpDir, 0777) - statinfo = os.stat(datadumpDir) - - dumpFile = datadumpDir + "/genetest.xx.dat" - files = glob.glob(dumpFile) - if len(files) > 0: - os.unlink(dumpFile) - - # insert all detections into table - sql = "SELECT \ - a.objID, \ - a.stackDetectID, \ - a.ippObjID, \ - a.ippDetectID, \ - a.filterID, \ - a.surveyID, \ - b.ra, \ - b.dec_, \ - b.raErr, \ - b.decErr, \ - b.zp, \ - b.zpErr, \ - b.expTime, \ - b.airMass, \ - " + str(self.skychunk.dataRelease) + " \ - FROM \ - StackDetection as a \ - JOIN " + self.scratchDb.dvoDetectionTable + " as b \ - ON (a.stackDetectID = b.detectID) where b.imageID = " + str(imageID) + \ - " INTO OUTFILE '" + dumpFile + "'" - - try: self.scratchDb.execute(sql) - except: - self.logger.info("failed to select data for stackDetectionCalib") - self.logger.infoPair("sql: ", sql) - raise - - sql = "LOAD DATA INFILE '" + dumpFile + "' INTO TABLE " + tableName - try: self.scratchDb.execute(sql) - except: - self.logger.info("failed to load data from infile for stackDetectionCalib") - self.logger.infoPair("sql: ", sql) - raise - - ## XXX write this with the select/insert - # self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.skychunk.dataRelease)) - - ''' - Populates the ObjectCalColor table - ''' - def populateObjectCalColor(self): - - self.logger.infoPair("Procesing table", "ObjectCalColor table") - - sql = "INSERT INTO ObjectCalColor (\ - objID \ - ,ippObjID \ - ) \ - SELECT \ - DISTINCT objID \ - ,ippObjID \ - FROM StackDetection" - self.scratchDb.execute(sql) - - self.scratchDb.updateFilterID("ObjectCalColor", self.filter) - self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.skychunk.dataRelease)) - + + + def deleteNULLippDetectIDs(self, tableName): + + self.logger.infoPair("Deleting NULL ippdetectIDs from table", tableName) + sql = "DELETE FROM " + tableName + " WHERE \ + gippDetectID is NULL AND \ + rippDetectID is NULL AND \ + iippDetectID is NULL AND \ + zippDetectID is NULL AND \ + yippDetectID is NULL " + self.logger.infoPair('sql',sql) + self.scratchDb.execute(sql) ''' @@ -955,13 +1098,9 @@ self.logger.debug("Altering PSPS tables") # heather uncommented the unique line -- well that no worky - #self.scratchDb.makeColumnUnique("StackDetection", "objID") - self.scratchDb.createIndex("StackDetection", "ippDetectID") - self.scratchDb.createIndex("StackDetection", "stackDetectID") + #self.scratchDb.makeColumnUnique("StackObject", "objID") + self.scratchDb.createIndex("StackObject", "ippDetectID") + self.scratchDb.createIndex("StackObject", "stackDetectID") self.scratchDb.createIndex("StackApFlx", "ippDetectID") self.scratchDb.createIndex("StackModelFit", "ippDetectID") - - # dec is reserved in MySQL, so STILTS replaces if with dec_, which is not the name of - # the column in PSPS - self.scratchDb.execute("ALTER TABLE StackDetectionCalib CHANGE dec_ `dec` double") return True @@ -979,10 +1118,37 @@ return True + ''' + Insert objid (from dvo) into table + ''' + def insertDvoIDs(self, table, filterName): + self.logger.infoPair("getting imageID for ","filter") + if (filterName == 'g.00000'): + self.header=self.gheader + if (filterName == 'r.00000'): + self.header=self.rheader + if (filterName == 'i.00000'): + self.header=self.iheader + if (filterName == 'z.00000'): + self.header=self.zheader + if (filterName == 'y.00000'): + self.header=self.yheader + + imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) + self.logger.infoPair("obtained","imageID") + self.logger.debug("Updating table '" + table + "' with DVO IDs...") + sql = "INSERT IGNORE INTO " + table + " ( \ + objID ) \ + SELECT (objID) FROM " + self.scratchDb.dvoDetectionTable + " \ + WHERE " + self.scratchDb.dvoDetectionTable + ".imageID = " + str(imageID) + self.logger.infoPair("sql",sql) + self.scratchDb.execute(sql) + + ''' Updates provided table with DVO IDs from DVO table ''' - def updateDvoIDs(self, table): - self.logger.infoPair("getting","imageID") + def updateDvoIDs(self, table, filterName): + self.logger.infoPair("getting imageID for ","filter") imageID = self.scratchDb.getImageIDFromExternID(self.header['IMAGEID']) self.logger.infoPair("obtained","imageID") @@ -1014,5 +1180,4 @@ print "update dvo sql: ", sql - # response = raw_input("update dvo...") self.scratchDb.execute(sql) @@ -1022,36 +1187,137 @@ ''' def populatePspsTables(self): - + self.logger.infoPair("starting","populatePspsTables"); self.skipBatch = False - if not self.useFullTables: - if not self.getIDsFromDVO(): + if not self.getIDsFromDVO(): return False - #self.logger.infoPair("populating","StackMeta") - self.populateStackMeta() - #self.logger.infoPair("populating","StackDetection") - self.populateStackDetection() - - if self.stackType != "NIGHTLY_STACK": - #self.logger.infoPair("populating","StackModelFit") - self.populateStackModelFit() - #self.logger.infoPair("populating","StackApFlx") + #do some basics first + # populate stack meta per filter + # insert objid/stackdetectid per filter to each table + self.tablesToExport=[] + + if self.gstackID > 0: + self.logger.infoPair("populating","gStackMeta") + self.populateStackMeta("g.00000") + self.insertDvoIDs("StackObject","g.00000") + self.insertDvoIDs("StackModelFit","g.00000") + self.insertDvoIDs("StackApFlx","g.00000") + self.tablesToExport.append("gStackMeta") + if self.rstackID > 0: + self.logger.infoPair("populating","rStackMeta") + self.populateStackMeta("r.00000") + self.insertDvoIDs("StackObject","r.00000") + self.insertDvoIDs("StackModelFit","r.00000") + self.insertDvoIDs("StackApFlx","r.00000") + self.tablesToExport.append("rStackMeta") + if self.istackID > 0: + self.logger.infoPair("populating","iStackMeta") + self.populateStackMeta("i.00000") + self.insertDvoIDs("StackObject","i.00000") + self.insertDvoIDs("StackModelFit","i.00000") + self.insertDvoIDs("StackApFlx","i.00000") + self.tablesToExport.append("iStackMeta") + if self.zstackID > 0: + self.logger.infoPair("populating","zStackMeta") + self.populateStackMeta("z.00000") + self.insertDvoIDs("StackObject","z.00000") + self.insertDvoIDs("StackModelFit","z.00000") + self.insertDvoIDs("StackApFlx","z.00000") + self.tablesToExport.append("zStackMeta") + if self.ystackID > 0: + self.logger.infoPair("populating","yStackMeta") + self.populateStackMeta("y.00000") + self.insertDvoIDs("StackObject","y.00000") + self.insertDvoIDs("StackModelFit","y.00000") + self.insertDvoIDs("StackApFlx","y.00000") + self.tablesToExport.append("yStackMeta") + + self.logger.infoPair("populating ","StackObject") + # need to index on objid and ippdetectid or it is stupidly slow + self.scratchDb.createIndex("StackObject", "objID") + #populate from dvo the (f)ippDetectID into StackObject + self.populateStackIPPDetectIDFromDvo("StackObject",1) + #index the ippDetectids for the next step + self.scratchDb.createIndex("StackObject", "gippDetectID") + self.scratchDb.createIndex("StackObject", "rippDetectID") + self.scratchDb.createIndex("StackObject", "iippDetectID") + self.scratchDb.createIndex("StackObject", "zippDetectID") + self.scratchDb.createIndex("StackObject", "yippDetectID") + self.populateStackObject() + self.populateRandomsAndSkyCellID() + # stackObject should be done, pending a few NULL columns (need help) + + if self.stackType != "NIGHTLY_STACK": + + self.logger.infoPair("populating ","StackModelFit") + + self.scratchDb.createIndex("StackModelFit", "objID") + #populate from dvo the (f)ippdetectid into stackModelFit + self.populateStackIPPDetectIDFromDvo("StackModelFit",0) + #index the ippDetectIds for the next step + self.scratchDb.createIndex("StackModelFit", "gippDetectID") + self.scratchDb.createIndex("StackModelFit", "rippDetectID") + self.scratchDb.createIndex("StackModelFit", "iippDetectID") + self.scratchDb.createIndex("StackModelFit", "zippDetectID") + self.scratchDb.createIndex("StackModelFit", "yippDetectID") + #cull the entries that should not be (how?) + self.deleteNULLippDetectIDs("StackModelFit") + #populate stackModelFit + if self.gstackID > 0: + self.logger.infoPair("populating g filter for stackModelFit, stackmetaid =", self.gstackID) + self.populateStackModelFit("g") + if self.rstackID > 0: + self.logger.infoPair("populating r filter for stackModelFit, stackmetaid =", self.rstackID) + self.populateStackModelFit("r") + if self.istackID > 0: + self.logger.infoPair("populating i filter for stackModelFit, stackmetaid =", self.istackID) + self.populateStackModelFit("i") + if self.zstackID > 0: + self.logger.infoPair("populating z filter for stackModelFit, stackmetaid =", self.zstackID) + self.populateStackModelFit("z") + if self.ystackID > 0: + self.logger.infoPair("populating y filter for stackModelFit, stackmetaid =", self.ystackID) + self.populateStackModelFit("y") + + self.copyRandomsAndSkyCellIDFromStackObject("StackModelFit") + + self.logger.infoPair("populating ","StackApFlx") + + self.scratchDb.createIndex("StackApFlx", "objID") + #populate from dvo the (f)ippDetectID into StackApFlx + self.populateStackIPPDetectIDFromDvo("StackApFlx",0) + #index the ippDetectIDs for the next step + self.scratchDb.createIndex("StackApFlx", "gippDetectID") + self.scratchDb.createIndex("StackApFlx", "rippDetectID") + self.scratchDb.createIndex("StackApFlx", "iippDetectID") + self.scratchDb.createIndex("StackApFlx", "zippDetectID") + self.scratchDb.createIndex("StackApFlx", "yippDetectID") + #populate StackApFlx + self.deleteNULLippDetectIDs("StackApFlx") self.populateStackApFlx() - #self.logger.infoPair("populating","StackToImage") + self.copyRandomsAndSkyCellIDFromStackObject("StackApFlx") + #cull the entries that should not be (how?) + + + self.logger.infoPair("populating","StackToImage") self.populateStackToImage() - #self.logger.infoPair("populating","SkinnyObject") - self.populateSkinnyObject() - #self.logger.infoPair("populating","ObjectCalColor") - #self.populateObjectCalColor() - #self.logger.infoPair("populating","StackDetectionCalib") - self.populateStackDetectionCalib() - - self.setMinMaxObjID(["StackDetection"]) - - if self.totalDetections < 1: - - self.logger.error("No detections to publish") - return False - + + self.setMinMaxObjID(["StackObject"]) + # self.updateStackTypeID("StackModelFit") + + ##if self.totalDetections < 1: + + ##self.logger.error("No detections to publish") + ##return False + self.tablesToExport.append("StackToImage") + self.tablesToExport.append("StackObject") + self.tablesToExport.append("StackModelFit") + self.tablesToExport.append("StackApFlx") + + self.logger.infoPair("finishing","populatePspsTables"); return True - + ''' + overidding this method - use regex to trimoff letter before stack + ''' + def exportPspsTablesToFits(self, regex="(.*)"): + return super(StackBatch, self).exportPspsTablesToFits("(Stack.*)") Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema10.txt =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema10.txt (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema10.txt (revision 36826) @@ -0,0 +1,132 @@ +Notes on Schema Version 9 + + +Proposed Image Products and PSPS Tables and Views + +IMAGE PRODUCTS + +Survey | Description | Image Products Saved + +3pi | 3pi | 3pi warps, 3pi stacks +CNP | Celestial North Pole (3pi)| CNP warps, stacks +SS | ecliptic w band data | w warps, w stack +MDD | Medium Deep Deep | MD warps, deep stacks +MDT | Medium Deep Template | template stacks +MDN | Medium Deep Nightly Stack | nightly stacks +MDN | Medium Deep Off Year | off year template stacks +M31 | M31 | M31 warps, M31 stacks, template +STS | STS | STS warps, STS template - best iq stack + + +Survey | Description | Image Products NOT Saved, but analyzed for difference detections and discarded + +3pi | difference |3pi warps - stack +CNP | difference |CNP warps - stack +SS | difference image | w warps - stack +MDD | difference image | MD warps - off year template stacks +MDO | difference image | MDN stacks - off year template stacks +M31 | difference image | M31 warps - stack +STS | difference image | STS warps - template + + + +PSPS TABLES + + +Fundamental IPP Data Products Origin + +Detection detections found in detrended images +ObjectThin positonal and flag data, no fluxes +MeanObject extracted Mean properties as determined from individual Detections. +StackObject Stack Detections in all filters, with forced upperlimits, separate rows for primary, secondary, best stack detections (80% time only primary) +StackApFlx all filters in one table, unconvolved and convolved apertures (up to 6 , 8, 10(?) pixels in all bands) +StackModelFit all filters in one table. Models of high signal to noise objects in low density regions. +DiffDet difference detections found in difference image = image - template +DiffObject associations of Difference Detections +ForcedWarpMeasure forced measurements on each warp in all filters at the position of an object found in a stack. Includes StackApFlx-like table +ForcedMeanObject like MeanObject, but using the Forced Mean Measurements to form the Mean. + +w band TBD + + +Derived Data Product Tables + +pcsPhotoZ populated from PCS +pcsPhotoZProbability populated from PCS +pcsStarGalQSOSep populated from PCS +pcsStellarParams populated from PCS +mopsMovingObject populated from MOPS database +ppmObj proper motion and parallax populated from IPP +tcsClassification populated from transient server database +galExtinction populated from Dust team +stargal populated from various algorithms, combos of ipp parameters, fits to population as function of magnitude, or Support Vector Machine +GoodObject selected and populated post-facto +StarGalaxySep selected and populated post-facto + + +Observational Metadata and System MetaData Tables + +CameraConfig +DetectionFlags +DiffMeta +Filter +FitModel +ForcedWarpMeta +FrameMeta +ImageFlags +ImageMeta add psf 3nd and 4th moments +Mask +MovingObjectToDet +MovingObjectToDiffDet +MovingObjToObject +ObjectFilterFlags flag info not in current schema but in PSI interface... +ObjectInfoFlags +ObjectQualityFlags +PhotoCal +ProjectionCell +ProcessingVersion +SkyCell +StackDetectionFlags +StackMeta +StackToImage +StackType +Survey +TelescopeConfig +fitModel +pcsGalaxyModel +pcsPhotoZRecipe +pcsSGSepRecipe +pcsStarModel +pcsStellarParamsRecipe + + + +PSPS VIEWS - examples + +ObjectMean = ObjectThin join MeanObject +ObjectStack = ObjectThin join StackObject +ObjectApFlx = ObjectThin join StackApFlx +ObjectModel = ObjectThin join StackModelFit +ObjectStackAll = ObjectThin join StackObject and StackAp and StackModelFit +ObjectForcedMean = ObjectThin join ForcedMeanObject +ObjectPrimary = ObjectThin join StackObject join StackApFlx where primaryDection is true +ObjectBest = ObjectThin join StackObject join StackApFlx where bestDetection is true + + +PSPS Indicies + +DetectionID +ObjectID +BatchID +StackTypeID +MovingObjID +DiffObjID +RandomID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. +RandomStackID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. + + + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema11.txt =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema11.txt (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema11.txt (revision 36826) @@ -0,0 +1,132 @@ +Notes on Schema Version 9 + + +Proposed Image Products and PSPS Tables and Views + +IMAGE PRODUCTS + +Survey | Description | Image Products Saved + +3pi | 3pi | 3pi warps, 3pi stacks +CNP | Celestial North Pole (3pi)| CNP warps, stacks +SS | ecliptic w band data | w warps, w stack +MDD | Medium Deep Deep | MD warps, deep stacks +MDT | Medium Deep Template | template stacks +MDN | Medium Deep Nightly Stack | nightly stacks +MDN | Medium Deep Off Year | off year template stacks +M31 | M31 | M31 warps, M31 stacks, template +STS | STS | STS warps, STS template - best iq stack + + +Survey | Description | Image Products NOT Saved, but analyzed for difference detections and discarded + +3pi | difference |3pi warps - stack +CNP | difference |CNP warps - stack +SS | difference image | w warps - stack +MDD | difference image | MD warps - off year template stacks +MDO | difference image | MDN stacks - off year template stacks +M31 | difference image | M31 warps - stack +STS | difference image | STS warps - template + + + +PSPS TABLES + + +Fundamental IPP Data Products Origin + +Detection detections found in detrended images +ObjectThin positonal and flag data, no fluxes +MeanObject extracted Mean properties as determined from individual Detections. +StackObject Stack Detections in all filters, with forced upperlimits, separate rows for primary, secondary, best stack detections (80% time only primary) +StackApFlx all filters in one table, unconvolved and convolved apertures (up to 6 , 8, 10(?) pixels in all bands) +StackModelFit all filters in one table. Models of high signal to noise objects in low density regions. +DiffDet difference detections found in difference image = image - template +DiffObject associations of Difference Detections +ForcedWarpMeasure forced measurements on each warp in all filters at the position of an object found in a stack. Includes StackApFlx-like table +ForcedMeanObject like MeanObject, but using the Forced Mean Measurements to form the Mean. + +w band TBD + + +Derived Data Product Tables + +pcsPhotoZ populated from PCS +pcsPhotoZProbability populated from PCS +pcsStarGalQSOSep populated from PCS +pcsStellarParams populated from PCS +mopsMovingObject populated from MOPS database +ppmObj proper motion and parallax populated from IPP +tcsClassification populated from transient server database +galExtinction populated from Dust team +stargal populated from various algorithms, combos of ipp parameters, fits to population as function of magnitude, or Support Vector Machine +GoodObject selected and populated post-facto +StarGalaxySep selected and populated post-facto + + +Observational Metadata and System MetaData Tables + +CameraConfig +DetectionFlags +DiffMeta +Filter +FitModel +ForcedWarpMeta +FrameMeta +ImageFlags +ImageMeta add psf 3nd and 4th moments +Mask +MovingObjectToDet +MovingObjectToDiffDet +MovingObjToObject +ObjectFilterFlags flag info not in current schema but in PSI interface... +ObjectInfoFlags +ObjectQualityFlags +PhotoCal +ProjectionCell +ProcessingVersion +SkyCell +StackDetectionFlags +StackMeta +StackToImage +StackType +Survey +TelescopeConfig +fitModel +pcsGalaxyModel +pcsPhotoZRecipe +pcsSGSepRecipe +pcsStarModel +pcsStellarParamsRecipe + + + +PSPS VIEWS - examples + +ObjectMean = ObjectThin join MeanObject +ObjectStack = ObjectThin join StackObject +ObjectApFlx = ObjectThin join StackApFlx +ObjectModel = ObjectThin join StackModelFit +ObjectStackAll = ObjectThin join StackObject and StackAp and StackModelFit +ObjectForcedMean = ObjectThin join ForcedMeanObject +ObjectPrimary = ObjectThin join StackObject join StackApFlx where primaryDection is true +ObjectBest = ObjectThin join StackObject join StackApFlx where bestDetection is true + + +PSPS Indicies + +DetectionID +ObjectID +BatchID +StackTypeID +MovingObjID +DiffObjID +RandomID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. +RandomStackID between 1 and 2^16 = 4,294,967,296 so you pick how many random objects you want , + i.e 10,000 pick randomid < 10,000 and if indexed, this will be very fast. + + + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema12.txt =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema12.txt (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema12.txt (revision 36826) @@ -0,0 +1,2 @@ +Heather cleaned up the column names and types to make them consistent. + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema13.txt =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema13.txt (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/Notes_on_Schema13.txt (revision 36826) @@ -0,0 +1,8 @@ +changes by EAM: +* added lensing terms to ForcedWarpMeasurement & ForcedMeanObject +* removed fluxPeak and momentRH generally +* changed psfWidMajor,Minor to psfMajorFWHM,psfMinorFWHM +* added psfCore (k term in psf model : 1 / (1 + kz + k^1.667) +* fixed stack flux & mag names +* set units for moments to be in arcsec everywhere. + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/GoodObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/GoodObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/GoodObject (revision 36826) @@ -0,0 +1,4 @@ +objName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +goodObjectID dimensionless BIGINT 8 NA index for post-facto "good objects" subset from ObjID Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/StarGalaxySep =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/StarGalaxySep (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/StarGalaxySep (revision 36826) @@ -0,0 +1,9 @@ +objName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +sgbinarySep1 dimensionless TINYINT 2 255 binary flag for star galaxy separator - technique 1 +sgbinarySep2 dimensionless TINYINT 2 255 binary flag for star galaxy separator - technique 2 +sgbinarySep3 dimensionless TINYINT 2 255 binary flag for star galaxy separator - technique 3 +sgSep1 dimensionless REAL 4 -999 probility of star/galaxy technique 1 +sgSep2 dimensionless REAL 4 -999 probility of star/galaxy technique 2 +sgSep3 dimensionless REAL 4 -999 probility of star/galaxy technique 3 Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/galExtinction =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/galExtinction (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/galExtinction (revision 36826) @@ -0,0 +1,28 @@ +galExtinction +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number + +From Doug and Eddie ? + +E(B-V) + +gGalext +rGalext +iGalext +zGalext +yGalext +wGalext +probability vector + +stellar parameters +Teff +g_surf +[Fe/H] + + + + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/mopsMovingObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/mopsMovingObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/mopsMovingObject (revision 36826) @@ -0,0 +1,28 @@ +Name Unit Data Type Size Default Value Description + +MovingObjName dimensionless STRING x -999 MPC Name +MovingObjType dimensionless STRING x -999 Type of moving object (NEO, PHA, COMET, ...) +MovingObjID dimensionless BIGINT 8 -999 Moving Object identifier +OrbEpoch date DATE -999 The epoch of osculation of the orbital elements. +Moid au REAL 4 -999 Minimum orbit intersection distance +OrbM dimensionles REAL 4 -999 Mean anomaly at the epoch. +OrbT date DATE -999 Date of perihelion passage. +ORBn deg/day REAL 4 -999 Mean daily motion (in degrees/day). +ORBa AU REAL 4 -999 Semimajor axis (in AU). +ORBz 1/AU REAL 4 -999 Reciprocal semimajor axis (in 1/AU). +ORBq AU REAL 4 -999 Perihelion distance (in AU). +ORBe dimensionless REAL 4 -999 Orbital eccentricity. +OrbP years REAL 4 -999 Orbital period (in years). +OrbPeri degrees REAL 4 -999 The J2000.0 argument of perihelion (in degrees). +OrbNode degrees REAL 4 -999 The J2000.0 longitude of the ascending node (in degrees). +OrbIncl degrees REAL 4 -999 The J2000.0 inclination (in degrees). +P1 dimensionless REAL 4 -999 P1 first component of P vector +P2 dimensionless REAL 4 -999 P2 second component of P vector +P3 dimensionless REAL 4 -999 P3 third component of P vector +Q1 dimensionless REAL 4 -999 Q1 first component of Q vector +Q2 dimensionless REAL 4 -999 Q2 second component of Q vector +Q3 dimensionless REAL 4 -999 Q3 third component of Q vector +U dimensionless REAL 4 -999 Uncertainty parameter. +H dimensionless REAL 4 -999 Absolute visual magnitude. A table converting H to a diameter is available. +G dimensionless REAL 4 -999 Slope parameter. + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZ =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZ (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZ (revision 36826) @@ -0,0 +1,21 @@ +pcsPhotoZ +Name Unit Data Type Size Default Value Description + +objID BIGINT 8 NA ODM object identifier index +ippObjID BIGINT 8 NA IPP object number +pcsPhotoZRecipeID INT 4 -999 Recipe of the photoZ calculation +galExt REAL 4 -999 Galactic extinction E(B-V) used for photoz determination (derived from Schlegel Maps) +photoz_1 REAL 4 -999 PhotoZ for best fitting template +photozErr_1 REAL 4 -999 Estimated error for this tempalte +chi2_1 REAL 4 -999 Chi^2 for this template +modelID_1 INT 4 -999 Identifier for the template for this recipe +photoz_2 REAL 4 -999 PhotoZ for second template +photozErr_2 REAL 4 -999 Estimated error for this tempalte +chi2_2 REAL 4 -999 Chi^2 for this template +modelID_2 INT 4 -999 Identifier for the template for this recipe +flags_gtfit INT 4 -999 warning flags (e.g. missing band(s), unreliable galExt value etc.), TBD +av REAL 4 -999 determined extinction +chi2 REAL 4 -999 chi2 for the best fitting stellar template +modelID INT 4 -999 modelID of the best fitting stellar template +flags_stfit INT 4 -999 warning flags from stellar template fitting +starGalQSOSep INT 4 -999 0: Object is a star (chi2 Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZProbabiity =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZProbabiity (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsPhotoZProbabiity (revision 36826) @@ -0,0 +1,37 @@ +PhotoZProbability +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +pcsPhotoZRecipeID dimensionless INT 4 -999 recipe for the photoz determination +modelID dimensionless INT 4 -999 modelID of the template +sp1 dimensionless REAL 4 -999 spline params 1 +sp2 dimensionless REAL 4 -999 spline params 2 +sp3 dimensionless REAL 4 -999 spline params 3 +sp4 dimensionless REAL 4 -999 spline params 4 +sp5 dimensionless REAL 4 -999 spline params 5 +sp6 dimensionless REAL 4 -999 spline params 6 +sp7 dimensionless REAL 4 -999 spline params 7 +sp8 dimensionless REAL 4 -999 spline params 8 +sp9 dimensionless REAL 4 -999 spline params 9 +sp10 dimensionless REAL 4 -999 spline params 10 +sp11 dimensionless REAL 4 -999 spline params 11 +sp12 dimensionless REAL 4 -999 spline params 12 +sp13 dimensionless REAL 4 -999 spline params 13 +sp14 dimensionless REAL 4 -999 spline params 14 +sp15 dimensionless REAL 4 -999 spline params 15 +sp16 dimensionless REAL 4 -999 spline params 16 +sp17 dimensionless REAL 4 -999 spline params 17 +sp18 dimensionless REAL 4 -999 spline params 18 +sp19 dimensionless REAL 4 -999 spline params 19 +sp20 dimensionless REAL 4 -999 spline params 20 +sp21 dimensionless REAL 4 -999 spline params 21 +sp22 dimensionless REAL 4 -999 spline params 22 +sp23 dimensionless REAL 4 -999 spline params 23 +sp24 dimensionless REAL 4 -999 spline params 24 +sp25 dimensionless REAL 4 -999 spline params 25 +sp26 dimensionless REAL 4 -999 spline params 26 +sp27 dimensionless REAL 4 -999 spline params 27 +sp28 dimensionless REAL 4 -999 spline params 28 +sp29 dimensionless REAL 4 -999 spline params 29 +sp30 dimensionless REAL 4 -999 spline params 30 Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStarGalQSOSep =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStarGalQSOSep (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStarGalQSOSep (revision 36826) @@ -0,0 +1,10 @@ +pcsStarGalQSOSep +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +pcsSGSepRecipeIDdimensionless INT 4 NA recipe for the star/galaxy/quasar separation +probStar percentage FLOAT 8 -999 probability object is a star +probGalaxy percentage FLOAT 8 -999 probability object is a galaxy +probQuasar percentage FLOAT 8 -999 probability object is a quasar +flags dimensionless INT 4 -999 flags Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStellarParams =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStellarParams (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/pcsStellarParams (revision 36826) @@ -0,0 +1,11 @@ +pcsStellarParams s +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +pcsStellarParamsRecipeID dimensionless INT 4 -999 recipe for the star/galaxy/quasar separation +Teff dimensionless REAL 4 -999 N/A +A0 dimensionless REAL 4 -999 N/A +Fe_H dimensionless REAL 4 -999 N/A +flags dimensionless INT 4 -999 flags + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/ppmObj =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/ppmObj (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/ppmObj (revision 36826) @@ -0,0 +1,25 @@ +PPM +Description: Proper motions and parallax associated with defined objects. + +Name Unit Data Type Size Default Value Description +objID BIGINT 8 NA ODM object identifier +ippObjID BIGINT 8 NA IPP object identifier index +ppmID BIGINT 8 NA ppm object identifier ( ppmToObject metatable?) +meanEpoch FLOAT 8 -999 mean epoch of observations in Mean Julian Days +elapseTime REAL 4 -999 Elapsed time of astrometric observations +muRa REAL 4 -999 proper motion in RA +muRaErr REAL 4 -999 error in proper motion in RA +muDec REAL 4 -999 proper motion in DEC +muDecErr REAL 4 -999 error in proper motion in Dec +mu REAL 4 -999 total proper motion +muErr REAL 4 -999 error in total proper motion +theta REAL 4 -999 angle of parallax vector from N in degrees +plx REAL 4 -999 total parallax +plxErr REAL 4 -999 error in total parallax +nAstro SMALLINT 2 -999 number of measures in astrometric fit +rmsRA REAL 4 -999 rms of astrometric error in RA +rmsDec REAL 4 -999 rms of astrometric error in Dec +wiggle TINYINT 1 255 flag/pointer to astrometric orbit + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/stargal =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/stargal (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/stargal (revision 36826) @@ -0,0 +1,17 @@ + + +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +StarProb1 fraction FLOAT 8 -999 fractional probability object is a star, Recipe 1 +GalProb1 fraction FLOAT 8 -999 fractional probability object is a galaxy, Recipe 1 +StarProb2 fraction FLOAT 8 -999 fractional probability object is a star, Recipe 2 +GalProb2 fraction FLOAT 8 -999 fractional probability object is a galaxy, Recipe 2 +StarProb3 fraction FLOAT 8 -999 fractional probability object is a star, Recipe 3 +GalProb3 fraction FLOAT 8 -999 fractional probability object is a galaxy, Recipe 3 + + +comment - may come from different combinations of ipp parameters, model fit as function of magnitude, +or Support Vector Machine result. + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/tcsClassification =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/tcsClassification (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Derived_Data_Product_Tables/tcsClassification (revision 36826) @@ -0,0 +1,16 @@ +galExtinction +Name Unit Data Type Size Default Value Description + +objID dimensionless BIGINT 8 NA ODM object identifier index +ippObjID dimensionless BIGINT 8 NA IPP object number +diffobjID dimensionless BIGINT 8 NA IPP object number + +From Stephen and Ken? + +TransientName eg. PS1-10jh +transientClass TDE +ATEL -999 Astronomical Telegram number + + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/Detection (revision 36826) @@ -0,0 +1,59 @@ +objID dimensionless BIGINT 8 NA ODM object identifier +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +detectID dimensionless BIGINT 8 NA ODM detection identifier +ippObjID dimensionless BIGINT 8 NA detection id generated by IPP +ippDetectID dimensionless BIGINT 8 NA detection id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifier +imageID dimensionless BIGINT 8 NA image id hash of frame identifier & OTA/CCD number +randomDetID dimensionless BIGINT 8 NA random detection Id +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days +xPos raw pixels REAL 4 -999 measured x on CCD from PSF fit +yPos raw pixels REAL 4 -999 measured y on CCD from PSF fit +xPosErr raw pixels REAL 4 -999 estimated error in x +yPosErr raw pixels REAL 4 -999 estimated error in y +pltScale arcsec/pix REAL 4 -999 plate scale at this location (from DVO) +posAngle degrees REAL 4 -999 position angle (sky-to-chip) at this location (from DVO) +------------------------------------------------------ Note - next 8 lines came from previous DetCalib table +ra degrees FLOAT 8 NA Right Ascension +dec degrees FLOAT 8 NA Declination +raErr arcsec REAL 4 NA Right Ascension error +decErr arcsec REAL 4 NA Declination error +zp magnitude REAL 4 0 zeropoint (only thing needed to convert adu/sec to AB mag), note no zpErr +telluricExt magnitude REAL 4 NA estimated extinction due to non-photometric conditions +expTime seconds REAL 4 -999 exposure time (informational) +airMass dimensionless REAL 4 0 airmass (informational) +------------------------------------------------------ +psfFlux adu/seconds REAL 4 -999 PSF instrumental flux (in analog-to-digital units/sec) or adu/sec +psfFluxErr adu/seconds REAL 4 -999 estimated error in flux +psfMajorFWHM arcsec REAL 4 -999 model PSF width in major axis +psfMinorFWHM arcsec REAL 4 -999 model PSF width in minor axis +psfTheta degrees REAL 4 -999 model PSF orientation angle degrees East of North +psfCore dimensionless REAL 4 -999 model PSF core parameter [k in (1 + kz + z^1.66)] +psfQf dimensionless REAL 4 -999 PSF coverage factor +psfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of pixels totally unmasked +psfChiSq dimensionless REAL 4 -999 reduced-chisq of the psf model fit. (ndof?, npix?) +psfLikelihood dimensionless REAL 4 -999 PSF likelihood +momentXX arcsec^2 REAL 4 -999 moment XX +momentXY arcsec^2 REAL 4 -999 moment XY +momentYY arcsec^2 REAL 4 -999 momeny YY +momentR1 arcsec REAL 4 -999 first radial moment +momentRH arcsec REAL 4 -999 half light radius +momentM3C arcsec^n REAL 4 -999 3rd moment sin(theta) +momentM3S arcsec^n REAL 4 -999 3rd moment cos(theta) +momentM4C arcsec^n REAL 4 -999 4th moment sin(theta) +momentM4S arcsec^n REAL 4 -999 4th moment cos(theta) +apFlux adu/seconds REAL 4 -999 Aperture flux +apFluxErr adu/seconds REAL 4 -999 Aperture flux error +apFillF dimensionless REAL 4 -999 Aperture fill factor +kronFlux adu/seconds REAL 4 -999 Kron flux +kronFluxErr adu/seconds REAL 4 -999 Kron flux error +kronRad arcsec REAL 4 -999 Kron radius +kronRadErr arcsec REAL 4 -999 Kron radius error +sky adu/sec/pixel REAL 4 -999 PSF sky level at source +skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +infoFlag dimensionless BIGINT 8 -999 flag indicating provenance information +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. +-- +-- [231 bytes/det] Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetObject (revision 36826) @@ -0,0 +1,34 @@ + +diffObjName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S x procesing version +diffObjID dimensionless BIGINT 8 NA ODM object identifier index +diffgcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +ippObjID dimensionless BIGINT 8 NA IPP object number +surveyID dimensionless TINYINT 1 255 Survey ID +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +htmID dimensionless BIGINT 8 NA HTM index +zoneID dimensionless INT 4 NA Zone index +randomID dimensionless BIGINT 8 0 Random integer between 0 and 2^32 = 4,394,967,296 remaining ramdom integer 0 +batchID dimensionless BIGINT 8 -999 +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +objInfoFlag dimensionless BIGINT 8 -999 flag indicating odd properties +qualityFlag dimensionless TINYINT 1 0 subset of objInfoFlag - flags related to goodness of object +consistencyFlag dimensionless SMALLINT2 0 Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation +ra degrees FLOAT 8 NA Right Ascension +dec degrees FLOAT 8 NA Declination +raErr arcsec REAL 4 0.0 Right Ascension error +decErr arcsec REAL 4 0.0 Declination error +cx dimensionless FLOAT 8 NA Cartesian x +cy dimensionless FLOAT 8 NA Cartesian y +cz dimensionless FLOAT 8 NA Cartesian z +lambda degrees FLOAT 8 -999 ecliptic longitude +beta degrees FLOAT 8 -999 ecliptic latitude +l degrees FLOAT 8 -999 galactic longitude +b degrees FLOAT 8 -999 galactic latitude +qfPerfect dimensionless REAL 4 -999 qfPerfect +dataRelease dimensionless TINYINT 1 NA Data release Version of this table +nDetections dimensionless SMALLINT2 -999 total number of detection measurements in all filters +ng dimensionless SMALLINT2 -999 number of detections in g filter +nr dimensionless SMALLINT2 -999 number of detections in g filter +ni dimensionless SMALLINT2 -999 number of detections in g filter +nz dimensionless SMALLINT2 -999 number of detections in g filter +ny dimensionless SMALLINT2 -999 number of detections in g filter Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/DiffDetection (revision 36826) @@ -0,0 +1,69 @@ +diffObjID dimensionless BIGINT 8 NA diff object identifier +diffgcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +diffDetID dimensionless BIGINT 8 NA diff detection identifier +diffDetMetaID dimensionless BIGINT 8 NA diff detection meta identifier +ippDetectID dimensionless BIGINT 8 NA ipp detection id generated by IPP +ippObjID dimensionless BIGINT 8 NA ipp object id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifierr +randomDiffID dimensionless BIGINT 8 NA random diff id +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation of positive detection in Mean Julian Days of positive detection +xPos pixels REAL 4 -999 measured x on diff image sky cell PSF fit +yPos pixels REAL 4 -999 measured y on diff image sky cell from PSF fit +xPosErr pixels REAL 4 -999 estimated error in x +yPosErr pixels REAL 4 -999 estimated error in y +pltScale arcsec/pix REAL 4 -999 plate scale at this location (from DVO) +posAngle degrees REAL 4 -999 position angle (sky-to-chip) at this location (from DVO) +---------------------------------------------------- next 8 came from old DiffDetCalib, zp derived from cpt tables of BOTH images +ra degrees FLOAT 8 NA Right Ascension +dec degrees FLOAT 8 NA Declination +raErr arcsec REAL 4 NA Right Ascension error -> posErr because RA,Dec same? +decErr arcsec REAL 4 NA Declination error +zp magnitude REAL 4 0 zeropoint (all that is needed to convert adu/sec to AB mag), no zpErr +telluricExt magnitude REAL 4 NA estimated extinction due to non-photometric conditions +expTime seconds REAL 4 -999 exposure time (informational) +airMass dimensionless REAL 4 0 airmass (informational) +----------------------------------------------------- attributes of positive detection +psfFlux adu/seconds REAL 4 -999 PSF instrumental flux (in analog-to-digital units/sec) or adu/sec +psfFluxErr adu/seconds REAL 4 -999 estimated error in flux +psfMajorFWHM arcsec REAL 4 -999 model PSF width in major axis +psfMinorFWHM arcsec REAL 4 -999 model PSF width in minor axis +psfTheta degrees REAL 4 -999 model PSF orientation angle degrees East of North +psfCore dimensionless REAL 4 -999 model PSF core parameter [k in (1 + kz + z^1.66)] +psfQf dimensionless REAL 4 -999 PSF coverage factor +psfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of pixels totally unmasked +psfChiSq dimensionless REAL 4 -999 reduced-chisq of the psf model fit. (ndof?, npix?) +psfLikelihood dimensionless REAL 4 -999 PSF likelihood +momentXX arcsec^2 REAL 4 -999 moment XX +momentXY arcsec^2 REAL 4 -999 moment XY +momentYY arcsec^2 REAL 4 -999 momeny YY +momentR1 arcsec REAL 4 -999 first radial moment +momentRH arcsec REAL 4 -999 half light radius +momentM3C arcsec^3 REAL 4 -999 3rd moment sin(theta) +momentM3S arcsec^3 REAL 4 -999 3rd moment cos(theta) +momentM4C arcsec^4 REAL 4 -999 4th moment sin(theta) +momentM4S arcsec^4 REAL 4 -999 4th moment cos(theta) +apFlux adu/seconds REAL 4 -999 Aperture flux +apFluxErr adu/seconds REAL 4 -999 Aperture flux error +apFillF dimensionless REAL 4 -999 Aperture fill factor +kronFlux adu/seconds REAL 4 -999 Kron flux +kronFluxErr adu/seconds REAL 4 -999 Kron flux error +kronRad raw pixels REAL 4 -999 Kron radius +kronRadErr raw pixels REAL 4 -999 Kron radius error +------------------------------------------------------- from diff image cmf/smf files +diffNPos dimensionless REAL 4 -999 npix > 3sigma +diffFRatio dimensionless REAL 4 -999 fPos/(fPos +fNeg) +diffNBad dimensionless REAL 4 -999 nPos/(nPos+nNeg) +diffNMask dimensionless REAL 4 -999 nPos/(nPos + nMask) +diffNAll dimensionless REAL 4 -999 nPos/(nPos + nMask + nNeg) +diffPdist arcsec REAL 4 -999 distance to positive match source +diffNdist arcsec REAL 4 -999 distance to negative match source +diffPSN dimensionless REAL 4 -999 signal to noise of pos match source +diffNSN dimensionless REAL 4 -999 signal to noise of neg match source +------------------------------------------------------- +sky adu/sec/pixel REAL 4 -999 PSF sky level at source +skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +infoFlag dimensionless BIGINT 8 -999 flag indicating provenance information +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedMeanObject (revision 36826) @@ -0,0 +1,62 @@ +objID dimensionless BIGINT 8 NA ODM object identifier index +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +ippObjID dimensionless BIGINT 8 NA IPP object identifier +nDetections dimensionless SMALLINT 2 -999 total number of detection measurements in all filters +batchID dimensionless BIGINT 8 NA batch id +----------------------------------------------------- replicate to bottom for each filter +gStackID INT 4 stack image id (how to carry through IPP + DVO) +gnTotal dimensionless SMALLINT 2 -999 number of detections in g filter +gnIncPSFMag dimensionless SMALLINT 2 -999 number of detections included in robust psf mean in g filter +gnIncKronMag dimensionless SMALLINT 2 -999 number of detections included in robust mean of Kron mag in g filter +gnIncApMag dimensionless SMALLINT 2 -999 number of detections included in robust mean of Ap mag in g filter +gnIncR5 dimensionless SMALLINT 2 -999 number of detections included in robust mean of ApFlx 5 in g filter +gnIncR6 dimensionless SMALLINT 2 -999 number of detections included in robust mean of ApFlx 6 in g filter +------------------------------------------------------- one of the above also corresponds to the moments +gPSFMag AB magnitude REAL 4 -999 robust mean g psf AB magnitude (sigma clipping or MAP or go to 5 or 7 number sumary? 1/12 is ~9th percentile +gPSFMagErr AB magnitude REAL 4 -999 estimated error of g psf AB magnitude mean (individual errors added in quadrature) +gPSFMagStd dimensionless REAL 4 -999 standard deviation of measurements included in the robust mean +gKronMag AB magnitude REAL 4 -999 mean g kron AB magnitude +gKronMagErr AB magnitude REAL 4 -999 estimated error of g kron AB magnitude mean +gKronMagStd dimensionless REAL 4 -999 standard deviation of measurements included in the robust mean +gApMag AB magnitude REAL 4 -999 mean g kron AB magnitude +gApMagErr AB magnitude REAL 4 -999 estimated error of g kron AB magnitude mean +gApMagStd dimensionless REAL 4 -999 standard deviation of measurements included in the robust mean +------------------------------------------------------ +gFmeanflxR5 adu/seconds REAL 4 -999 Flux inside r = 5 in g unconvolved stack +gFmeanflxR5Err adu/seconds REAL 4 -999 estimated error is flxR5 +gFmeanflxR5Std adu/seconds REAL 4 -999 estimated standard deviation in flxR5 +gFmeanflxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gFmeanflxR6 adu/seconds REAL 4 -999 Flux inside r = 6 in g unconvolved stack +gFmeanflxR6Err adu/seconds REAL 4 -999 estimated error is flxR6 +gFmeanflxR6Std adu/seconds REAL 4 -999 estimated standard deviation in flxR6 +gFmeanflxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gFlags dimensionless BIGINT 8 NA flags for g forced +------------------------------------------------------ +gLensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term +gLensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term +gLensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term +gLensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term +gLensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term +gLensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term +gLensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term +gLensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term +gLensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term +gLensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term +gLensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term +gLensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term +gLensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term +gLensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term +gLensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term +gLensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term +gLensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term +gLensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term +gLensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term +gLensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term +------------------------------------------------------ +gGamma arcsec^2 REAL 4 -999 second moment (Mxx + Myy) +gE1 dimensionless REAL 4 -999 polarization, KSB-corrected (Mxx - Myy) / (Mxx + Myy) +gE2 dimensionless REAL 4 -999 polarization, KSB-corrected 2Mxy / (Mxx + Myy) + +-- duplicated for other filters +-- [184 byte/obj/filter + 22 byte/obj] + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ForcedWarpMeasurement (revision 36826) @@ -0,0 +1,82 @@ +objID dimensionless BIGINT 8 NA ODM object identifier +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +detectID dimensionless BIGINT 8 NA ODM detection identifier +ippObjID dimensionless BIGINT 8 NA ippobject id generated by IPP +ippDetectID dimensionless BIGINT 8 NA detection id generated by IPP +filterID dimensionless TINYINT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +surveyID dimensionless TINYINT 1 NA survey flag identifier +forcedWarpID dimensionless BIGINT 8 NA IPP warp_id +randomWarpID dimensionless BIGINT 8 NA Random warp_id +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +obsTime MeanJulianDays FLOAT 8 -999 Time of mid observation in Mean Julian Days +------------------------------------------------------ +zp magnitude REAL 4 0 zeropoint (only thing needed to convert adu/sec to AB mag), note no zpErr +telluricExt magnitude REAL 4 NA estimated extinction due to non-photometric conditions +expTime seconds REAL 4 -999 exposure time (informational) +airMass dimensionless REAL 4 0 airmass (informational) +------------------------------------------------------ +psfFlux adu/seconds REAL 4 -999 PSF instrumental flux (in analog-to-digital units/sec) or adu/sec +psfFluxErr adu/seconds REAL 4 -999 estimated error in flux +psfMajorFWHM arcsec REAL 4 -999 model PSF width in major axis +psfMinorFWHM arcsec REAL 4 -999 model PSF width in minor axis +psfTheta degrees REAL 4 -999 model PSF orientation angle degrees East of North +psfCore dimensionless REAL 4 -999 model PSF core parameter [k in (1 + kz + z^1.66)] +psfQf dimensionless REAL 4 -999 PSF coverage factor +psfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of pixels totally unmasked +psfChiSq dimensionless REAL 4 -999 reduced-chisq of the psf model fit. (ndof?, npix?) +psfLikelihood dimensionless REAL 4 -999 PSF likelihood +momentXX arcsec^2 REAL 4 -999 moment XX +momentXY arcsec^2 REAL 4 -999 moment XY +momentYY arcsec^2 REAL 4 -999 momeny YY +momentR1 arcsec^1 REAL 4 -999 first radial moment +momentRH arcsec REAL 4 -999 to be half light radius +momentM3C arcsec^3 REAL 4 -999 3rd moment sin(theta) +momentM3S arcsec^3 REAL 4 -999 3rd moment cos(theta) +momentM4C arcsec^4 REAL 4 -999 4th moment sin(theta) +momentM4S arcsec^4 REAL 4 -999 4th moment cos(theta) + +lensObjSmearX11 arcsec^-2 REAL 4 -999 per-object smear polarizability X11 term +lensObjSmearX12 arcsec^-2 REAL 4 -999 per-object smear polarizability X12 term +lensObjSmearX22 arcsec^-2 REAL 4 -999 per-object smear polarizability X22 term +lensObjSmearE1 arcsec^-2 REAL 4 -999 per-object smear polarizability E1 term +lensObjSmearE2 arcsec^-2 REAL 4 -999 per-object smear polarizability E2 term +lensObjShearX11 dimensionless REAL 4 -999 per-object shear polarizability X11 term +lensObjShearX12 dimensionless REAL 4 -999 per-object shear polarizability X12 term +lensObjShearX22 dimensionless REAL 4 -999 per-object shear polarizability X22 term +lensObjShearE1 dimensionless REAL 4 -999 per-object shear polarizability E1 term +lensObjShearE2 dimensionless REAL 4 -999 per-object shear polarizability E2 term +lensPSFSmearX11 arcsec^-2 REAL 4 -999 psf model smear polarizability X11 term +lensPSFSmearX12 arcsec^-2 REAL 4 -999 psf model smear polarizability X12 term +lensPSFSmearX22 arcsec^-2 REAL 4 -999 psf model smear polarizability X22 term +lensPSFSmearE1 arcsec^-2 REAL 4 -999 psf model smear polarizability E1 term +lensPSFSmearE2 arcsec^-2 REAL 4 -999 psf model smear polarizability E2 term +lensPSFShearX11 dimensionless REAL 4 -999 psf model shear polarizability X11 term +lensPSFShearX12 dimensionless REAL 4 -999 psf model shear polarizability X12 term +lensPSFShearX22 dimensionless REAL 4 -999 psf model shear polarizability X22 term +lensPSFShearE1 dimensionless REAL 4 -999 psf model shear polarizability E1 term +lensPSFShearE2 dimensionless REAL 4 -999 psf model shear polarizability E2 term + +apFlux adu/seconds REAL 4 -999 Aperture flux +apFluxErr adu/seconds REAL 4 -999 Aperture flux error +apFillF dimensionless REAL 4 -999 Aperture fill factor +kronFlux adu/seconds REAL 4 -999 Kron flux +kronFluxErr adu/seconds REAL 4 -999 Kron flux error +kronRad arcsec REAL 4 -999 Kron radius +kronRadErr arcsed REAL 4 -999 Kron radius error +------------------------------------------------------ +flxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +flxR5Err adu/seconds REAL 4 -999 estimated error is flxR5 +flxR5Std adu/seconds REAL 4 -999 estimated standard deviation in flxR5 +flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +flxR6 adu/seconds REAL 4 -999 Flux inside r = 6 +flxR6Err adu/seconds REAL 4 -999 estimated error is flxR6 +flxR6Std adu/seconds REAL 4 -999 estimated standard deviation in flxR6 +flxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +------------------------------------------------------ +sky adu/sec/pixel REAL 4 -999 PSF sky level at source +skyErr adu/sec/pixel REAL 4 -999 estimated error in sky +infoFlag dimensionless BIGINT 8 -999 flag indicating provenance information +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. Recalibrations do not affect this value. + +-- [295 byte/det] Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/MeanObject (revision 36826) @@ -0,0 +1,27 @@ + +-- The "best available" measurements refer first to those objects with qfPerfect > 0.85 and good flags, +-- but descends down a "waterfall logic" in ipp so that there exists a mean mag for less than ideal objects. The logic is given in Magnier etal 2013. + +objID dimensionless BIGINT 8 NA ODM object identifier index +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +gQfPerfect dimensionless REAL 4 -999 maximum psf qfPerfect from best available g band detections +gMeanPSFMag AB magnitude REAL 4 -999 mean g psf AB magnitude from best available measurements (best available as current defined by ipp "waterfall logic" +gMeanPSFMagErr AB magnitude REAL 4 -999 estimated error of best availalbe g-band psf AB magnitudes (individual errors added in quadrature) +gnIncPSFMag dimensionless SMALLINT2 -999 number of best available detections included in mean psfMag +gMeanPSFMagStd dimensioless REAL 4 -999 standard deviation of best available measurements - compared with MeanErr this is one measure of true variability +gMinPSFMag AB magnitude REAL 4 -999 minimum best available PSFMag - min and max are given for PSFMag as they may indicate variability. +gMaxPSFMag AB magnitude REAL 4 -999 maximum best available PSFMag + +gMeanKronMag AB magnitude REAL 4 -999 mean g kron AB magnitude from best available measurements +gMeanKronMagErr AB magnitude REAL 4 -999 estimated error of mean g kron AB magnitude +gnIncKronMag dimensionless SMALLINT2 -999 number of best available detections included mean Kron mag +gMeanKronMagStd dimensioless REAL 4 -999 standard deviation of best available measurements + +gMeanApMag AB magnitude REAL 4 -999 mean g kron AB magnitude from best available measurements +gMeanApMagErr AB magnitude REAL 4 -999 estimated error of g kron AB magnitude mean +gnIncApMag dimensionless SMALLINT2 -999 number of detections included in best available mean of Kron mag in g filter +gMeanApMagStd dimensioless REAL 4 -999 standard deviation of best available measurements +gFlags dimensionless BIGINT 8 NA flags for g mean +-- duplicate for other filters + +[62 byte/obj/filter + 8 byte/filter] Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/ObjectThin (revision 36826) @@ -0,0 +1,51 @@ + +objName string VAR 8 NA IAU Name constructed from objID postion PS1.x JHHMMSS.SS+/-DDMMSS.S where x is leading number of procesing version +objID dimensionless BIGINT 8 NA ODM object identifier index +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +ippObjID dimensionless BIGINT 8 NA IPP object number +surveyID dimensionless TINYINT 1 255 Survey ID +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +htmID dimensionless BIGINT 8 NA HTM index +zoneID dimensionless INT 4 NA Zone index +randomID dimensionless BIGINT 8 0 Random integer between 0 and 2^32 = 4,394,967,296 remaining ramdom integer 0 +batchID dimensionless BIGINT 8 -999 batch id +dvoRegionID dimensionless REAL 4 -999 identifier of dvo spatial region - dvo cat id +dataRelease dimensionless TINYINT 1 NA Data release Version of this table +objInfoFlag dimensionless BIGINT 8 -999 flag indicating odd properties +qualityFlag dimensionless TINYINT 1 0 subset of objInfoFlag - flags related to goodness of object +consistencyFlag dimensionless SMALLINT 2 0 Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation +raStack degrees FLOAT 8 -999 Right Ascension - best weighted RA across filters from stack WHENEVER an object is detected in the stack +decStack degrees FLOAT 8 -999 Declination - best weighted Dec across filters from stack WHENEVER an object is detected in the stack +raStackErr arcsec REAL 4 -999 Error in stack Right Ascension, note will have a systematic floor, and is not necessary the best ra,dec for all objects +decStackErr arcsec REAL 4 -999 Error in stack Declination, note will have a systematic floor, and is not necessarily the best ra,dec for all objects +raMean degrees FLOAT 8 -999 Right Ascension from the Mean of RA from individual detections if it exists. This is for information only +decMean degrees FLOAT 8 -999 Declination from the Mean of Dec from individual detections if it exists This is for information only. +raMeanErr arcsec REAL 4 -999 Combination of the measurement Errors in the Right Ascension from the mean detections if they exist +decMeanErr arcsec REAL 4 -999 Combination of the measurement Errors in the Declination measurements if they exist +raMeanStd arcsec Real 4 -999 standard deviation of Right Ascension measurements in detections if they exist +decMeanStd arcsec REAL 4 -999 standard deviation of the Declination measurements if they exist +-------------------------------------------------------------------------------------------------------- +-- Note: the following three sets of coordinate systems are based on stack if it exists, and mean if it does not. +-- This means cone or box searches work on the stack and consistent with Primary and Best stack detection. +-- For objects without a stack postion, (moving, transient, or false dectection) a cone search may be incomplete for +-- an object within less than 1 arc sec of the boundary edge. But science cases here will want to work from Detection +-- Table anyway, or can vary cone search by 1'' radius to check for incompleteness. +cx dimensionless FLOAT 8 NA Cartesian x from Stack RA,Dec WHENEVER a stack detection exists, and Mean RA, Dec (mean of one?) when it doesn't +cy dimensionless FLOAT 8 NA Cartesian y from Stack RA,Dec WHENEVER a stack detection exists, and mean RA, Dec (mean of one?) when it doesn't +cz dimensionless FLOAT 8 NA Cartesian z from Stack RA,Dec WHENEVER a stack detection exists, ditto +lambda degrees FLOAT 8 -999 ecliptic longitude from Stack RA,Dec WHENEVER A STACK MEASURMENT EXISTS and Mean RA,Dec (mean of one?) otherwise +beta degrees FLOAT 8 -999 ecliptic latitude ditto +l degrees FLOAT 8 -999 galactic longitude ditto +b degrees FLOAT 8 -999 galactic latitude ditto +--------------------------------------------------------------------------------- +nStackObjectRows dimensionless SMALLINT 2 -999 number of independent StackObjectRows this object has. +-- zero if not detected in the Stack, 1 if there is a primary, 2 or more if it is in an overlap region +nStackDetections dimensionless TINYINT 2 255 which filter bands it is detected in (not forced) or if it was masked in a band. +nDetections dimensionless SMALLINT 2 -999 total number of detection measurements in all filters +ng dimensionless SMALLINT 2 -999 number of detections in individual g filter exposures +nr dimensionless SMALLINT 2 -999 number of detections in individual r filter exposures +ni dimensionless SMALLINT 2 -999 number of detections in individual i filter exposures +nz dimensionless SMALLINT 2 -999 number of detections in individual z filter exposures +ny dimensionless SMALLINT 2 -999 number of detections in individual y filter exposures + +[197 byte/obj] Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackApFlx (revision 36826) @@ -0,0 +1,60 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +surveyID dimensionless TINYINT 1 0 stack type identifier +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +dataRelease dimensionless TINYINT 1 NA Data release when this table was loaded. +------------------------------------------------------- replicate to end for each filter g,r,i,z,y +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +gippObjID dimensionless BIGINT 8 NA ipp object identifier +gippDetectID dimensionless BIGINT 8 NA stack identifier +--------------------------------------------------------unconvolved +gflxR3 adu/seconds REAL 4 -999 Flux inside r = 3 +gflxR3Err adu/seconds REAL 4 -999 estimated error is flxR3 +gflxR3Std adu/seconds REAL 4 -999 estimated standard deviation in flxR3 +gflxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gflxR4 adu/seconds REAL 4 -999 Flux inside r = 4 +gflxR4Err adu/seconds REAL 4 -999 estimated error is flxR4 +gflxR4Std adu/seconds REAL 4 -999 estimated standard deviation in flxR4 +gflxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gflxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +gflxR5Err adu/seconds REAL 4 -999 estimated error is flxR5 +gflxR5Std adu/seconds REAL 4 -999 estimated standard deviation in flxR5 +gflxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +---------------------------------------------------- convolved across all bands to 6 or 8 pixels +gc1flxR3 adu/seconds REAL 4 -999 Flux inside r = 3 +gc1flxR3Err adu/seconds REAL 4 -999 estimated error is c1flxR3 +gc1flxR3Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR3 +gc1flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc1flxR4 adu/seconds REAL 4 -999 Flux inside r = 4 +gc1flxR4Err adu/seconds REAL 4 -999 estimated error is c1flxR4 +gc1flxR4Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR4 +gc1flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc1flxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +gc1flxR5Err adu/seconds REAL 4 -999 estimated error is c1flxR5 +gc1flxR5Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR5 +gc1flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +---------------------------------------------------- convolved across all bands to 6 or 8 pixels +gc2flxR3 adu/seconds REAL 4 -999 Flux inside r = 3 +gc2flxR3Err adu/seconds REAL 4 -999 estimated error in c2flxR3 +gc2flxR3Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR3 +gc2flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc2flxR4 adu/seconds REAL 4 -999 Flux inside r = 4 +gc2flxR4Err adu/seconds REAL 4 -999 estimated error in c2flxR4 +gc2flxR4Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR4 +gc2flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc2flxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +gc2flxR5Err adu/seconds REAL 4 -999 estimated error in c2flxR5 +gc2flxR5Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR5 +gc2flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 + +-- repeat for other filters with appropriate leading prefex. +-- [168 byte/filter/obj + 44 byte/obj] + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFit =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFit (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackModelFit (revision 36826) @@ -0,0 +1,204 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. +-- the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +dataRelease dimensionless TINYINT 1 NA Data release when this table was loaded +-------------------------------------------------------- replicate below for all filters (grizy) +glogC dimensionless REAL 4 -999 concentration index +glogA dimensionless REAL 4 -999 asymmetry index +gclump dimensionless REAL 4 -999 clumpiness factor +-------------------------------------------------------- replicate below for all filters (grizy) +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +gippObjID dimensionless BIGINT 8 NA ipp object identifier +gippDetectID dimensionless BIGINT 8 NA ipp detect identifier +gdeVRadius arcsec REAL 4 -999 deVaucouleurs radius +gdeVRadiusErr arcsec REAL 4 -999 estimated error in deVaucouleurs radius +gdeVFlux adu/seconds REAL 4 -999 deVaucouleurs flux +gdeVFluxErr adu/seconds REAL 4 -999 estimated error in deVaucouleurs flux +gdeVAb dimensionless REAL 4 -999 deVaucoulerus axis ratio +gdeVAbErr dimensionless REAL 4 -999 estimated error in deVaucoulerus axis ratio +gdeVPhi degrees REAL 4 -999 estmated phi of deVaucouleurs axis. Degrees East of North. +gdeVPhiErr degrees REAL 4 -999 estmated error of phi of deVaucouleurs axis +graDeVOff arcsec REAL 4 -999 Offset in RA of deVaucouleurs fit from PSF RA +gdecDeVOff arcsec REAL 4 -999 Offset in DEC of deVaucouleurs fit from PSF DEC +graDeVOffErr arcsec REAL 4 -999 estimated error in ra offset +gdecDeVOffErr arcsec REAL 4 -999 estimated error in dec offset +gdeVCf dimensionless REAL 4 -999 deVaucouleurs fit coverage factor +gdeVLikelihood dimensionless REAL 4 -999 deVaucouleurs fit likelihood factor +---------------------------------------------------------------------------------------- +gexpRadius arcsec REAL 4 -999 Exponential fit radius +gexpRadiusErr arcsec REAL 4 -999 estimated error in Exponential fit radius +gexpFlux adu/seconds REAL 4 -999 Exponential fit flux +gexpFluxErr adu/seconds REAL 4 -999 estimated error in expFlux +gexpAb dimensionless REAL 4 -999 Exponential fit axis ratio +gexpAbErr dimensionless REAL 4 -999 estimated error in Exponential fit axis ratio +gexpPhi degrees REAL 4 -999 estmated phi of Exponential axis +gexpPhiErr degrees REAL 4 -999 estmated error of phi of Exponential axis +graExpOff dimensionless REAL 4 -999 Offset in RA of Exponential fit from PSF RA +gdecExpOff dimensionless REAL 4 -999 Offset in DEC of Exponential fit from PSF DEC +graExpOffErr dimensionless REAL 4 -999 estimated error in raExpOff +gdecExpOffErr dimensionless REAL 4 -999 estimated error in decExpOff +gexpCf dimensionless REAL 4 -999 Exponential fit coverage factor +gexpLikelihood dimensionless REAL 4 -999 Exponential fit likelihood factor +--------------------------------------------------------------------------------------- +gserRadius arcsec REAL 4 -999 Sersic radius +gserRadiusErr arcsed REAL 4 -999 estimated error in Sersic radius +gserFlux adu/seconds REAL 4 -999 Sersic Flux +gserFluxErr adu/seconds REAL 4 -999 estimated error in serFlux +gserAb dimensionless REAL 4 -999 Sersic axis ratio +gserAbErr dimensionlessn REAL 4 -999 estimated error in Sersic axis ratio +gserNu dimensionlessn REAL 4 -999 Sersic index +gserNuErr diemensionless REAL 4 -999 estimated error in Sersic index +gserPhi degrees REAL 4 -999 estmated phi of Sersic axis +gserPhiErr degrees REAL 4 -999 estmated error of phi of Sersic axis +graSerOff dimensionless REAL 4 -999 Offset in RA of Sersic fit from PSF RA +gdecSerOff dimensionless REAL 4 -999 Offset in DEC of Sersic fit from PSF DEC +graSerOffErr dimensionless REAL 4 -999 estimated error in raSerOff +gdecSerOffErr dimensionless REAL 4 -999 estimated error in decSerOff +gserCf dimensionless REAL 4 -999 Sersic fit coverage factor +gserLikelihood dimensionless REAL 4 -999 Sersic fit likelihood factor +--------------------------------------------------------------------------------------- +gpetRadius pixels REAL 4 -999 Petrosian radius +gpetRadiusErr pixels REAL 4 -999 estimated error inPetrosian radius +gpetFlux adu/sec REAL 4 -999 Petrosian Flux +gpetFluxErr adu/sec REAL 4 -999 estimated error in petFlux +gpetR50 pixels REAL 4 -999 Petrosian radius at 50% light +gpetR50Err pixels REAL 4 -999 estimated error inPetrosian radius at 50% light +gpetR90 pixels REAL 4 -999 Petrosian radius at 90% light +gpetR90Err pixels REAL 4 -999 estimated error in Petrosian radius at 90% light +gpetCf dimensionless REAL 4 -999 Petrosian fit coverage factor +--------------------------------------------------------unconvolved +gflxR1 adu/seconds REAL 4 -999 Flux inside r = 1 +gflxR1Err adu/seconds REAL 4 -999 estimated error is flxR1 +gflxR1Std adu/seconds REAL 4 -999 estimated standard deviation in flxR1 +gflxR1Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gflxR2 adu/seconds REAL 4 -999 Flux inside r = 2 +gflxR2Err adu/seconds REAL 4 -999 estimated error is flxR2 +gflxR2Std adu/seconds REAL 4 -999 estimated standard deviation in flxR2 +gflxR2Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gflxR3 adu/seconds REAL 4 -999 Flux inside r = 3 +gflxR3Err adu/seconds REAL 4 -999 estimated error is flxR3 +gflxR3Std adu/seconds REAL 4 -999 estimated standard deviation in flxR3 +gflxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gflxR4 adu/seconds REAL 4 -999 Flux inside r = 4 +gflxR4Err adu/seconds REAL 4 -999 estimated error is flxR4 +gflxR4Std adu/seconds REAL 4 -999 estimated standard deviation in flxR4 +gflxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gflxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +gflxR5Err adu/seconds REAL 4 -999 estimated error is flxR5 +gflxR5Std adu/seconds REAL 4 -999 estimated standard deviation in flxR5 +gflxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gflxR6 adu/seconds REAL 4 -999 Flux inside r = 6 +gflxR6Err adu/seconds REAL 4 -999 estimated error is flxR6 +gflxR6Std adu/seconds REAL 4 -999 estimated standard deviation in flxR6 +gflxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gflxR7 adu/seconds REAL 4 -999 Flux inside r = 7 +gflxR7Err adu/seconds REAL 4 -999 estimated error is flxR7 +gflxR7Std adu/seconds REAL 4 -999 estimated standard deviation in flxR7 +gflxR7Fill dimensionless REAL 4 -999 estimated filling factor at R7 +gflxR8 adu/seconds REAL 4 -999 Flux inside r = 8 +gflxR8Err adu/seconds REAL 4 -999 estimated error is flxR8 +gflxR8Std adu/seconds REAL 4 -999 estimated standard deviation in flxR8 +gflxR8Fill dimensionless REAL 4 -999 estimated filling factor at R8 +gflxR9 adu/seconds REAL 4 -999 Flux inside r = 9 +gflxR9Err adu/seconds REAL 4 -999 estimated error is flxR9 +gflxR9Std adu/seconds REAL 4 -999 estimated standard deviation in flxR9 +gflxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9 +gflxR10 adu/seconds REAL 4 -999 Flux inside r = 10 +gflxR10Err adu/seconds REAL 4 -999 estimated error is flxR10 +gflxR10Std adu/seconds REAL 4 -999 estimated standard deviation in flxR10 +gflxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10 +---------------------------------------------------- convolved across all bands to 6 or 8 pixels +gc1flxR1 adu/seconds REAL 4 -999 Flux inside r = 1 +gc1flxR1Err adu/seconds REAL 4 -999 estimated error is c1flxR1 +gc1flxR1Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR1 +gc1flxR1Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc1flxR2 adu/seconds REAL 4 -999 Flux inside r = 2 +gc1flxR2Err adu/seconds REAL 4 -999 estimated error is c1flxR2 +gc1flxR2Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR2 +gc1flxR2Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc1flxR3 adu/seconds REAL 4 -999 Flux inside r = 3 +gc1flxR3Err adu/seconds REAL 4 -999 estimated error is c1flxR3 +gc1flxR3Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR3 +gc1flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc1flxR4 adu/seconds REAL 4 -999 Flux inside r = 4 +gc1flxR4Err adu/seconds REAL 4 -999 estimated error is c1flxR4 +gc1flxR4Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR4 +gc1flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc1flxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +gc1flxR5Err adu/seconds REAL 4 -999 estimated error is c1flxR5 +gc1flxR5Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR5 +gc1flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gc1flxR6 adu/seconds REAL 4 -999 Flux inside r = 6 +gc1flxR6Err adu/seconds REAL 4 -999 estimated error is c1flxR6 +gc1flxR6Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR6 +gc1flxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gc1flxR7 adu/seconds REAL 4 -999 Flux inside r = 7 +gc1flxR7Err adu/seconds REAL 4 -999 estimated error is c1flxR7 +gc1flxR7Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR7 +gc1flxR7Fill dimensionless REAL 4 -999 estimated filling factor at R7 +gc1flxR8 adu/seconds REAL 4 -999 Flux inside r = 8 +gc1flxR8Err adu/seconds REAL 4 -999 estimated error is c1flxR8 +gc1flxR8Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR8 +gc1flxR8Fill dimensionless REAL 4 -999 estimated filling factor at R8 +gc1flxR9 adu/seconds REAL 4 -999 Flux inside r = 9 +gc1flxR9Err adu/seconds REAL 4 -999 estimated error is c1flxR9 +gc1flxR9Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR9 +gc1flxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9 +gc1flxR10 adu/seconds REAL 4 -999 Flux inside r = 10 +gc1flxR10Err adu/seconds REAL 4 -999 estimated error is c1flxR10 +gc1flxR10Std adu/seconds REAL 4 -999 estimated standard deviation in c1flxR10 +gc1flxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10 +gc2flxR1 adu/seconds REAL 4 -999 Flux inside r = 1 +gc2flxR1Err adu/seconds REAL 4 -999 estimated error is c2flxR1 +gc2flxR1Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR1 +gc2flxR1Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc2flxR2 adu/seconds REAL 4 -999 Flux inside r = 2 +gc2flxR2Err adu/seconds REAL 4 -999 estimated error is c2flxR2 +gc2flxR2Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR2 +gc2flxR2Fill dimensionless REAL 4 -999 estimated filling factor at R1 +gc2flxR3 adu/seconds REAL 4 -999 Flux inside r = 3 +gc2flxR3Err adu/seconds REAL 4 -999 estimated error is c2flxR3 +gc2flxR3Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR3 +gc2flxR3Fill dimensionless REAL 4 -999 estimated filling factor at R3 +gc2flxR4 adu/seconds REAL 4 -999 Flux inside r = 4 +gc2flxR4Err adu/seconds REAL 4 -999 estimated error is c2flxR4 +gc2flxR4Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR4 +gc2flxR4Fill dimensionless REAL 4 -999 estimated filling factor at R4 +gc2flxR5 adu/seconds REAL 4 -999 Flux inside r = 5 +gc2flxR5Err adu/seconds REAL 4 -999 estimated error is c2flxR5 +gc2flxR5Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR5 +gc2flxR5Fill dimensionless REAL 4 -999 estimated filling factor at R5 +gc2flxR6 adu/seconds REAL 4 -999 Flux inside r = 6 +gc2flxR6Err adu/seconds REAL 4 -999 estimated error is c2flxR6 +gc2flxR6Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR6 +gc2flxR6Fill dimensionless REAL 4 -999 estimated filling factor at R6 +gc2flxR7 adu/seconds REAL 4 -999 Flux inside r = 7 +gc2flxR7Err adu/seconds REAL 4 -999 estimated error is c2flxR7 +gc2flxR7Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR7 +gc2flxR7Fill dimensionless REAL 4 -999 estimated filling factor at R7 +gc2flxR8 adu/seconds REAL 4 -999 Flux inside r = 8 +gc2flxR8Err adu/seconds REAL 4 -999 estimated error is c2flxR8 +gc2flxR8Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR8 +gc2flxR8Fill dimensionless REAL 4 -999 estimated filling factor at R8 +gc2flxR9 adu/seconds REAL 4 -999 Flux inside r = 9 +gc2flxR9Err adu/seconds REAL 4 -999 estimated error is c2flxR9 +gc2flxR9Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR9 +gc2flxR9Fill dimensionless REAL 4 -999 estimated filling factor at R9 +gc2flxR10 adu/seconds REAL 4 -999 Flux inside r = 10 +gc2flxR10Err adu/seconds REAL 4 -999 estimated error is c2flxR10 +gc2flxR10Std adu/seconds REAL 4 -999 estimated standard deviation in c2flxR10 +gc2flxR10Fill dimensionless REAL 4 -999 estimated filling factor at R10 + +-- repeat for other filters with appropriate leading prefex. + +-- [728 byte/filter/obj + 44 byte/obj] + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/Fundamental_IPP_Data_Product_Tables/StackObject (revision 36826) @@ -0,0 +1,63 @@ + +objID dimensionless BIGINT 8 NA ODM object identifier +gcObjID dimensionless BIGINT 8 NA ODM galactic coordinates identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +skyCellID dimensionless REAL 4 -999 single float giving PrimaryCell.SkyCell identifier +randomStackObjID dimensionless BIGINT 8 NA random stack id +stackDetectRowID dimensionless BIGINT 8 NA identifies which of up to 5 possible stackdetctions this Row refers to: Primary, up to 3 Secondary, 1 best +primaryDetection dimensionless TINYINT 1 255 identifies if this row is a primary stack detection. True about 80% of time. +bestDetection dimensionless TINYINT 1 255 identifies if this row is a best detection or not. Also set if only a primary detection exists. + the Primary Dectection, 0 to 3 possible secondary dectections, and the "best" per filter. +dvoRegionID dimensionless REAL 4 -999 identifier to dvo spatial region +dataRelease dimensionless TINYINT 1 NA Data release when this detection was originally taken. +------------------------------------------------------------------------------------------ +-- attributes for all stack detections in all filters (these are currently forced anyway) +-- Replicate the section below for each filter, changing the prefex ("g" in this example) +------------------------------------------------------------------------------------------ +gippObjID dimensionless BIGINT 8 NA object ID generated by IPP +gippDetectID dimensionless BIGINT 8 NA detection ID generated by IPP +gstackDetectID dimensionless BIGINT 8 NA ODM detection identifier +gstackMetaID dimensionless BIGINT 8 NA stack identifier +gxPos arcsec REAL 4 -999 measured x in sky Cell from PSF fit +gyPos arcsec REAL 4 -999 measured y in sky Cell from PSF fit +gxPosErr arcsec REAL 4 -999 estimated error in x +gyPosErr arcsec REAL 4 -999 estimated error in y +gra degrees FLOAT 8 NA Right Ascension +gdec degrees FLOAT 8 NA Declination +graErr arcsec REAL 4 NA Right Ascension error +gdecErr arcsec REAL 4 NA Declination error (same as RA error - need?) +gzp magnitude REAL 4 0 zeropoint , no error given +gexpTime seconds REAL 4 -999 total exposure time (informational) +gpsfMajorFWHM arcsec REAL 4 -999 PSF width in major axis +gpsfMinorFWHM arcsec REAL 4 -999 PSF width in minor axis +gpsfTheta degrees REAL 4 -999 PSF orientation angle. Degrees East of North. +gpsfCore dimensionless REAL 4 -999 PSF core parameter [k in (1 + kz + z^1.66)] +gpsfLikelihood dimensionless REAL 4 -999 PSF likelihood +gpsfQf dimensionless REAL 4 -999 PSF coverage factor +gpsfQfPerfect dimensionless REAL 4 -999 PSF-weighted fraction of arcsec totally unmasked +gpsfChiSq dimensionless REAL 4 -999 PSF-weighted fraction of arcsec to. +gmomentXX arcsec^2 REAL 4 -999 moment XX +gmomentXY arcsec^2 REAL 4 -999 moment XY +gmomentYY arcsec^2 REAL 4 -999 moment YY +gmomentR1 arcsec REAL 4 -999 moment R1 +gmomentRH arcsec REAL 4 -999 moment RH - or half light radius +gPSFFlux adu/sec REAL 4 -999 g psf flux from stack +gPSFFluxErr adu/sec REAL 4 -999 estimated error of g psf flux from stack +gApFlux adu/seconds REAL 4 -999 Aperture magnitude +gApFluxErr adu/seconds REAL 4 -999 Aperture magnitude error +gApFillFac dimensionless REAL 4 -999 Aperture fill factor - ~ redundant with psfQF, keep? +gKronFlux adu/seconds REAL 4 -999 Kron flux +gKronFluxErr adu/seconds REAL 4 -999 Kron flux error +gKronRad arcsec REAL 4 -999 Kron radius +gPSFMag AB magnitude REAL 4 -999 g psf AB magnitude from stack +gPSFMagErr AB magnitude REAL 4 -999 estimated error of g psf AB magnitude from stack +gApMag AB magnitude REAL 4 -999 g Aperture AB magnitude from stack +gApMagErr AB magnitude REAL 4 -999 estimated error of g Ap Mag AB magnitude from stack +gKronMag AB magnitude REAL 4 -999 g kron AB magnitude from stack +gKronMagErr AB magnitude REAL 4 -999 estimated error of g kron AB magnitude from stack +gsky adu/secs/arcsec REAL 4 -999 PSF sky level at source (adu) +gskyErr adu/secs/arcsec REAL 4 -999 estimated error in sky +ginfoFlag dimensionless BIGINT 8 -999 indicator of strange propeties +gnFrames dimensionless INT 4 -999 Number of frames contributing to source + +-- [220 byte/filter/obj + 44 byte/obj] Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/CameraConfig =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/CameraConfig (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/CameraConfig (revision 36826) @@ -0,0 +1,12 @@ +CameraConfig +Description: Camera configuration + +Name Unit DataType Size Default ValueDescription + +cameraID dimensionless SMALLINT 2 -999 camera identifier (i.e. GPC1, ISP) +cameraConfigID dimensionless SMALLINT 2 -999 camera configuration +endDate dimensionless REAL 4 -999 ending date of this configuration in MJD +configDetails string VARCHAR(MAX) 8000 0x00 NA text/table giving the details of the camera configuration + NEED TO CHANGE and pass camera x from raw data file + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DetectionFlags =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DetectionFlags (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DetectionFlags (revision 36826) @@ -0,0 +1,10 @@ +DetectionFlags + +Description: Flags used to interpret the information flag provided +with each detection (infoFlag in 'Detection' table) + +Name Unit Data Type Size Default Value Description + +name VARCHAR(32) 32 NA Name of flag +value BIGINT 8 NA Flag value +description VARCHAR(2000) 2000 NA Description of the flag Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DiffMeta =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DiffMeta (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DiffMeta (revision 36826) @@ -0,0 +1,26 @@ +diffMetaID dimensionless BIGINT 8 NA diff identifier +batchID dimensionless BIGINT 8 NA batch identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA filter id +diffTypeID dimensionless TINYINT 1 0 diff type identifier +posImageID dimensionless BIGINT 8 NA positive image identifier +negImageID dimensionless BIGINT 8 NA negative image identifier +skyCellID dimensionless REAL 4 NA skycell region identifier +photoCalID dimensionless INT 4 NA photometry code +magSat dimensionless REAL 4 -999 saturation magnitude level +analVer dimensionless SMALLINT 2 -999 analysis version index +expTime seconds REAL 4 -999 exposure time of positive image +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 deg/pixel FLOAT 8 -999 scale factor for RA +cdelt2 deg/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DiffToImage =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DiffToImage (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/DiffToImage (revision 36826) @@ -0,0 +1,2 @@ +diffMetaID BIGINT 8 NA diff identifier +imageID BIGINT 8 NA hashed exposure-ccdID identifier Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Filter =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Filter (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Filter (revision 36826) @@ -0,0 +1,3 @@ +FilterID dimensionless TINYNT 1 NA filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... +filterType VARCHAR(2) 2 NA filter name: g,r,i,z,y, etc. +filterSpec VARCHAR(MAX) 8000 0x00 No comment Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/FitModel =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/FitModel (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/FitModel (revision 36826) @@ -0,0 +1,42 @@ +fitModelID SMALLINT 2 NA indentifier of alternate model fit +name VARCHAR(30) 30 model name +description VARCHAR(MAX) 8000 string describing the model +param1 VARCHAR(MAX) 8000 description of model parameter 1 +param2 VARCHAR(MAX) 8000 description of model parameter 2 +param3 VARCHAR(MAX) 8000 description of model parameter 3 +param4 VARCHAR(MAX) 8000 description of model parameter 4 +param5 VARCHAR(MAX) 8000 description of model parameter 5 +param6 VARCHAR(MAX) 8000 description of model parameter 6 +param7 VARCHAR(MAX) 8000 description of model parameter 7 +param8 VARCHAR(MAX) 8000 description of model parameter 8 +param9 VARCHAR(MAX) 8000 description of model parameter 9 +param10 VARCHAR(MAX) 8000 description of model parameter 10 +param11 VARCHAR(MAX) 8000 description of model parameter 11 +param12 VARCHAR(MAX) 8000 description of model parameter 12 +param13 VARCHAR(MAX) 8000 description of model parameter 13 +param14 VARCHAR(MAX) 8000 description of model parameter 14 +param15 VARCHAR(MAX) 8000 description of model parameter 15 +param16 VARCHAR(MAX) 8000 description of model parameter 16 +param17 VARCHAR(MAX) 8000 description of model parameter 17 +param18 VARCHAR(MAX) 8000 description of model parameter 18 +param19 VARCHAR(MAX) 8000 description of model parameter 19 +param20 VARCHAR(MAX) 8000 description of model parameter 20 +param21 VARCHAR(MAX) 8000 description of model parameter 21 +param22 VARCHAR(MAX) 8000 description of model parameter 22 +param23 VARCHAR(MAX) 8000 description of model parameter 23 +param24 VARCHAR(MAX) 8000 description of model parameter 24 +param25 VARCHAR(MAX) 8000 description of model parameter 25 +param26 VARCHAR(MAX) 8000 description of model parameter 26 +param27 VARCHAR(MAX) 8000 description of model parameter 27 +param28 VARCHAR(MAX) 8000 description of model parameter 28 +param29 VARCHAR(MAX) 8000 description of model parameter 29 +param30 VARCHAR(MAX) 8000 description of model parameter 30 +param31 VARCHAR(MAX) 8000 description of model parameter 31 +param32 VARCHAR(MAX) 8000 description of model parameter 32 +param33 VARCHAR(MAX) 8000 description of model parameter 33 +param34 VARCHAR(MAX) 8000 description of model parameter 34 +param35 VARCHAR(MAX) 8000 description of model parameter 35 +param36 VARCHAR(MAX) 8000 description of model parameter 36 +param37 VARCHAR(MAX) 8000 description of model parameter 37 +param38 VARCHAR(MAX) 8000 description of model parameter 38 +param39 VARCHAR(MAX) 8000 description of model parameter 39 Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ForcedWarpMeta =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ForcedWarpMeta (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ForcedWarpMeta (revision 36826) @@ -0,0 +1,33 @@ +forcedWarpID dimensionless BIGINT 8 NA stack identifier +batchID dimensionless BIGINT 8 NA batch identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA fnilutemr eidriecnatl ifidier +skyCellID dimensionless REAL 4 NA skycell region identifier +photoCalID dimensionless INT 4 NA photometry code +magSat dimensionless REAL 4 -999 saturation magnitude level +analVer dimensionless SMALLINT 2 -999 analysis version index +expTime seconds REAL 4 -999 exposure time +completMag magnitudes REAL 4 -999 95% completion level in mag +astroScat dimensionless REAL 4 -999 astrometric scatter for chip +photoScat dimensionless REAL 4 -999 photometric scatter for chip +nAstroRef dimensionless INT 4 -999 number of astrometric reference sources +nPhoRef dimensionless INT 4 -999 number of photometric reference sources +psfModelID dimensionless VARCHAR(MAX) 8000 PSF model identifier +psfFwhm_mean arcsec FLOAT 8 -999 mean PSF FWHM in the stack +psfFwhm_max arcsec FLOAT 8 -999 maximum PSF FWHM in the stack +photoZero magnitudes REAL 4 -999 local derived photometric zero point +photoColor magnitudes REAL 4 -999 local derived photometric color term +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 deg/pixel FLOAT 8 -999 scale factor for RA +cdelt2 deg/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ForcedWarpToImage =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ForcedWarpToImage (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ForcedWarpToImage (revision 36826) @@ -0,0 +1,2 @@ +forcedWarpID BIGINT 8 NA forced warp identifier +imageID BIGINT 8 NA hashed exposure-ccdID identifier Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/FrameMeta =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/FrameMeta (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/FrameMeta (revision 36826) @@ -0,0 +1,49 @@ +frameID dimensionless INT 4 NA unique exposure/frame identifier. +frameName dimensionless VARCHAR(32) 32 NA frame name provided by camera software +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA filter identifier +cameraID dimensionless SMALLINT 2 NA camera identifier +cameraConfigID dimensionless SMALLINT 2 NA camera configuration identifier +telescopeID dimensionless SMALLINT 2 NA telescope identifier +analysisVer alphanumeric VARCHAR(MAX) 8000 IPP software analysis release +p1Recip alphanumeric VARCHAR(MAX) 8000 IPP phase 1 MD5 Checksum +p2Recip alphanumeric VARCHAR(MAX) 8000 IPP phase 2 MD5 Checksum +p3Recip alphanumeric VARCHAR(MAX) 8000 IPP phase 3 MD5 Checksum +nOTA alphanumeric SMALLINT 2 -999 number of valid OTA/CCD images in this exposure +photoScat magnitude REAL 4 -999 global photometric scatter +numPhotoRef dimensionless INT 4 -999 number of photometric reference sources +expStart days FLOAT 8 -999 exposure start time in MJD +expTime seconds REAL 4 -999 exposure time +airmass dimensionless REAL 4 -999 airmass at mid-exposure +raBore degrees FLOAT 8 -999 RA of telescope boresight +decBore degrees FLOAT 8 -999 DEC of telescope boresight +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 pc001001 FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 degrees/pixel FLOAT 8 -999 scale factor for RA +cdelt2 degrees/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +polyOrder dimensionless TINYINT 1 255 polynomial order of astrometry fit +pca1x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +calibModNum dimensionless SMALLINT 2 NA calibration modification number +batchID dimensionless BIGINT 8 NA batch ID +dataRelease dimensionless TINYINT 1 NA Data release Version of this table Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ImageFlags =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ImageFlags (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ImageFlags (revision 36826) @@ -0,0 +1,3 @@ +name VARCHAR(32) 32 NA Name of flag +value BIGINT 8 NA Flag value +description VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ImageMeta =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ImageMeta (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ImageMeta (revision 36826) @@ -0,0 +1,74 @@ +imageID dimensionless BIGINT 8 NA unique ID for each image, hashed from frameID and ccdID +frameID dimensionless INT 4 NA frame identifier, also known as exposure number +ccdID dimensionless SMALLINT 2 NA ccd cell identifier in the focal plane +photoCalID dimensionless INT 4 NA photometry reduction code identifier +filterID dimensionless TINYINT 1 NA filter ID +bias ADU REAL 4 -999 detector bias level +biasScat ADU REAL 4 -999 scatter in bias level +sky ADU/pixel REAL 4 -999 mean sky level +skyScat ADU/pixel REAL 4 -999 scatter in mean sky level +nDetect dimensionless INT 4 -999 number of detections on CCD +magSat magnitude REAL 4 -999 saturation magnitude level +completMag magnitude REAL 4 -999 95% completion level in mag +astroScat arcsec REAL 4 -999 astrometric scatter for chip +photoScat magitude REAL 4 -999 photometric scatter for chip +numAstroRef dimensionless INT 4 -999 number of astrometric reference sources +numPhotoRef dimensionless INT 4 -999 number of photometric reference sources +nx pixels SMALLINT 2 -999 chip dimension in x +ny pixels SMALLINT 2 -999 chip dimension in y +psfModelID dimensionless INT 4 -999 PSF model identifier +psfFwhm arcsec REAL 4 -999 model psf full width at half maximum at chip center +psfWidMajor arcsec REAL 4 -999 model PSF parameters at chip center +psfWidMinor arcsec REAL 4 -999 model PSF parameters at chip center +psfTheta degrees REAL 4 -999 model PSF parameters at chip center +momentXX pixels REAL 4 -999 moment XX +momentXY pixels REAL 4 -999 moment XY +momentYY pixels REAL 4 -999 moment YY +momentM3C pixels squared REAL 4 -999 moment M3C +momentM3S pixels squared REAL 4 -999 moment M3S +momentM4C pixels squared REAL 4 -999 moment M4C +momentM4S pixels squared REAL 4 -999 moment M4S +momentR1 pixels REAL 4 -999 moment R1 +momentRH sqrt pixels REAL 4 -999 moment RH +apResid magnitide REAL 4 -999 corrected aperture residual +dapResid magnitide REAL 4 -999 scatter of aperture corrections +detectorID alphanumeric VARCHAR(32) 32 identifier for actual CCD chip +qaFlags dimensionless BIGINT 8 -999 Q/A flags for this OTA +detrend1 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 1 +detrend2 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 2 +detrend3 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 3 +detrend4 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 4 +detrend5 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 5 +detrend6 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 6 +detrend7 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 7 +detrend8 alphanumeric VARCHAR(MAX) 8000 identifier of detrend image 8 +photoZero magnitude REAL 4 -999 local derived photometric zero point +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 pixels FLOAT 8 -999 reference pixel value for RA +crpix2 pixels FLOAT 8 -999 reference pixel value for DEC +cdelt1 degrees/pixel FLOAT 8 -999 scale factor for RA +cdelt2 degrees/pixel FLOAT 8 -999 scale factor for DEC +pc001001 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 unknown FLOAT 8 -999 elements of rotation/Dcale matrix +polyOrder dimensionless TINYINT 1 255 polynomial order of astrometry fit +pca1x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca1x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x3y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y3 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x2y0 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x1y1 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +pca2x0y2 dimensionless FLOAT 8 -999 polynomial coefficients for the astrometric fit +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Version of this table Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Mask =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Mask (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Mask (revision 36826) @@ -0,0 +1,9 @@ +maskID BIGINT 8 NA mask identifier +partitionKey BIGINT 8 NA fget(mask_center_ra, mask_center_dec, zoneH) +frameID INT 4 NA reference to the P2FrameMeta table where the mask is defined +imageID BIGINT 8 NA reference to the P2FrameMeta table where the mask is +ra FLOAT 8 NA rdae finin ethde center +dec FLOAT 8 NA dec in the center +radius FLOAT 8 NA No comment +maskString VARCHAR(MAX) 8000 string representation fo the mask region +maskBinary VARBINARY(MAX) 8000 0x Binary object managed by the spatial library Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToDet =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToDet (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToDet (revision 36826) @@ -0,0 +1,2 @@ +MovingObjID BIGINT 8 NA Moving Object identifier +DetectionID BIGINT 8 NA detection identifier Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToDiffDet =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToDiffDet (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToDiffDet (revision 36826) @@ -0,0 +1,2 @@ +MovingObjID BIGINT 8 NA Moving Object identifier +DiffDetID BIGINT 8 NA difference image detection identifier Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToObject =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToObject (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/MovingObjToObject (revision 36826) @@ -0,0 +1,2 @@ +MovingObjID BIGINT 8 NA Moving Object identifier +ObjID BIGINT 8 NA Object identifier Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectFilterFlags =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectFilterFlags (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectFilterFlags (revision 36826) @@ -0,0 +1,3 @@ +name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectInfoFlags =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectInfoFlags (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectInfoFlags (revision 36826) @@ -0,0 +1,3 @@ +name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectQualityFlags =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectQualityFlags (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ObjectQualityFlags (revision 36826) @@ -0,0 +1,3 @@ +name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/PhotoCal =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/PhotoCal (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/PhotoCal (revision 36826) @@ -0,0 +1,11 @@ +photoCalID INT 4 NA numerical code for this reduction +filterID TINYINT 1 NA filter id +photoCodeDesc VARCHAR(MAX) 8000 Photometry reduction code name +AB REAL 4 1.0 AB magnitude +zeropoint REAL 4 -999 photometric zero point (excluding long term variation) +extinction REAL 4 -999 photometric extinction term (excluding long term variation) +colorterm REAL 4 -999 photometric color term (excluding long term variation) +colorExtn REAL 4 -999 photometric color dependent extinction (excluding long term variation) +orphanCalColor REAL 4 -999 color adopted for magnitude calculation +orphanCalColorErr REAL 4 -999 error in calibrating color +startDate day REAL 4 NA starting date of this configuration in MJD Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ProjectionCell =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ProjectionCell (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/ProjectionCell (revision 36826) @@ -0,0 +1,22 @@ +projectionCellID INT 4 NA projection cell unique identifier +partitionKey BIGINT 8 NA fGetPanObjID(crval1,crval2) +primaryCellRegionID BIGINT 8 NA reference to the primary cell region definition in the Region table +projectionCellRegionID BIGINT 8 NA reference to the projectio cell region definition in the +ctype1 VARCHAR(MAX) 8000 NA nRaemgieo no ft aabsltero)metric projection in RA +ctype2 VARCHAR(MAX) 8000 NA name of astrometric projection in DEC +crval1 deg FLOAT 8 NA RA corresponding to reference pixel +crval2 deg FLOAT 8 NA DEC corresponding to reference pixel +crpix1 FLOAT 8 NA reference pixel value for RA +crpix2 FLOAT 8 NA reference pixel value for DEC +pc001001 FLOAT 8 NA elements of rotation/Dcale matrix +pc001002 FLOAT 8 NA elements of rotation/Dcale matrix +pc002001 FLOAT 8 NA elements of rotation/Dcale matrix +pc002002 FLOAT 8 NA elements of rotation/Dcale matrix +raNEcornerPC degrees FLOAT 8 -999 ra of NE corner +decNEcornerPC degrees FLOAT 8 -999 dec of NE corner +raNWcornerPC degrees FLOAT 8 -999 ra of NE corner +decNWcornerPC degrees FLOAT 8 -999 dec of NE corner +raSEcornerPC degrees FLOAT 8 -999 ra of NE corner +decSEcornerPC degrees FLOAT 8 -999 dec of NE corner +raSWcornerPC degrees FLOAT 8 -999 ra of NE corner +decSWcornerPC degrees FLOAT 8 -999 dec of NE corner Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/PsConstant =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/PsConstant (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/PsConstant (revision 36826) @@ -0,0 +1,4 @@ +name VARCHAR(32) 32 NA constant's name +value FLOAT 8 NA constant's value +unit VARCHAR(32) 32 unit of constant +description VARCHAR(2000) 2000 description of the constant Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Region =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Region (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Region (revision 36826) @@ -0,0 +1,7 @@ +regionID dimensionless BIGINT 8 NA Unique Region identifier +id dimensionless BIGINT 8 NA Custom Identifier +type string VARCHAR(16) 16 NA Region type +comment string VARCHAR(MAX) 8000 Comment +area sqdeg? FLOAT 8 0 Area covered by the region +region dimensionless Binary VARBINARY(MAX) 8000 0x Binary object managed by the spatial library + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/RegionPatch =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/RegionPatch (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/RegionPatch (revision 36826) @@ -0,0 +1,15 @@ +Unit Data Type Size Default Value Description +regionid BIGINT 8 NA No comment +convexid BIGINT 8 NA No comment +patchid INT 4 NA No comment +type (16) NA No comment +radius FLOAT 8 0 No comment +ra degree FLOAT 8 0 Right Ascension +dec FLOAT 8 0 No comment +x FLOAT 8 0 No comment +y FLOAT 8 0 No comment +z FLOAT 8 0 No comment +c FLOAT 8 0 No comment +htmid BIGINT 8 0 No comment +area FLOAT 8 0 No comment +convexString (MAX) No comment Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/SkyCell =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/SkyCell (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/SkyCell (revision 36826) @@ -0,0 +1,12 @@ +skyCellID dimensionless I INT 4 NA indentifier of the sky cell +projectionCellID INT 4 -999 indentifier of the projection cell +partitionKey BIGINT 8 -999 fgetPANobjID(skyCell_center_ra, skyCell_center_dec, zHeight) +regionID BIGINT 8 -999 region identifier in the Region table +raNEcorner degrees FLOAT 8 -999 ra of NE corner +decNEcorner degrees FLOAT 8 -999 dec of NE corner +raNWcorner degrees FLOAT 8 -999 ra of NE corner +decNWcorner degrees FLOAT 8 -999 dec of NE corner +raSEcorner degrees FLOAT 8 -999 ra of NE corner +decSEcorner degrees FLOAT 8 -999 dec of NE corner +raSWcorner degrees FLOAT 8 -999 ra of NE corner +decSWcorner degrees FLOAT 8 -999 dec of NE corner Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackDetectionFlags =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackDetectionFlags (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackDetectionFlags (revision 36826) @@ -0,0 +1,3 @@ +Name dimensionless VARCHAR(32) 32 Name of flag +value dimensionless BIGINT 8 NA Flag value +description dimensionless VARCHAR(2000) 2000 Description of the flag Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackMeta =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackMeta (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackMeta (revision 36826) @@ -0,0 +1,35 @@ +stackMetaID dimensionless BIGINT 8 NA stack identifier +batchID dimensionless BIGINT 8 NA batch identifier +surveyID dimensionless TINYINT 1 NA survey flag identifier +filterID dimensionless TINYINT 1 NA fnilutemr eidriecnatl ifidier +stackTypeID dimensionless TINYINT 1 0 stack type identifier +skyCellID dimensionless REAL 4 NA skycell region identifier +photoCalID dimensionless INT 4 NA photometry code +magSat dimensionless REAL 4 -999 saturation magnitude level +analVer dimensionless SMALLINT 2 -999 analysis version index +expTime seconds REAL 4 -999 exposure time +nP2Images dimensionless SMALLINT 2 -999 number of P2 images contributing to this cell +completMag magnitudes REAL 4 -999 95% completion level in mag +astroScat dimensionless REAL 4 -999 astrometric scatter for chip +photoScat dimensionless REAL 4 -999 photometric scatter for chip +nAstroRef dimensionless INT 4 -999 number of astrometric reference sources +nPhoRef dimensionless INT 4 -999 number of photometric reference sources +psfModelID dimensionless VARCHAR(MAX) 8000 PSF model identifier +psfFwhm_mean arcsec FLOAT 8 -999 mean PSF FWHM in the stack +psfFwhm_max arcsec FLOAT 8 -999 maximum PSF FWHM in the stack +photoZero magnitudes REAL 4 -999 local derived photometric zero point +photoColor magnitudes REAL 4 -999 local derived photometric color term +ctype1 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in RA +ctype2 alphanumeric VARCHAR(MAX) 8000 name of astrometric projection in DEC +crval1 degrees FLOAT 8 -999 RA corresponding to reference pixel +crval2 degrees FLOAT 8 -999 DEC corresponding to reference pixel +crpix1 dimensionless FLOAT 8 -999 reference pixel value for RA +crpix2 dimensionless FLOAT 8 -999 reference pixel value for DEC +cdelt1 deg/pixel FLOAT 8 -999 scale factor for RA +cdelt2 deg/pixel FLOAT 8 -999 scale factor for DEC +pc001001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc001002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002001 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +pc002002 dimensionless FLOAT 8 -999 elements of rotation/Dcale matrix +calibModNum dimensionless SMALLINT 2 NA calibration modification number +dataRelease dimensionless TINYINT 1 NA Data release Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackToImage =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackToImage (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackToImage (revision 36826) @@ -0,0 +1,2 @@ +stackMetaID BIGINT 8 NA stack identifier +imageID BIGINT 8 NA hashed exposure-ccdID identifier Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackType =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackType (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/StackType (revision 36826) @@ -0,0 +1,3 @@ +stackTypeID TINYINT 1 -999 stack type identifier +name VARCHAR(20) 20 stack type name +description VARCHAR(MAX) 8000 stack type description Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Survey =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Survey (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/Survey (revision 36826) @@ -0,0 +1,4 @@ +SurveyID TINYINT 1 NA index of survey type +name VARCHAR(6) 6 survey name +description VARCHAR(MAX) 8000 +survey description Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/TelescopeConfig =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/TelescopeConfig (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/TelescopeConfig (revision 36826) @@ -0,0 +1,16 @@ +TelescopeConfig +Description: Telescope configuration + +Name Unit DataType Size Default ValueDescription + +TelescopeID dimensionless SMALLINT 2 -999 Telescope identifier (i.e. PS1, ISP) +TeleConfigID dimensionless SMALLINT 2 -999 Telescope Configuration ID +MountModel string 8 NA Mount Model Name +MirrorModel string 8 NA Mirror Model Name +Observera string 8 NA Observers Name +endDate MeanJulianDays FLOAT 8 -999 ending date of this configuration in MJD +configDetails string VARCHAR(MAX) 8000 0x00 NA text/table giving the details of the telescope configuration + + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsGalaxyModel =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsGalaxyModel (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsGalaxyModel (revision 36826) @@ -0,0 +1,3 @@ +galaxyModelID dimensionless INT 4 NA modelID +description dimensionless VARCHAR(128) 128 model description, spectral type etc. +priors dimensionless VARCHAR(128) 128 priors applied to this model, may also split into 7 columns Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsPhotoZRecipe =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsPhotoZRecipe (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsPhotoZRecipe (revision 36826) @@ -0,0 +1,2 @@ +photozRecipeID dimensionless TINYINT 1 -999 recipe index +description dimensionless VARCHAR(MAX) 8000 string\ describing\ the\ recipe Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsSGSepRecipe =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsSGSepRecipe (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsSGSepRecipe (revision 36826) @@ -0,0 +1,2 @@ +sgSepRecipeID dimensionless INT 4 NA Star/Galaxy Seperator Recuipe ID +description dimensionless VARCHAR(128) 128 Description Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsStarModel =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsStarModel (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsStarModel (revision 36826) @@ -0,0 +1,2 @@ +starModelID dimensionless INT 4 -999 modelID +description dimensionless VARCHAR(128) 128 model description, spectral type etc Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsStellarParamsRecipe =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsStellarParamsRecipe (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_TABLES/MetaData_Tables/pcsStellarParamsRecipe (revision 36826) @@ -0,0 +1,3 @@ +stellarParamsRecipeID dimensionless INT 4 NA Na +description dimensionless VARCHAR(128) 128 Description:wq + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_VIEWS/README =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_VIEWS/README (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_VIEWS/README (revision 36826) @@ -0,0 +1,5 @@ +See Notes_on_Schema 03.txt +for what tables are combined into what views. +Once there is concensus on tables, then will combine tables to make views. Meanwhile can concatente yourself from Notes. + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_VIEWS/Views =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_VIEWS/Views (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/SchemaNew16.0/PSPS_VIEWS/Views (revision 36826) @@ -0,0 +1,17 @@ + +PSPS VIEWS - examples + +ObjectMean = ObjectThin join MeanObject +ObjectStack = ObjectThin join StackObject +ObjectApFlx = ObjectThin join StackApFlx +ObjectModel = ObjectThin join StackModelFit +ObjectStackAll = ObjectThin join StackObject and StackAp and StackModelFit +ObjectForcedMean = ObjectThin join ForcedMeanObject +ObjectPrimary = ObjectThin join StackObject join StackApFlx where primaryDection is true +ObjectBest = ObjectThin join StackObject join StackApFlx where bestDetection is true + + + + + + Index: /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/pspsSchema2xml.pl =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/pspsSchema2xml.pl (revision 36826) +++ /branches/eam_branches/ipp-20140423/ippToPsps/pspsschema/pspsSchema2xml.pl (revision 36826) @@ -0,0 +1,375 @@ +#!/usr/bin/perl -w + +####################################################################################### +# +# Script that searches a dir containing PSPS schema files, finds those that comtain the +# tables on interest then parses them into the VOTable format. Also generates trac-wiki +# formatted tables. +# +####################################################################################### + +use warnings; +use strict; +use IPC::Cmd 0.36 qw( can_run run ); +use XML::Writer; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $schemaPath = undef; +my $type = undef; + +# get user args +GetOptions( + 'schema|s=s' => \$schemaPath, + 'type|t=s' => \$type, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +# tell off user for not providing proper args +pod2usage( + -msg => "\n Required options:\n\n". + "-s \n". + "-t \n". + -exitval => 3 + ) unless +defined $schemaPath and +defined $type; + +if ( + $type ne "init" && + $type ne "detection" && + $type ne "diff" && + $type ne "stack" && + $type ne "object" ) { + + print "Don't understand type '$type'\n"; + die; +} + +open(WIKI, ">tables.wiki") or die("Error"); + +my $enumsHeader = ucfirst($type)."BatchEnums"; + +# VOTable file +my $votOutput = new IO::File(">tables.vot"); +my $votWriter = new XML::Writer(OUTPUT => $votOutput, DATA_MODE => 1, DATA_INDENT=>2); +$votWriter->xmlDecl('UTF-8'); +$votWriter->startTag('VOTABLE', "version" => "1.1"); +$votWriter->startTag('RESOURCE'); + +if ($type eq "init") {createInit();} +elsif ($type eq "detection") {createDetections();} +elsif ($type eq "diff") {createDiffs();} +elsif ($type eq "stack") {createStacks();} +elsif ($type eq "object") {createObjects();} + +# finish up XML +$votWriter->endTag(); # end RESOURCE tag +$votWriter->endTag(); # end of TABLE tag +$votWriter->end(); + +close WIKI; + + +####################################################################################### +# +# Finds the schema file containing the table name passed in +# +####################################################################################### +sub findSchemaFile { + my ($tableName) = @_; + + opendir(DIR, $schemaPath) or print "Cannot open '$schemaPath'\n" and return ""; + my @files= readdir(DIR); + closedir(DIR); + + foreach my $f (@files) { + + if ($f =~ m/.*\.sql$/) { + #print "....$f\n"; + + open FILE, "<$schemaPath/$f" or next; + + while () { + if ($_ =~ m/.*CREATE TABLE\s+dbo\.$tableName\s*\(/i) { + + close (FILE); + return "$schemaPath/$f"; + } + } + + close (FILE); + + } + } + + print "Could not find table '$tableName'\n"; + return ""; +} + +####################################################################################### +# +# Creates detection batch tables +# +####################################################################################### +sub createInit { + + parseTable("Filter"); + parseTable("FitModel"); + # parseTable("PhotozRecipe"); + parseTable("Survey"); + parseTable("CameraConfig"); + parseTable("PhotoCal"); + parseTable("SkyCell"); + parseTable("ProjectionCell"); + parseTable("Region"); + parseTable("StackType"); + parseTable("ImageFlags"); + parseTable("DetectionFlags"); +} + +####################################################################################### +# +# Creates initialisation batch tables +# +####################################################################################### +sub createDetections { + + parseTable("FrameMeta"); + parseTable("ImageMeta"); + parseTable("Detection"); + parseTable("SkinnyObject"); + # parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Creates difference batch tables +# +####################################################################################### +sub createDiffs { + + parseTable("StackMeta"); + parseTable("StackToImage"); + # parseTable("StackLowSigDelta"); + # parseTable("StackHighSigDelta"); + # parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Creates stack batch tables +# +####################################################################################### +sub createStacks { + + parseTable("StackMeta"); + parseTable("StackDetection"); + parseTable("StackApFlx"); + parseTable("StackModelFit"); + parseTable("StackToImage"); + parseTable("SkinnyObject"); + # parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Creates object batch tables +# +####################################################################################### +sub createObjects { + + parseTable("Object"); + #parseTable("ObjectCalColor"); +} + +####################################################################################### +# +# Parses a particular table from the SQL file and converts it to an XML description +# +####################################################################################### +sub parseTable { + my ($tableName, $newName) = @_; + + my $path = findSchemaFile($tableName); + + if ($path eq "") {return;} + + # sort out table name, either same as in schema or as defined + my $tableNameOut; + if ($newName) { $tableNameOut = $newName;} + else {$tableNameOut = $tableName} + + + print WIKI "|| '''!".$tableNameOut."''' ||||||||||||||\n"; + print WIKI "|| '''ODM attribute''' || '''units''' || '''Data type''' || '''Default''' || '''Description''' || '''IPP source''' || '''IPP variable/notes''' ||\n"; + $votWriter->startTag('TABLE', + "name" => "$tableNameOut"); + $votWriter->dataElement('DESCRIPTION', "VOTable description of PSPS table $tableNameOut"); + $votWriter->startTag('PARAM', + "arraysize" => "1", + "datatype" => "char", + "ucd" => "meta.bib.author", + "name" => "Author", + "value" => "PSPS"); + $votWriter->endTag(); + + open (SCHEMA, $path); + + my $reading = 0; + my $found = 0; + my $colNum = 0; + my $table; + + while () { + chomp; + + if ($_ =~ m/.*CREATE TABLE\s+dbo\.([a-zA-Z0-9.]+)\s*\(/i) { + + $table = $1; + + if ($table eq $tableName) { + + $reading = 1; + $found = 1; + $colNum = 0; + next; + } + } + + if($reading && $_ =~ m/^\s*\)\s*/) {$reading = 0;} + + if(!$reading) {next;} + + my $line = $_; + $line =~ s/[\s]*$//; + + if (!$line) {next;} + if (length($line) < 5) {next;} + if ($line =~ m/^\s*--/) {next;} + if ($line =~ m/\/\*/) {next;} + if ($line =~ m/\*\//) {next;} + + $colNum = processLine($line, $tableName, $colNum); + + } + + if (!$found) {print "Could not find table '$tableName'\n";} + + $votWriter->startTag('DATA'); + $votWriter->startTag('TABLEDATA'); + $votWriter->endTag(); # end TABLEDDATA tag + $votWriter->endTag(); # end DATA tag + $votWriter->endTag(); # end TABLE tag + + close SCHEMA; +} + +####################################################################################### +# +# Processes a line from the PSPS schema table description and converts to XML +# +####################################################################################### +sub processLine { + my ($line, $tableName, $colNum) = @_; + + my $name; + my $typeStr; + my $type; + my $comment; + my $fullComment; + my $default; + my $unit = "unknown"; + my $mstype = "unknown"; + + # parse line + if ($line =~ m/\s*([a-zA-Z0-9()-_\[\]]+)\s+(.*)\s*--\/(.*)/) { + + $name = $1; + $typeStr = $2; + $comment = $3; + + # neaten up the comment + $comment =~ s/<[\/]{0,1}column>//g; + $comment =~ s/^[\s]*//; + $comment =~ s/[\s]*$//; + if ($comment =~ m/(.*)/ ) {$comment = $2; $unit = $1;} + $fullComment = "$comment (unit = $unit)"; + } + # no comment case + elsif ($line =~ m/\s*([a-zA-Z0-9()-_\[\]]+)\s+(.*)/) { + + $name = $1; + $typeStr = $2; + $comment = "No comment"; + $fullComment = $comment; + } + else { + + print "In '$tableName', can't process: '$line'\n"; + return $colNum + } + + my $ISBINARY = 0; + + # remove [] from name + $name =~ s/[\[\]]//g; + + # sort out type + if ($typeStr =~ m/([a-zA-Z0-9()-]*)[ \t]+.*/) { + + $type = $1; + $mstype = $1; + $type =~ s/BIGINT/TLONGLONG/; + $type =~ s/SMALLINT/TSHORT/; + $type =~ s/TINYINT/TBYTE/; + $type =~ s/INT/TLONG/; + $type =~ s/FLOAT/TDOUBLE/; + $type =~ s/REAL/TFLOAT/; + $type =~ s/DATE/TSTRING/; + if ($type =~ m/^VARCHAR.*/) {$type = "TSTRING";} + if ($type =~ m/^VARBINARY.*/) {$type = "TSTRING"; $ISBINARY = 1;} + } + + # get default value + if ($typeStr =~ m/.*DEFAULT[ \t]*(.*)/i) { + + $default = $1; + $default =~ s/,//; + $default =~ s/[\(\)]//g; + $default =~ s/[\s]*$//; + $default =~ s/'//g; + if ($type ne "TSTRING" && !$default) {$default = "0";} + if ($ISBINARY && $default eq "0x") {$default = "";} + } + elsif ($type eq "TSTRING") {$default = " ";} + else {$default = 0;} + + if ($type eq "TSTRING" && $default eq "") {$default = " ";} + + $colNum++; + + print WIKI "|| ".$name."||".$unit."||".$mstype."||".$default."||".$comment."|| || ||\n"; + + # get VOTable type + my $votType = "undef"; + my $arraySize = 1; + if ($type eq "TBYTE") {$votType = "unsignedByte";} + elsif ($type eq "TSHORT") {$votType = "short";} + elsif ($type eq "TLONG") {$votType = "int";} + elsif ($type eq "TLONGLONG") {$votType = "long";} + elsif ($type eq "TFLOAT") {$votType = "float";} + elsif ($type eq "TDOUBLE") {$votType = "double";} + elsif ($type eq "TSTRING") {$votType = "char"; $arraySize = 100;} + + $votWriter->startTag('FIELD', + "name" => $name, + "arraysize" => $arraySize, + "datatype" => $votType, + "unit" => $unit); + $votWriter->dataElement('DESCRIPTION', $comment); + $votWriter->endTag(); + + return $colNum; +} Index: /branches/eam_branches/ipp-20140423/ippToPsps/xmlschema/BatchManifest.xsd =================================================================== --- /branches/eam_branches/ipp-20140423/ippToPsps/xmlschema/BatchManifest.xsd (revision 36825) +++ /branches/eam_branches/ipp-20140423/ippToPsps/xmlschema/BatchManifest.xsd (revision 36826) @@ -42,4 +42,8 @@ + + + + Index: /branches/eam_branches/ipp-20140423/ppStack/src/ppStackCombineFinal.c =================================================================== --- /branches/eam_branches/ipp-20140423/ppStack/src/ppStackCombineFinal.c (revision 36825) +++ /branches/eam_branches/ipp-20140423/ppStack/src/ppStackCombineFinal.c (revision 36826) @@ -3,4 +3,43 @@ // This is the doomsday switch. // #define TESTING // Enable test output + +//MEH -- adhoc addition to blank mask border of final stack since rejection different/none on order of KERNEL.SIZE with overlap in CombineInitial +static void stackBorderMask(psImage *image, // Image to mark as blank + psImage *mask, // Mask to mark as blank (or NULL) + psImage *variance, // Weight map to mark as blank (or NULL) + int numCols, int numRows, // Size of image + int size, // Size to mark blank + psImageMaskType blank // Blank mask value + ) +{ + for (int y = size; y < numRows - size; y++) { + for (int x = 0; x < size; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + for (int x = numCols - size; x < numCols; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + } + for (int y = 0; y < size; y++) { + for (int x = 0; x < numCols; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + } + for (int y = numRows - size; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + } + return; +} + bool ppStackCombineFinal(ppStackThreadData *stack, psArray *covariances, ppStackOptions *options, @@ -20,4 +59,7 @@ psAssert(recipe, "We've thrown an error on this before."); float poorFrac = psMetadataLookupF32(NULL, recipe, "POOR.FRACTION"); // Fraction for "poor" + + int sizeBlank = psMetadataLookupS32(NULL, recipe, "MASK.BLANKBORDER"); // Pixels to mask BLANK from edge + psImageMaskType maskBlank = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels // Grow the list of rejected pixels, if desired @@ -109,4 +151,10 @@ #endif + //MEH blank mask/manual reject border on final stack -- + if (sizeBlank > 0) { + stackBorderMask(outRO->image,outRO->mask,outRO->variance,numCols,numRows,sizeBlank,maskBlank); + stackBorderMask(expRO->image,expRO->mask,expRO->variance,numCols,numRows,sizeBlank,0); + } + if (options->stats) { // only add the timer if it has not been set (convolved stack) Index: /branches/eam_branches/ipp-20140423/ppStack/src/ppStackReadout.c =================================================================== --- /branches/eam_branches/ipp-20140423/ppStack/src/ppStackReadout.c (revision 36825) +++ /branches/eam_branches/ipp-20140423/ppStack/src/ppStackReadout.c (revision 36826) @@ -116,4 +116,6 @@ bool useVariance = psMetadataLookupBool(&mdok, recipe, "VARIANCE"); // Use variance for rejection? bool safe = psMetadataLookupBool(&mdok, recipe, "SAFE"); // Be safe when combining small numbers of pixels + + int nminpix = psMetadataLookupS32(&mdok, recipe, "NMINPIX"); // Minimum input per pixel to combine with psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe @@ -161,5 +163,5 @@ if (!pmStackCombine(outRO, NULL, stack, maskBad, maskSuspect, maskBlank, kernelSize, iter, - combineRej, combineSys, combineDiscard, useVariance, safe, false)) { + combineRej, combineSys, combineDiscard, useVariance, safe, nminpix, false)) { psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts with rejection."); psFree(stack); @@ -218,4 +220,6 @@ bool useVariance = psMetadataLookupBool(&mdok, recipe, "VARIANCE"); // Use variance for rejection? bool safe = psMetadataLookupBool(&mdok, recipe, "SAFE"); // Be safe when combining small numbers of pixels + + int nminpix = psMetadataLookupS32(&mdok, recipe, "NMINPIX"); // Minimum input per pixel to combine with psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.VAL"); // Name of bits to mask for bad @@ -275,5 +279,5 @@ if (!pmStackCombine(outRO, expRO, stack, maskBad, maskSuspect, maskBlank, 0, iter, combineRej, - combineSys, combineDiscard, useVariance, safe, rejected)) { + combineSys, combineDiscard, useVariance, safe, nminpix, rejected)) { psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts."); psFree(stack); Index: /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relastro.part1 =================================================================== --- /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relastro.part1 (revision 36825) +++ /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relastro.part1 (revision 36826) @@ -5,6 +5,5 @@ endif -set REALRUN = 1 - +set REALRUN = 0 set catdir_3pi = $argv[1] Index: /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relphot.3pinew =================================================================== --- /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relphot.3pinew (revision 36825) +++ /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relphot.3pinew (revision 36826) @@ -18,5 +18,5 @@ set filters = "g,r,i,z,y" - set options = "-imfreeze -mosaic" + set options = "-images -imfreeze -mosaic" set options = "$options -D MAG_LIM 32.0" set options = "$options -D SIGMA_LIM 0.1" @@ -35,5 +35,5 @@ ## note the boundary tree location is hard-wired - set options = "$options -boundary-tree ~ippdvo/tess.3pi.fits" + set options = "$options -boundary-tree ~ippdvo/tesstables/tess.3pi.fits" set options = "$options $filters" @@ -75,5 +75,5 @@ set filters = "g,r,i,z,y" - set options = "-imfreeze -mosaic" + set options = "-images -imfreeze -mosaic" set options = "$options -D MAG_LIM 32.0" set options = "$options -D SIGMA_LIM 0.1" @@ -89,5 +89,5 @@ ## note the boundary tree location is hard-wired - set options = "$options -boundary-tree ~ippdvo/RINGS.V3.tree.v2.fits" + set options = "$options -boundary-tree ~ippdvo/tesstables/tess.3pi.fits" set options = "$options -nloop 0" Index: /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relphot.mdnew =================================================================== --- /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relphot.mdnew (revision 36825) +++ /branches/eam_branches/ipp-20140423/tools/dvodb_calib/gene.run.relphot.mdnew (revision 36826) @@ -18,5 +18,5 @@ set filters = "g,r,i,z,y" - set options = "-imfreeze -mosaic" + set options = "-images -imfreeze -mosaic" set options = "$options -D MAG_LIM 32.0" set options = "$options -D SIGMA_LIM 0.1" @@ -35,5 +35,5 @@ ## note the boundary tree location is hard-wired - set options = "$options -boundary-tree ~ippdvo/RINGS.V3.tree.v2.fits" + set options = "$options -boundary-tree ~ippdvo/tesstables/tess.md.fits" set options = "$options $filters" @@ -75,5 +75,5 @@ set filters = "g,r,i,z,y" - set options = "-imfreeze -mosaic" + set options = "-images -imfreeze -mosaic" set options = "$options -D MAG_LIM 32.0" set options = "$options -D SIGMA_LIM 0.1" @@ -89,5 +89,5 @@ ## note the boundary tree location is hard-wired - set options = "$options -boundary-tree ~ippdvo/tess.md.fits" + set options = "$options -boundary-tree ~ippdvo/tesstables/tess.md.fits" set options = "$options -nloop 0" Index: /branches/eam_branches/ipp-20140423/tools/dvodb_calib/run.relastro.part1 =================================================================== --- /branches/eam_branches/ipp-20140423/tools/dvodb_calib/run.relastro.part1 (revision 36825) +++ /branches/eam_branches/ipp-20140423/tools/dvodb_calib/run.relastro.part1 (revision 36826) @@ -22,5 +22,5 @@ # try 1000 meas per deg^2, but if memory if available, go to more if (1) then - set options = "-update-chips" + set options = "-images -update-chips" set options = "$options -D RELASTRO_SRC_MEAS_TOOFEW 1" set options = "$options -D RELASTRO_SIGMA_LIM 0.1" Index: /branches/eam_branches/ipp-20140423/tools/get_p_data.pl =================================================================== --- /branches/eam_branches/ipp-20140423/tools/get_p_data.pl (revision 36826) +++ /branches/eam_branches/ipp-20140423/tools/get_p_data.pl (revision 36826) @@ -0,0 +1,124 @@ +#! /usr/local/bin/perl -w + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use DateTime; + +my $pi = 4 * atan2(1,1); + +sub asin { atan2($_[0], sqrt(1 - $_[0] * $_[0])) } + + +my $ipprc = PS::IPP::Config->new(); # IPP Configuration +my $siteConfig = $ipprc->{_siteConfig}; + +my $dbname; +unless(defined($dbname)) { + $dbname = 'gpc1'; +} +my $db = init_gpc_db(); + +my $neblocate = '/home/panstarrs/ipp/psconfig//ipp-20120802.lin64/bin/neb-locate'; +my $fields = '/home/panstarrs/ipp/psconfig//ipp-20120802.lin64/bin/fields'; + +my $db_query; +my $arg; +my $max_arg; +my $dt = DateTime->now(time_zone => 'Pacific/Honolulu'); +$max_arg = $dt->ymd; +$dt->subtract( days => 1); +$arg = $dt->ymd; +open(OUT, ">/home/panstarrs/ippm/pointing_data/pointing.${arg}"); +$db_query = "select exp_name,dateobs,alt,az,airmass,filter,ra * 180 / PI(),decl * 180 / PI(),CONCAT(path_base,'.smf'),camProcessedExp.quality,camProcessedExp.fault from camRun JOIN camProcessedExp USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) where dateobs > '$arg' AND dateobs <= '$max_arg' AND camRun.label LIKE '%nightly%' ORDER BY dateobs"; + +print OUT "## $db_query\n"; +my $data_ref = $db->selectall_arrayref( $db_query ); + +print OUT "#exp_name dateobs_D dateobs_T filter RA DEC ROT ALT AZ SHUTOPEN SHUTCLOS SHUTCUTC SHUTOUTC AST_R0 AST_D0 AST_T0 AST_S0 AST_RS AST_DS RA_TRUE DEC_TRUE AZALTTIM\n"; +foreach my $row_ref (@{ $data_ref }) { + my ($exp_name,$dateobs,$alt,$az,$airmass,$filter,$ra,$dec,$smf,$quality,$fault) = @{ $row_ref }; + my $disk_file = `$neblocate --server http://ippc03/nebulous --path $smf`; + chomp($disk_file); + + open(H,"echo '$disk_file' | $fields AST_R0 AST_D0 AST_T0 AST_S0 AST_RS AST_DS POSANGLE ROT |"); + my ($file,$r0,$d0,$t0,$s0,$rs,$ds,$pos,$rot,$sop,$scl,$scut,$sout); + my ($sop_D,$sop_T,$scl_D,$scl_T,$scut_D,$scut_T,$sout_D,$sout_T,$azalttim); + while () { + chomp; + ($file,$r0,$d0,$t0,$s0,$rs,$ds,$pos,$rot) = split /\s+/; + } + close(H); + + open(H,"echo '$disk_file' | $fields -x 1 SHUTOPEN SHUTCLOS SHUTCUTC SHUTOUTC AZALTTIM|"); + while () { + chomp; + ($file,$sop_D,$sop_T,$scl_D,$scl_T,$scut_D,$scut_T,$sout,$azalttim) = split /\s+/; + $sop = "${sop_D}T${sop_T}"; + $scl = "${scl_D}T${scl_T}"; + $scut = "${scut_D}U${scut_T}"; +# $sout = "$sout_D $sout_T"; + } + close(H); + + my ($xsp,$ysp,$x,$y,$R,$sinF,$cosF,$rho,$sinQ,$cosQ,$sinD,$cosD,$dec_T,$ra_T,$sinA,$cosA); + # Calculate pixel scale in radians + my $ra_bore = $ra; + my $dec_bore = $dec; + $xsp = 7.08161543662353E-05 * $pi / 180.0; + $ysp = 7.08794419479912E-05 * $pi / 180.0; +# Construct radian offsets values + $x = $r0 * $xsp; + $y = $d0 * $ysp; +# Calculate phi angle + $R = sqrt($x * $x + $y * $y); + if ($R == 0) { + $sinF = 0; + $cosF = 1; + } + else { + $sinF = $x / $R; + $cosF = -1.0 * $y / $R; + } +# Calculate theta angle + $rho = sqrt(1 + $R * $R); + $sinQ = 1 / $rho; + $cosQ = $R / $rho; +# Scale factor for declination + $sinD = sin($dec_bore * $pi / 180.0); + $cosD = cos($dec_bore * $pi / 180.0); +# Calculate final values + $dec_T = asin($sinQ * $sinD - $cosQ * $cosF * $cosD) * (180.0 / $pi); + $sinA = 1.0 * $cosQ * $sinF; + $cosA = 1.0 * $cosQ * $cosF * $sinD + $sinQ * $cosD; + $ra_T = atan2($sinA,$cosA) * (180.0 / $pi) + $ra_bore; + + print OUT"$exp_name $dateobs $filter $ra $dec $rot $alt $az $sop $scl $scut $sout $r0 $d0 $t0 $s0 $rs $ds $ra_T $dec_T $azalttim\n"; +} + +close(OUT); + + +sub init_gpc_db { + ## change to use the siteConfig setting, while readonly probably do not want to use a readonly replicated DB incase it gets behind + my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER'); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "RO_DBUSER"); + my $dbpass = metadataLookupStr($ipprc->{_siteConfig}, "RO_DBPASSWORD"); + die "database configuration not set up" unless defined($dbserver); + die "database configuration not set up" unless defined($dbuser); + die "database configuration not set up" unless defined($dbpass); + #my $dbserver = 'ippdb01'; + #my $dbuser = 'ippuser'; + #my $dbpass = 'ippuser'; + use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; + $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + return($db); +} +