Index: branches/czw_branch/20160809/DataStoreServer/scripts/dsprodindex =================================================================== --- branches/czw_branch/20160809/DataStoreServer/scripts/dsprodindex (revision 39718) +++ branches/czw_branch/20160809/DataStoreServer/scripts/dsprodindex (revision 39719) @@ -75,6 +75,8 @@ # # XXX: the spec doesn't say what should happen in this case. - # Here we follow the conductor implementation and return the whole list. - # This may not be the right thing to do. It might be better to throw an error + # We used to follow the conductor implementation and return the whole list. + # but that is definitely not the right thing to do in our context. + # Exit with an error + exit 404; } } Index: branches/czw_branch/20160809/DataStoreServer/scripts/dsrootindex =================================================================== --- branches/czw_branch/20160809/DataStoreServer/scripts/dsrootindex (revision 39718) +++ branches/czw_branch/20160809/DataStoreServer/scripts/dsrootindex (revision 39719) @@ -14,7 +14,24 @@ my $PS_EXIT_CONFIG_ERROR = 3; +# set this variable to redirect listings of the root datastore directory +# to the password protected php page on the postage stamp server web site. +# Since the data store is restricted by IP address now this is no longer necessary +my $redirect_root_to_pstamp = 0; +if ($redirect_root_to_pstamp) { + + print ' + + + + + + '; + + exit 0; +} + my $dbh = getDBHandle(); -my $stmt = $dbh->prepare("SELECT * FROM dsProduct"); +my $stmt = $dbh->prepare("SELECT * FROM dsProduct ORDER BY type"); $stmt->execute(); @@ -36,4 +53,5 @@ $row->{type}, $row->{description}; + # XXX EAM : security by obfuscation print $line; } Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/dsgetindex =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/cgi/dsgetindex (revision 39718) +++ branches/czw_branch/20160809/DataStoreServer/web/cgi/dsgetindex (revision 39719) @@ -20,4 +20,6 @@ my $PS_EXIT_CONFIG_ERROR = 3; my $PS_EXIT_DATA_ERROR = 5; + +my $redirect_root_to_pstamp = 0; my $uri = shift; @@ -132,4 +134,5 @@ print @$stdout_buf; } + $error_code = 0 if !defined $error_code; } else { if (0 && $html_mode) { @@ -199,12 +202,24 @@ # return link - print a({-href=>"./index.txt"}, "Text Version"); + if (!$redirect_root_to_pstamp || $program ne "dsrootindex") { + print a({-href=>"./index.txt"}, "Text Version"); + } if ($#path >= 1) { - print "    \n"; - - my $up = join('/', @path[0..$#path-1]); - print a({-href=>".."}, "Up to $up"); - + # including the up links in the data store display causes + # wget -r to follow them up which makes a mess + # Turn them off if the user agent looks like wget + my $display_up_links = (lc($ENV{HTTP_USER_AGENT}) =~ /wget/) ? 0 : 1; + if ($display_up_links) { + print "    \n"; + my $up = join('/', @path[0..$#path-1]); + if ($redirect_root_to_pstamp and $program eq "dsprodindex") { + print a({-href=>"http://pstamp.ipp.ifa.hawaii.edu/dsroot.php"}, "Up to $up"); + } elsif ($program eq 'dsrootindex') { + # no up link in dsroot + } else { + print a({-href=>".."}, "Up to $up"); + } + } } print "

\n"; @@ -244,23 +259,47 @@ if ($nolink) { print $toks[0]; - } - - else { - # assumes id is always first field - my $wpath = "./$toks[0]"; - - # if this is a file request, use a download cgi - if ($wpath =~ /\.fits\s*$/) { - # This doesn't work through the proxy - # The /ds-cgi gets tried on alala - # $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/' + } else { + # First column is link to an entity down one level in data store (product, fileset, file) + # assumes id is always first field + my $wpath = "./$toks[0]"; + + # if this is a file request, use a download cgi + my $fits = 0; + if ($wpath =~ /\.fits\s*$/) { + # This doesn't work through the proxy + # The /ds-cgi gets tried on alala + # $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/' + $fits = 1; + } + + if ($fits) { + print a({-href=>$wpath, -type=>'image/x-fits'}, $toks[0]); + } else { + print a({-href=>$wpath}, $toks[0]); + } + + if ($program eq 'dsrootindex') { + # for root listing make most recent fileset a link as well unless there are no filesets + my $fileset = $toks[1]; + $fileset =~ s/^\s+//; + $fileset =~ s/\s+$//; + if ($fileset ne 'none') { + print ""; + print "<$celltag>"; + # drop the product from the list of tokens + my $product = shift @toks; + # elmiminate any whitesapace + $product =~ s/^\s+//; + $product =~ s/\s+$//; + print a({-href=>"./$product/$fileset"}, $fileset); } - - print a({-href=>$wpath, -type=>'image/x-fits'}, $toks[0]); - } + } + } + shift @toks; print ""; - foreach my $val (@toks[1..$#toks]) { + # foreach my $val (@toks[1..$#toks]) { + foreach my $val (@toks) { print "<$celltag>"; print $val; Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/findskycalcmf.pl =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/cgi/findskycalcmf.pl (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/cgi/findskycalcmf.pl (revision 39719) @@ -0,0 +1,167 @@ +#!/usr/bin/env perl + +# findsskycalcmf.pl Locate a skycal cmf file for a skycell. + +use strict; +use warnings; + +use DBI; +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $verbose; + +my $skycal_id; +my $skycell_id; +my $tess_id; +my $filter; +my $release; + +# NOTE: We do not use the ipp configuration to simplify running from a CGI script +my $dbname = "gpc1"; +my $dbserver = "scidbm"; +my $dbuser = "ippuser"; +my $dbpassword = "ippuser"; + + +GetOptions( + 'skycell_id=s' => \$skycell_id, + 'tess_id=s' => \$tess_id, + 'filter=s' => \$filter, + 'skycal_id=s' => \$skycal_id, + 'release=s' => \$release, + 'dbname=s' => \$dbname, + 'verbose|v' => \$verbose, +) or pod2usage (2); + +pod2usage( -msg => "Required options: (--tess_id and --skycell_id and --filter) or --skycal_id", + -exitval => 3) + unless $skycal_id or ($filter and $tess_id and $skycell_id); + +my ($filename, $cmf); +if (defined $skycal_id) { + ($filename, $cmf) = find_cmf_by_skycal_id($skycal_id) +} elsif (defined $tess_id and defined $skycell_id and defined $filter) { + ($filename, $cmf) = find_cmf_by_skycell($tess_id, $skycell_id, $filter, $release); +} else { + # should have been trapped above + die "not enough parameters\n"; +} + + +# FIX THESE PATHS! +# set up the environment +$ENV{NEB_SERVER}="http://nebserver.ipp.ifa.hawaii.edu:80/nebulous"; +$ENV{PERL5LIB} .= ":/home/panstarrs/bills/psconfig/debug.lin64/lib:/home/panstarrs/bills/psconfig/debug.lin64/lib/perl5"; + +my $neb_locate = "/home/panstarrs/bills/psconfig/debug.lin64/bin/neb-locate -p"; +if ($cmf) { + my $resolved = `$neb_locate $cmf`; + if (!$resolved) { + print STDERR "failed to resolve $cmf\n"; + exit 2 if (!$resolved) + } + # HERE IS THE OUTPUT FOR A SUCCESSFUL LOOKUP + print "$filename $resolved"; +} else { + # the find function prints an appropriate error if it fails + exit 2; +} + +exit 0; + +sub open_db { + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database.\n"; + + return $dbh; +} + +sub find_cmf_by_skycal_id { + my $skycal_id = shift; + + my $query = "SELECT * from skycalRun JOIN skycalResult USING(skycal_id) WHERE skycal_id = $skycal_id"; + + my $dbh = open_db(); + my $stmt = $dbh->prepare($query); + if (!$stmt->execute()) { + print "DBI error\n"; + return undef; + } + my $results = $stmt->fetchrow_hashref(); + if (!$results) { + print "results for skycal_id: $skycal_id not found\n"; + return undef; + } + + if ($results->{fault}) { + my $fault = $results->{fault}; + $fault .= " (GONE)" if $fault == 26; + print "camRun $skycal_id has fault $fault\n"; + return undef; + } + + my $path_base = $results->{path_base}; + + my $cmf = $path_base . ".cmf"; + + return parse_filename($cmf); +} + +sub find_cmf_by_skycell { + my ($tess_id, $skycell_id, $filter, $release) = @_; + + my $where_clause = " (tess_id = '$tess_id' AND skycell_id = '$skycell_id' and filter LIKE '$filter%')"; + + # prioritize by release priority values + my $order = "priority DESC"; + if ($release) { + $where_clause .= " AND ippRelease.release_name = '$release'"; + } + + my $query = "SELECT skycal_id, skycalResult.quality, skycalResult.fault, skycalResult.path_base, priority\n" + . " FROM ippRelease JOIN relStack using(rel_id) JOIN skycalResult using(skycal_id) " + . " JOIN stackRun USING(stack_id, tess_id, skycell_id, filter)" + . " WHERE $where_clause \n ORDER by $order limit 1;"; + + print STDERR "$query\n" if $verbose; + + my $dbh = open_db(); + my $stmt = $dbh->prepare($query); + if (!$stmt->execute()) { + print "DBI error\n"; + return undef; + } + my $results = $stmt->fetchrow_hashref(); + if (!$results) { + print "results not found\n"; + return undef; + } + my $skycal_id = $results->{skycal_id}; + if ($results->{fault}) { + my $fault = $results->{fault}; + if ($fault == 26) { + $fault .= " (GONE)"; + } + print "skycalResult $skycal_id has fault $fault\n"; + return undef; + } elsif ($results->{quality} != 0) { + print "skycalRun $skycal_id has poor quality $results->{quality}\n"; + return undef; + } + + + my $path_base = $results->{path_base}; + + my $cmf = $path_base . ".cmf"; + return parse_filename($cmf); +} + +sub parse_filename { + my $cmf = shift; + my $filename = basename($cmf); + return ($filename, $cmf); +} Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/findsmf.pl =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/cgi/findsmf.pl (revision 39718) +++ branches/czw_branch/20160809/DataStoreServer/web/cgi/findsmf.pl (revision 39719) @@ -35,8 +35,8 @@ # NOTE: We do not use the ipp configuration to simplify running from a CGI script -my $dbname = "XXX"; -my $dbserver = "XXX"; -my $dbuser = "XXX"; -my $dbpassword = "XXX"; +my $dbname = "gpc1"; +my $dbserver = "scidbm"; +my $dbuser = "ippuser"; +my $dbpassword = "ippuser"; @@ -77,4 +77,5 @@ exit 2 if (!$resolved) } + # HERE IS THE OUTPUT FOR A SUCCESSFUL LOOKUP print "$filename $resolved"; } else { Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/listsmfs.pl =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/cgi/listsmfs.pl (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/cgi/listsmfs.pl (revision 39719) @@ -0,0 +1,172 @@ +#!/usr/bin/env perl + +# listsmf.pl Print a listing of information about smf files for processing meeting +# certain selection criteria. + +use strict; +use warnings; + +use DBI; +use File::Basename; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my $verbose; + +my $cam_id; +my $exp_id; +my $exp_name; +my $filter; +my $label; +my $data_group; +my $dateobs_min; +my $dateobs_max; +my $release; +my $getMagicked; + +# NOTE: We do not use the ipp configuration to get this information in order to simplify running from a CGI script +# XXX: probably should be using some local file though. +# XXX: take a look at my module dsdbh which has the function getDBHandle(); +# +my $dbname = "gpc1"; +my $dbserver = "scidbm"; +my $dbuser = "ippuser"; +my $dbpassword = "ippuser"; + + +my $defaultToToday = 1; + +GetOptions( + 'label=s' => \$label, + 'data_group=s' => \$data_group, + 'dateobs_min=s' => \$dateobs_min, + 'dateobs_max=s' => \$dateobs_max, + 'release=s' => \$release, + 'filter=s' => \$filter, + 'dbname=s' => \$dbname, + + 'exp_name=s' => \$exp_name, + 'exp_id=s' => \$exp_id, + 'cam_id=s' => \$cam_id, + 'verbose|v' => \$verbose, +) or pod2usage (2); + + +# if data_group is supplied we don't need to default the date +if ($data_group) { + $defaultToToday = 0; +} + +# XXX: since this is going to be used in a cgi script we should carefully vet the supplied +# date format. +if (!$dateobs_min) { + # default to today + if ($dateobs_max) { + print "Must supply begin date if max date is supplied.\n"; + exit 2; + } + if ($defaultToToday) { + my ($day, $month, $year) = (gmtime)[3,4,5]; + + $dateobs_min = sprintf "%4d-%02d-%02dT00:00:00Z", 1900+$year, 1+$month, $day; + print "$dateobs_min\n" if $verbose; + } +} else { + # add times if not supplied + if (!($dateobs_min =~ 'T')) { + $dateobs_min .= 'T00:00:00Z'; + print "$dateobs_min\n" if $verbose; + } + if (!($dateobs_max =~ 'T')) { + $dateobs_max .= 'T23:59:59Z'; + print "$dateobs_max\n" if $verbose; + } +} + + +my $dbh = open_db(); + +my $query = 'SELECT cam_id, camRun.label, camRun.state, quality, release_name, camProcessedExp.path_base,' + . ' exp_name, exp_id, filter, dateobs, comment' + . ' FROM camRun JOIN camProcessedExp USING(cam_id) JOIN chipRun using(chip_id) JOIN rawExp USING(exp_id)' + . ' LEFT JOIN relExp USING(cam_id, exp_id) LEFT JOIN ippRelease USING(rel_id)' + . " WHERE "; + +my $and = '';; +if ($data_group) { + $query .= " camRun.data_group = '$data_group'"; + $and = ' AND'; +} + +if ($dateobs_min) { + $query .= "$and dateobs > '$dateobs_min'"; +} + +if ($dateobs_max) { + $query .= " AND dateobs <= '$dateobs_max'"; +} + +# Should I use "LIKE" qualifiers here? +if ($label) { + $query .= " AND camRun.label = '$label'"; +} + +if ($release) { + if ((uc($release) eq 'NONE') or (uc($release) eq 'NULL')) { + $query .= " AND release_name IS NULL"; + } else { + $query .= " AND release_name = '$release'"; + } +} + +if ($filter) { + # if single character filter is supplied append wild card + # character to the filter string + if (length($filter) == 1) { + $filter .= '%'; + } + $query .= " AND filter LIKE '$filter'"; +} + +print "$query\n" if $verbose; + +my $statement = $dbh->prepare($query); + +$statement->execute(); + +my $numRows = 0; + +# Print Header line as a comment. +# XXX: rather than fine tuning the spacing on this I should have used the +# same format line that I use for the data below. + +print +'#cam_id smf_name quality state exp_name label release filter obs_date obs_time comment' . "\n"; +#1780291 o7616g0016o.1131500.cm.1780291.smf 0 full o7616g0016o QUB.nightlyscience none z.00000 2016-08-16 05:57:32 'Transient PS16cgx z band cell 45 65' + +while ( my $row = $statement->fetchrow_hashref() ) { + $numRows++; + + my $smf_name = basename($row->{path_base}) . '.smf'; + my $release_name = $row->{release_name}; + $release_name = 'none' unless $release_name; + + printf "%ld %s %5d %-7s %s %-20s %-14s %-11s %s '%s'\n", + $row->{cam_id}, $smf_name, $row->{quality}, $row->{state}, + $row->{exp_name}, $row->{label}, $release_name, $row->{filter}, + $row->{dateobs}, $row->{comment}; +} + +print "$numRows found\n" if $verbose; + +exit 0; + +sub open_db { + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database $dbname.\n"; + + return $dbh; +} + Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/getsmf.html =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/getsmf.html (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/getsmf.html (revision 39719) @@ -0,0 +1,117 @@ + + +IPP SMF Access Tools + + + +

+Access to IPP smf files +

+ +Go back up.
+ +

+getsmf

+ +IPP smf files contain the results of camera stage processing. +They are FITS format files containg tables with the photometric and +astronometric measurements for the +sources detected in a single exposure. +
+
+getsmf.php is a tool that may be used to retrieve an IPP smf file +given certain parameters. +

The url of the tool is +
    http://misc.ipp.ifa.hawaii.edu/getsmf.php
+ +When this page is accessed without parameters it simply displays an +error message. +With valid parameters the tool returns the contents of the selected +smf file. +

+Note: Obviously displaying the contents of a FITS file in a web browser is +not very useful. +getsmf.php is intended to be accessed directly using HTTP. +It is easy to write scripts that use the command line tools curl or wget +to retrieve smf files using this tool. +

+For example, the following two equivalent unix commands copy the contents of +the smf containing the results from the cameraRun with cam_id=1780283) +to a local files: +
+    curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -o 1780283.smf
+
+    wget 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -O 1780283-wget.smf
+
+ +The previous method allows the user to control the name of the local copy. +If one would like to use the original file name supplied by the server +the commands may be changed to: +
+    curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -O --remote-header-name
+
+    wget 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' --content-disposition
+
+
+In this example the name of the downloaded file is "o7616g0008o.1131492.cm.1780283.smf". +

+Note: The quote characters in these commands are needed to prevent +the '?' characters from being interpreted by user's shell. + +

parameters

+There are a few parameters that may be used to select specific smf files. +Parameters are supplied by appending them to the base url following +a '?' character. +

+If multiple parameters are needed +they should be separated by a '&' character. +

+

cam_id=value

+The examples above demonstrated selection by cam_id. When a +cam_id is supplied any other parameters supplied are ignored. + +

exp_name=value

+Selects an smf file for an exposure with the given name.

+For example supplying the parameter: 
exp_name=o7616g0008o
+gets the same result as the examples listed above. (As of the time +of this writing). +
+Note that exposures may be processed multiple times. If no other selection +parameters are found, the smf for the most recent processing is selected. +

release=value

+When used with exp_name, the release parameter specifies the +ipp release (processing version) for the smf file.
+For example: 
exp_name=o5275g0401o&release=3PI.PV3
retrieves the +smf processed for exposure o5275g0401o in 3PI.PV3. +If the release were changed to 3PI.nightly +the smf from the original nightly procecessing would be retrieved. +(At the time of this writing, if the release parameter were omitted +the 3PI.PV3 version would also be retrieved since it is currently +the latest processing.) +

data_group=value

+This is a rarely used parameter. +With exp_name, chooses smfs from camRuns processed as part of +the supplied data group. + +

Errors and Exceptions

+If an smf matching the supplied parameters is not found in the IPP, a message +is included in the response. The http request succeeds. +
+After a user retrieves a file in this manner it should be checked +to insure that it +is actually a FITS format file. + +For example the following getsmf request asks for a non-existent smf: +
+    curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=99999999999' -o test.smf
+
+After this command succeeds, the file test.smf is not a FITS file. +Instead it contains text. This fact and its contents may be found +with the following +commands. +
+    $ file test.smf
+    test.smf: ASCII text
+    $ cat test.smf
+    Could not find smf for cam_id: 99999999999.
+
Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/index.html =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/index.html (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/index.html (revision 39719) @@ -0,0 +1,29 @@ + + +IPP-Misc: SMF Access + + + +

+IPP-Misc: SMF Access +

+

+This page contains links to information about accessing +IPP SMF files +which contain the results of IPP camera stage processing. +

+These files are FITS format files containing multiple binary table extension +sections. These which contain the photometric and astronometric measurements +for the sources detected in a single Pan-STARRS exposure. +

+These files are the inital inputs to the PS calibration process. +

+The following links describe how to use the IPP-MISC SMF Access tools +to discover and retrive SMF files. +

+ + Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/listsmfs.html =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/listsmfs.html (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/listsmfs.html (revision 39719) @@ -0,0 +1,141 @@ + + + +listsmf + + + + +

+listsmfs +

+ +Go back up.
+

+listsmf.php is a tool that may be used to retrieve certain information +about SMF files from the IPP database. +
The url of the tool is +http://misc.ipp.ifa.hawaii.edu/listsmfs.php + +

+The list returned is controlled by parameters supplied +as part of the http request used to access the page. + +By default listsmfs displays the list of +smfs corresponding to the PS1 science exposures collected on +the current date (UTC). The output may be fine tuned using parameters. +The use of parameters is discussed below. +

+Here is some sample output. +

+#cam_id smf_name                           quality state   exp_name    label                release        filter      obs_date   obs_time  comment
+1783121 o7622g0062o.1134677.cm.1783121.smf       0 full    o7622g0062o OSS.nightlyscience   SSS.nightly    i.00000     2016-08-22 07:43:12  'OSSR.R19S6.16.Q.i ps1_32_2599 visit 1'
+1783120 o7622g0061o.1134678.cm.1783120.smf       0 full    o7622g0061o OSS.nightlyscience   SSS.nightly    i.00000     2016-08-22 07:42:16  'OSSR.R19S6.16.Q.i ps1_32_2493 visit 1'
+1783122 o7622g0063o.1134679.cm.1783122.smf       0 full    o7622g0063o OSS.nightlyscience   SSS.nightly    i.00000     2016-08-22 07:44:08  'OSSR.R19S6.16.Q.i ps1_32_2614 visit 1'
+... and so on.
+
+ +

+The first line, which begins with the "comment" character '#', is a +header line. This line lists the names of the columns shown +in the subsequent lines. +

+If no exposures are found that match the supplied parameters (and their +defaults) only the header line is displayed. +

The first 4 columns are the most important. They are very useful +for constructing requests to retrieve individual SMFs using +getsmf.php. + +

List of Columns

+ +

cam_id

+The IPP camera run identifier. +

smf_name

+The base name of the SMF file for the exposure (if any). +

quality

+The quality value for the camera processing. A non-zero value indicates +a problem with the processing. For example: unsuccessful astrometric +solution. In the event of a run with non zero quality +no SMF file will be available. +

state

+The current state of the camera run. +Runs that have completed processing will have +the state "full". SMF files will not be available for runs with any +state other than full. Runs with state "new" may finish processing +at a later time. +

exp_name

+The exposure name of the source exposure. (Also known as "Frame Name" +in PSPS and other contexts.) +

label

+The processing label for the camRun. +

release

+The Release that contains the exposures. Examples might be +3PI.nightly or SSS.nightly. +If the exposure has not been assigned to a release this column will have +the value "none". +

filter

+The filter used by the camera for the exposure. +

obs_date

+Date of the observation (UTC). +

obs_time

+Time of the observation (UTC). +

comment

+The comment string for the exposure. Note that single quote characters +have been added around the comment string as delimiters to +facilitate parsing. + + +

Parameters

+ +As mentioned above, listsmfs gives a list of the +exposures for the current (UTC) day by default. +Parameters may be used to +control the selection. Parameters are provided by appending +an HTML query string to the request URL. +

+For example the sample output shown above was generated by a request +with the parameter string '?release=SSS.nightly' appended to the URL +(on the date that this page was written of course). +

+If multiple parameters are to be supplied they should be separated +by a '&' character. For example '?release=SSS.nightly&filter=r' +would list exposures with the named release in the r band filter.. +

+Here is the list of the parameters. + + +

date_min and date_max

+The date/time limits for observations to be included by the listing. +The format for the datetime is + +
    YYYY-MM-DDTHH:MM:SS
+where the '-', 'T', and ':' characters are literal and the other characters are +numeric values for year, month, day, hours, minutes, and seconds. +

+For example 'date_min=2016-08-22T08:00:00&date_max=2016-08-22T09:00:00' +would select exposures observed between 0800 UTC and before 0900 UTC +on Augst 22. +

+Notes on date limits.

+The time portion may be omitted. For example the parameter pair +'date_min=2016-08-21&date_max=2016-08-22' will cause all exposures observed +on 2016-08-21 or 2016-08-22 to be listed. +

+If date_min is not supplied, any date_max value supplied is silently ignored. +Instead page uses the defaults for the date parameter: exposures for the current date are listed (if any). + +

filter

+Only exposures with the corresponding filter are listed. If a single +character value is supplied for filter, exposures from all filters that +begin with that letter are listed. + +

label

+The list is restricted to camRuns with the supplied label. + +

release

+The list is restrictred to exposures with supplied release name. + +

data_group

+The list is restrictred to camRuns with the supplied data group. +Note: This parameter is expected to be rarely used. + Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/scripting.html =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/scripting.html (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/scripting.html (revision 39719) @@ -0,0 +1,151 @@ + + +Scripting IPP SMF Access + + + +

+Scripting IPP SMF Access +

+ +Go back up.
+ +

+The process of retrieving SMF files is straightforward. +

+ +

+This page is primarily concerned with the first two items on this list. + +

+The two pages listsmfs.php and getsmf.php have a relatively straightforward +interface. UNIX command line tools such as curl or wget may be used to +access the interface. + +

Sample PERL script

+

+The perl script below gives an example of how to use curl to get +a list of the current day's smf files from listsmf.php and print curl +commands that may be used to download the files. + +

+        #!/usr/bin/env perl
+
+        use strict;
+        use warnings;
+
+        my $verbose = 0;
+
+        my $misc_url = 'http://misc.ipp.ifa.hawaii.edu';
+        my $list_url = "$misc_url/listsmfs.php";
+        my $get_url = "$misc_url/getsmf.php";
+
+        my $silent = ' --silent';
+        if ($verbose) {
+            $silent = '';
+        }
+
+        my $list_cmd = "curl $silent http://misc.ipp.ifa.hawaii.edu/listsmfs.php";
+
+        my $output = `$list_cmd`;
+
+        if ($? != 0) {
+            my $rc = $?;
+            print STDERR "Ooops something went wrong with the command $list_cmd\n";
+            print STDERR "command returned $rc\n";
+            exit $rc >> 8;
+        }
+
+        my @lines = split "\n", $output;
+
+        foreach my $line (@lines) {
+            # skip the header line and any others that begin with '#'
+            next if $line =~ '#';
+
+            # split the line at the single quote character that preceeds the
+            # comment string
+            my ($left, $right) = split "'", $line;
+            
+            # the right side of this split is the comment string ...
+            my $comment = $right;
+            
+            # ... and the left side of this split contains a string
+            # with the other parameters.
+            # The other parameters are separated by whitespace
+
+            # NOTE: One might have thought we could parse the header string 
+            # and find the column names but that implies more than the
+            # listsmf "interface" commits to at this time.
+
+            my ($cam_id, $smf_name, $quality, $state, $exp_name, $label, $release, $filter, $obs_date, $obs_time) = split " ", $left;
+
+            if ($quality != 0) {
+                print STDERR "camRun $cam_id had bad quality $quality\n";
+                next;
+            }
+
+            if ($state ne 'full') {
+                print STDERR "camRun is in state $state\n";
+                next;
+            }
+
+            # generate and print out a curl command that could be used to
+            # download the smf file to the current directory
+            my $this_cmd = "curl $silent '$get_url?cam_id=$cam_id' --output $smf_name";
+
+            print "$this_cmd\n";
+        }
+        # all done
+
+
+When executed the resulting output will look something like this: +
+        # 1783100 o7622g0010o.1134626.cm.1783100.smf 0 ICECUBE-160731A.01 i band dither 1 visit 1
+        curl  --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783100' --output o7622g0010o.1134626.cm.1783100.smf
+        # 1783101 o7622g0011o.1134627.cm.1783101.smf 0 ICECUBE-160731A.01 i band dither 2 visit 1
+        curl  --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783101' --output o7622g0011o.1134627.cm.1783101.smf
+        # 1783102 o7622g0012o.1134628.cm.1783102.smf 0 ICECUBE-160731A.01 i band dither 3 visit 1
+        curl  --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783102' --output o7622g0012o.1134628.cm.1783102.smf
+        # 1783103 o7622g0013o.1134629.cm.1783103.smf 0 ICECUBE-160731A.01 i band dither 4 visit 1
+        curl  --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783103' --output o7622g0013o.1134629.cm.1783103.smf
+        # 1783104 o7622g0014o.1134630.cm.1783104.smf 0 ICECUBE-160731A.01 i band dither 5 visit 1
+        curl  --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783104' --output o7622g0014o.1134630.cm.1783104.smf
+
+ +

+There are two lines for each exposure. The first (which begins with + a '#' character) lists prints selected metadata +for the smf and its source exposure. The second line gives +a curl command that would copy the file to the user's current directory. +

Note, while not suggested, one could take the output of listsmfs.php +and source it into a UNIX shell to execute the commands. +

+This script demonstrates how to +

+

+This simple script shows the most important steps necessary for successful +smf retrieval. + +

What about errors?

+

+For the sake of speed we chose to not include any information that +might be useful for detecting errors in the download process. The database +does contain anything (a checksum for example) +which could be used to gain confidence that a file was transmitted +successfully. + +

+In practice so far this has not shown to be an issue. getsmf.php has been +used to download hundreds of thousands of smf files. When users have +detected problem files, simply trying the download again seems to correct +the problem. Index: branches/czw_branch/20160809/DataStoreServer/web/php/getskycalcmf.php =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/php/getskycalcmf.php (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/php/getskycalcmf.php (revision 39719) @@ -0,0 +1,132 @@ +$command\n
"; + } + $command = escapeshellcmd($command); + $output = array(); + + exec($command, $output, $command_status); + + if ($command_status == 0) { + // we only expect one line of output + $len = count($output); + if ($len == 1) { + list($filename, $pathname) = explode(" ", $output[0]); + if ($filename && $pathname) { + if (!$debug) { + $gotFile = 1; + } else { + echo "$filename $pathname\n"; + echo "$command\n"; + } + } + } else { + echo "
unexpected output from $command: $output[0] $output[1]\n"; + } + } else { + if ($debug) { + echo "
command failed $command_status\n"; + } + } +} else { + // something went wrong with the paramters. Error message produced below. +} + +if ($gotFile) { + if ($rvar_list) { + echo "cmf : $filename\n"; + // echo "
path: $pathname\n
"; + } else { + // All systems are go. Time to write the output. + // First set up the header + header('Content-type: application/fits'); + header("Content-Disposition: attachment; filename=\"$filename\""); + $filesize = filesize($pathname); + header("Content-Length: $filesize"); + header('Expires: now'); + + // copy the contents of the file to the stream + readfile($pathname); + } +} else { + // XXX: Figure out how to stop wget from redirecting these error + // messages to the nasty filename + $error_string="Could not find cmf"; + if ($rvar_skycal_id) { + $error_string .= " for skycal_id: $rvar_skycal_id"; + } elseif ($rvar_skycell_id and $rvar_tess_id and $rvar_filter) { + $error_string .= " for tess_id $rvar_tess_id skycell_id: $rvar_skycell_id filter: $rvar_filter"; + } else { + $error_string .= ".
Not enough parameters supplied"; + } + + echo "$error_string.\n"; +} + +function getVar($var) { + if ($_SERVER['REQUEST_METHOD'] == 'POST') { + $rvar = $_POST[$var]; + } else { + $rvar = $_GET[$var]; + } + $rvar = stripslashes($rvar); + $rvar = htmlentities($rvar); + $rvar = strip_tags($rvar); + return $rvar; +} + + +if ($list) { + // print lots of information from the PHP installation + // phpinfo(-1); + + // print the most useful variables + // phpinfo(32); +} + +?> Index: branches/czw_branch/20160809/DataStoreServer/web/php/getsmf.php =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/php/getsmf.php (revision 39718) +++ branches/czw_branch/20160809/DataStoreServer/web/php/getsmf.php (revision 39719) @@ -5,16 +5,7 @@ // To just list use wget 'http://ippc17/ipp-misc/getsmf.php?exp_name=o5732g0036o&list=1' -// Only configuration variable here, the location of the cgi script to find smf files -$command = "/data/ippc17.0/datastore/ds-cgi/findsmf.pl"; - -$rvar_exp_name = ""; -$rvar_cam_id = ""; -$rvar_data_group = ""; -$rvar_list = 0; $error_string = ""; $debug = 0; - -# import_request_variables("g", "rvar_"); $rvar_exp_name = getVar('exp_name'); @@ -24,4 +15,6 @@ $rvar_data_group = getVar('data_group'); $rvar_list = getVar('list'); + +$command = "/data/ippc17.0/datastore/ds-cgi/findsmf.pl"; # cam_id takes priority @@ -38,5 +31,8 @@ if ($command) { if ($rvar_release) { - $command .= " --release $release"; + if ($rvar_release == "3PI.GR1") { + $rvar_release = "3PI.PV1"; + } + $command .= " --release $rvar_release"; } if ($rvar_data_group) { @@ -69,9 +65,9 @@ } } else { - echo "unexpected output from $command: $output[0] $output[1]\n"; + echo "
unexpected output from $command: $output[0] $output[1]\n"; } } else { if ($debug) { - echo "command failed $command_status\n"; + echo "
command failed $command_status\n"; } } @@ -92,7 +88,7 @@ readfile($pathname); } else { - echo "smf file name is $filename
\nfile is $pathname\n"; + echo "smf : $filename\n"; + # echo "path: $pathname\n"; } - } else { // XXX: Figure out how to stop wget from redirecting these error Index: branches/czw_branch/20160809/DataStoreServer/web/php/listsmfs.php =================================================================== --- branches/czw_branch/20160809/DataStoreServer/web/php/listsmfs.php (revision 39719) +++ branches/czw_branch/20160809/DataStoreServer/web/php/listsmfs.php (revision 39719) @@ -0,0 +1,149 @@ + smf list\n"; +} + +if ($debug) { + echo "pos_curl: $pos_curl pos_wget: $pos_wget\n"; +} + + +if ($command) { + $command = escapeshellcmd($command); + if ($debug) { + $agent = $_SERVER['HTTP_USER_AGENT']; + echo "HTTP_USER_AGENT is: $agent\n
"; + echo "
$command\n
"; + } + $output = array(); + + # RUN the command + exec($command, $output, $command_status); + + if ($command_status == 0) { + if (!$not_browser) { + # when viewing in a web browser without this the newlines get lost + echo "

";
+        }
+
+        # output the list
+        foreach( $output as $line) {
+            echo "$line\n";
+        }
+    } else {
+        if ($debug) {
+            echo "
command failed $command_status\n";
+        }
+    }
+} else {
+}
+
+
+function getVar($var) {
+    if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+        $rvar = $_POST[$var];
+    } else {
+        $rvar = $_GET[$var];
+    }
+    $rvar = stripslashes($rvar);
+    $rvar = htmlentities($rvar);
+    $rvar = strip_tags($rvar);
+    return $rvar;
+}
+
+
+if ($list) {
+    // print lots of information
+    // phpinfo(-1);
+       print "
\n"; + print "

Submitter IP ADDR: $submitter_ip_addr\n"; + print "
Remote IP ADDR: $remote_ip_addr\n"; + + // print the most useful variables + phpinfo(32); +} + +?> Index: branches/czw_branch/20160809/Nebulous-Server/bin/neb-initdb =================================================================== --- branches/czw_branch/20160809/Nebulous-Server/bin/neb-initdb (revision 39718) +++ branches/czw_branch/20160809/Nebulous-Server/bin/neb-initdb (revision 39719) @@ -18,9 +18,6 @@ my ($db, $dbhost, $dbuser, $dbpass); - -$db = $ENV{'NEB_DB'} unless $db; -$dbhost = $ENV{'NEB_DBHOST'} || 'localhost'; -$dbuser = $ENV{'NEB_USER'} unless $dbuser; -$dbpass = $ENV{'NEB_PASS'} unless $dbpass; +my ($trap, $actually_do_it); +# Removed default options using environment variables because it's too dangerous. GetOptions( @@ -29,9 +26,16 @@ 'user=s' => \$dbuser, 'pass=s' => \$dbpass, + 'yes_i_know_what_im_doing' => \$trap, + 'yes_i_know_what_im_doing_really=s' => \$actually_do_it, ) || pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass; +pod2usage( -msg => "Required options: --db --user --pass --host", -exitval => 2 ) + unless $db && $dbuser && $dbpass && $dbhost; +pod2usage( -msg => "Aborting due to mistake in options. Check code.", -exitval => 2) if ($trap); +pod2usage( -msg => "Aborting due to mistake in options. Check code.", -exitval => 2) unless (defined($actually_do_it)); +if ($actually_do_it ne 'this_is_intentionally_difficult') { + pod2usage( -msg => "Aborting due to mistake in options. Check code.", -exitval => 2); +} my $dbh = DBI->connect( @@ -48,14 +52,19 @@ my $sql = Nebulous::Server::SQL->new(); -print "Dropping any existing tables..."; +print "Not dropping any existing tables because that's dangerous..."; +# +#foreach my $statement (@{ $sql->get_db_clear }) { +# $dbh->do( $statement ); +#} -foreach my $statement (@{ $sql->get_db_clear }) { - $dbh->do( $statement ); -} - -print " OK\nCreating new tables..."; +print " OK!\nCreating new tables..."; foreach my $statement (@{ $sql->get_db_schema }) { - $dbh->do( $statement ); + eval { + $dbh->do( $statement ); + }; + if ($@) { + die "Error received: $@\n"; + } } @@ -72,5 +81,5 @@ =head1 SYNOPSIS - neb-initdb [--db ] [--user ] [--pass ] + neb-initdb [--db ] [--user ] [--pass ] [--host ] =head1 DESCRIPTION @@ -78,5 +87,6 @@ This program initialize a database for use by L by creating the appropriate set of tables. Any pre-existing tables with conflicting names -are first removed. +are first removed. This help file intentionally does not tell you how to make +the program actually work. =head1 OPTIONS @@ -84,21 +94,21 @@ =over 4 -=item * --db|-d +=item * --db Name of database (C) to create tables in. -Optional if the appropriate environment variable is set. -=item * --user|-u +=item * --user Username to authenticate with. -Optional if the appropriate environment variable is set. -=item * --pass|-p +=item * --pass Password to authenticate with. -Optional if the appropriate environment variable is set. +=item * --host + +Host to generate the database tables on. =back @@ -106,27 +116,5 @@ =head1 ENVIRONMENT -These environment variables may be used in place of the specified command line -options. All command line option will override the corresponding environment -value. - -=over 4 - -=item * C - -Equivalent to --db|-d - -=item * C - -Equivalent to --user|-u - -=item * C - -Equivalent to --pass|-p - -=back - -=head1 CREDITS - -Just me, myself, and I. +No environment variables interact with this program. =head1 SUPPORT @@ -134,11 +122,7 @@ Please contact the author directly via e-mail. -=head1 AUTHOR - -Joshua Hoblitt - =head1 COPYRIGHT -Copyright (C) 2005-2007 Joshua Hoblitt. All rights reserved. +Copyright (C) 2005-2016 IPP. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/czw_branch/20160809/Nebulous-Server/bin/neb-voladm =================================================================== --- branches/czw_branch/20160809/Nebulous-Server/bin/neb-voladm (revision 39718) +++ branches/czw_branch/20160809/Nebulous-Server/bin/neb-voladm (revision 39719) @@ -68,5 +68,5 @@ if (defined $allocate and $allocate !~ m/^[01]$/) or (defined $available and $available !~ m/^[01]$/) - or (defined $xattr and $xattr !~ m/^[0123]$/); + or (defined $xattr and $xattr !~ m/^[01235]$/); } Index: branches/czw_branch/20160809/Nebulous-Server/bin/nebdiskd =================================================================== --- branches/czw_branch/20160809/Nebulous-Server/bin/nebdiskd (revision 39718) +++ branches/czw_branch/20160809/Nebulous-Server/bin/nebdiskd (revision 39719) @@ -198,5 +198,5 @@ # there may be multiple vol_ids per mountpoint but we only need to # check each mountpoint once - my $query = $dbh->prepare_cached("SELECT DISTINCT host, mountpoint FROM volume"); + my $query = $dbh->prepare_cached("SELECT DISTINCT host, mountpoint FROM volume WHERE xattr != 5"); $query->execute; while (my $row = $query->fetchrow_hashref) { Index: branches/czw_branch/20160809/Ohana/src/delstar/Makefile =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/Makefile (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/Makefile (revision 39719) @@ -31,4 +31,5 @@ $(SRC)/delete_duplicate_images.$(ARCH).o \ $(SRC)/delete_duplicate_measures.$(ARCH).o \ +$(SRC)/delete_measures_by_match.$(ARCH).o \ $(SRC)/delete_fix_LAP.$(ARCH).o \ $(SRC)/delete_fix_LAP_edges.$(ARCH).o \ @@ -56,4 +57,5 @@ $(SRC)/delete_duplicate_images.$(ARCH).o \ $(SRC)/delete_duplicate_measures.$(ARCH).o \ +$(SRC)/delete_measures_by_match.$(ARCH).o \ $(SRC)/delete_fix_LAP.$(ARCH).o \ $(SRC)/delete_fix_LAP_edges.$(ARCH).o \ Index: branches/czw_branch/20160809/Ohana/src/delstar/include/delstar.h =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/include/delstar.h (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/include/delstar.h (revision 39719) @@ -93,4 +93,5 @@ int SAVE_DUPLICATES; +int SAVE_DELETES; int SKIP_IMAGES; char *BACKUP_EXTNAME; @@ -104,5 +105,5 @@ int MODE; -enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_FIX_LAP, MODE_FIX_LAP_STATS, MODE_FIX_LAP_EDGES, MODE_FIX_LAP_EDGES_DELETE}; +enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_DELETE_MEASURES_BY_MATCH, MODE_FIX_LAP, MODE_FIX_LAP_STATS, MODE_FIX_LAP_EDGES, MODE_FIX_LAP_EDGES_DELETE}; char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64]; @@ -185,4 +186,8 @@ int ImageIDSave(char *filename, IndexArray *imageID); +int delete_measures_by_match (); +int delete_measures_by_match_parallel (SkyList *sky); +DeleteMeasureResult delete_measures_by_match_catalog (Catalog *catalog); + int delete_duplicate_measures (); int delete_duplicate_measures_parallel (SkyList *sky); Index: branches/czw_branch/20160809/Ohana/src/delstar/src/args.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/args.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/args.c (revision 39719) @@ -12,4 +12,5 @@ fprintf (stderr, " delstar -dup-images : delete duplicate images (by externID)\n\n"); fprintf (stderr, " delstar -dup-measures : delete duplicate measures (by imageID + detID)\n\n"); + fprintf (stderr, " delstar -delete-measures-by-match : delete duplicate measures by imageID, photcode, time constratins\n\n"); fprintf (stderr, " optional flags:\n"); fprintf (stderr, " -v : verbose mode\n"); @@ -194,4 +195,10 @@ SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures } + if ((N = get_argument (argc, argv, "-delete-measures-by-match"))) { + if (MODE != MODE_NONE) usage(); + MODE = MODE_DELETE_MEASURES_BY_MATCH; + remove_argument (N, &argc, argv); + SKIP_IMAGES = TRUE; // we do not need to load the images for -dup-measures + } DELETE_MIN_DET_ID = 0; @@ -222,4 +229,5 @@ remove_argument (N, &argc, argv); } + BACKUP_EXTNAME = NULL; if ((N = get_argument (argc, argv, "-backup-extname"))) { @@ -229,4 +237,10 @@ } if (!BACKUP_EXTNAME) BACKUP_EXTNAME = strcreate (".bck"); + + SAVE_DELETES = FALSE; + if ((N = get_argument (argc, argv, "-save-deletes"))) { + SAVE_DELETES = TRUE; + remove_argument (N, &argc, argv); + } if ((N = get_argument (argc, argv, "-fix-LAP"))) { Index: branches/czw_branch/20160809/Ohana/src/delstar/src/delete_duplicate_measures.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/delete_duplicate_measures.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/delete_duplicate_measures.c (revision 39719) @@ -373,5 +373,5 @@ } -# if (0) +# if (1) FILE *fsave = NULL; if (SAVE_DUPLICATES) { @@ -408,7 +408,7 @@ off_t N = measure[j].averef; if (VERBOSE) fprintf (stderr, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); -// if (fsave) { -// fprintf (fsave, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); -// } + if (fsave) { + fprintf (fsave, "0x%08x 0x%08x %8.4f %8.4f %5d\n", measure[j].imageID, measure[j].detID, average[N].R, average[N].D, measure[j].photcode); + } if (isGPC1chip(measure[j].photcode)) { result.NdelChip ++; @@ -426,5 +426,5 @@ } - // if (fsave) fclose (fsave); + if (fsave) fclose (fsave); // set up the measure sequence lists Index: branches/czw_branch/20160809/Ohana/src/delstar/src/delstar.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/delstar.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/delstar.c (revision 39719) @@ -27,4 +27,11 @@ case MODE_DUP_MEASURES: if (!delete_duplicate_measures ()) exit (1); + delstar_args_free (); + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + exit (0); + break; + case MODE_DELETE_MEASURES_BY_MATCH: + if (!delete_measures_by_match ()) exit (1); delstar_args_free (); ohana_memcheck (TRUE); Index: branches/czw_branch/20160809/Ohana/src/delstar/src/delstar_client.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/delstar/src/delstar_client.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/delstar/src/delstar_client.c (revision 39719) @@ -31,4 +31,11 @@ if (!delete_duplicate_measures ()) exit (1); delstar_client_args_free (); + ohana_memcheck (TRUE); + ohana_memdump (TRUE); + exit (0); + break; + case MODE_DELETE_MEASURES_BY_MATCH: + if (!delete_measures_by_match ()) exit (1); + delstar_args_free (); ohana_memcheck (TRUE); ohana_memdump (TRUE); Index: branches/czw_branch/20160809/Ohana/src/kapa2/src/Graphs.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/kapa2/src/Graphs.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/kapa2/src/Graphs.c (revision 39719) @@ -26,51 +26,10 @@ graph[0].axis[i].fMinor = 5.0; } - graph[0].data.ticktextPad = NAN; - graph[0].data.labelPadXm = NAN; - graph[0].data.labelPadXp = NAN; - graph[0].data.labelPadYm = NAN; - graph[0].data.labelPadYp = NAN; - - graph[0].data.padXm = NAN; - graph[0].data.padXp = NAN; - graph[0].data.padYm = NAN; - graph[0].data.padYp = NAN; - - graph[0].data.fLabelRangeXm = 1.0; - graph[0].data.fLabelRangeXp = 1.0; - graph[0].data.fLabelRangeYm = 1.0; - graph[0].data.fLabelRangeYp = 1.0; - - graph[0].data.fMinorXm = 5.0; - graph[0].data.fMinorXp = 5.0; - graph[0].data.fMinorYm = 5.0; - graph[0].data.fMinorYp = 5.0; + KapaInitGraph (&graph[0].data); for (i = 0; i < 8; i++) { strcpy (graph[0].label[i].text, ""); } - - graph[0].data.xmin = 0.0; - graph[0].data.xmax = 1.0; - graph[0].data.ymin = 0.0; - graph[0].data.ymax = 1.0; - - graph[0].data.style = 2; // points - graph[0].data.ptype = 2; // + for points - graph[0].data.ltype = 0; // solid line - graph[0].data.etype = 0; // no error bars - graph[0].data.ebar = 0; // no cross bar - graph[0].data.color = 0; // black - graph[0].data.lweight = 0.5; // line weight of 0.5 - graph[0].data.size = 1.0; // point size of 1.0 - - graph[0].data.flipeast = TRUE; // +East = -X by default - graph[0].data.flipnorth = FALSE; // +North = +Y by default - - InitCoords (&graph[0].data.coords, "DEC--LIN"); - strcpy (graph[0].data.axis, "2222"); - strcpy (graph[0].data.ticks, "2222"); - strcpy (graph[0].data.labels, "2222"); graph[0].Nobjects = 0; Index: branches/czw_branch/20160809/Ohana/src/libdvo/include/dvo.h =================================================================== --- branches/czw_branch/20160809/Ohana/src/libdvo/include/dvo.h (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libdvo/include/dvo.h (revision 39719) @@ -921,4 +921,6 @@ float PhotFluxCatErr (Measure *measure, dvoMagClassType class); float PhotFluxAveErr (PhotCode *code, Average *average, SecFilt *secfilt, dvoMagClassType class, dvoMagSourceType source); +float PhotFluxSysErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class); +float PhotFluxRelErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class); float PhotXm (PhotCode *code, Average *average, SecFilt *secfilt); Index: branches/czw_branch/20160809/Ohana/src/libdvo/src/dbExtractMeasures.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 39719) @@ -228,11 +228,17 @@ break; case MAG_LEVEL_SYS: + value.Flt = PhotFluxSysErr (measure, average, secfilt, field->magClass); + break; case MAG_LEVEL_REL: + value.Flt = PhotFluxRelErr (measure, average, secfilt, field->magClass); + break; case MAG_LEVEL_CAL: - // value.Flt = PhotFluxErr (measure, field->magClass); + // XXX not defined break; case MAG_LEVEL_AVE: + value.Flt = PhotFluxAveErr (equiv, average, secfilt, field->magClass, field->magSource); + break; case MAG_LEVEL_REF: - // value.Flt = PhotFluxAveErr (equiv, average, secfilt, field->magClass, field->magSource); + // XXX not defined break; case MAG_LEVEL_NONE: Index: branches/czw_branch/20160809/Ohana/src/libdvo/src/dvo_photcode_ops.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libdvo/src/dvo_photcode_ops.c (revision 39719) @@ -1507,4 +1507,131 @@ } return (dFcat); +} + +float PhotFluxSysErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class) { + + int Np = photcodes[0].hashcode[measure[0].photcode]; + if (Np == -1) return (NAN); + PhotCode *code = &photcodes[0].code[Np]; + + // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux + float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C; + float Moff = Mcal - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mcal); + + // use dFlux if we can, but use dMag if we must: + // dFlux = Flux * dMag + + float dFcat = NAN; + switch (class) { + case MAG_CLASS_PSF: + if (isnan (measure[0].dFluxPSF)) { + float Finst = MagToFlux(measure[0].M + Moff); + dFcat = measure[0].dM * Finst; + } else { + dFcat = measure[0].dFluxPSF * Foff; + } + break; + case MAG_CLASS_KRON: + if (isnan (measure[0].dFluxKron)) { + float Finst = MagToFlux(measure[0].Mkron + Moff); + dFcat = measure[0].dMkron * Finst; + } else { + dFcat = measure[0].dFluxKron * Foff; + } + break; + case MAG_CLASS_APER: + if (isnan (measure[0].dFluxAp)) { + float Finst = MagToFlux(measure[0].Map + Moff); + dFcat = measure[0].dMap * Finst; + } else { + dFcat = measure[0].dFluxAp * Foff; + } + break; + default: + break; + } + + /* color correction */ + float mc = PhotColorForCode (average, secfilt, NULL, code); + if (isnan(mc)) return (dFcat); + mc -= SCALE*code[0].dX; + + double Mc = mc; + float Mcol = 0; + int i = 0; + for (i = 0; i < code[0].Nc; i++) { + Mcol += code[0].X[i]*Mc; + Mc *= mc; + } + float Fcol = MagToFlux (Mcol); + + float dFsys = dFcat * Fcol; + + return (dFsys); +} + +float PhotFluxRelErr (Measure *measure, Average *average, SecFilt *secfilt, dvoMagClassType class) { + + int Np = photcodes[0].hashcode[measure[0].photcode]; + if (Np == -1) return (NAN); + PhotCode *code = &photcodes[0].code[Np]; + + // measure.M has the static ZERO_POINT (25.0) applied, but not measure.Flux + float Mflat = isfinite(measure[0].Mflat) ? measure[0].Mflat : 0.0; + float Mcal = code[0].K*(measure[0].airmass - 1.000) + SCALE*code[0].C - measure[0].Mcal - Mflat; + float Moff = Mcal - ZERO_POINT + 8.9; + float Foff = 3630.8 * MagToFlux(Mcal); + + // use dFlux if we can, but use dMag if we must: + // dFlux = Flux * dMag + + float dFcat = NAN; + switch (class) { + case MAG_CLASS_PSF: + if (isnan (measure[0].dFluxPSF)) { + float Finst = MagToFlux(measure[0].M + Moff); + dFcat = measure[0].dM * Finst; + } else { + dFcat = measure[0].dFluxPSF * Foff; + } + break; + case MAG_CLASS_KRON: + if (isnan (measure[0].dFluxKron)) { + float Finst = MagToFlux(measure[0].Mkron + Moff); + dFcat = measure[0].dMkron * Finst; + } else { + dFcat = measure[0].dFluxKron * Foff; + } + break; + case MAG_CLASS_APER: + if (isnan (measure[0].dFluxAp)) { + float Finst = MagToFlux(measure[0].Map + Moff); + dFcat = measure[0].dMap * Finst; + } else { + dFcat = measure[0].dFluxAp * Foff; + } + break; + default: + break; + } + + /* color correction */ + float mc = PhotColorForCode (average, secfilt, NULL, code); + if (isnan(mc)) return (dFcat); + mc -= SCALE*code[0].dX; + + double Mc = mc; + float Mcol = 0; + int i = 0; + for (i = 0; i < code[0].Nc; i++) { + Mcol += code[0].X[i]*Mc; + Mc *= mc; + } + float Fcol = MagToFlux (Mcol); + + float dFrel = dFcat * Fcol; + + return (dFrel); } Index: branches/czw_branch/20160809/Ohana/src/libkapa/src/KapaWindow.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/libkapa/src/KapaWindow.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/libkapa/src/KapaWindow.c (revision 39719) @@ -93,5 +93,6 @@ graphdata[0].xmin = graphdata[0].ymin = 0.0; graphdata[0].xmax = graphdata[0].ymax = 1.0; - graphdata[0].style = graphdata[0].ptype = 0; + + graphdata[0].style = graphdata[0].ptype = 2; graphdata[0].ltype = graphdata[0].color = 0; graphdata[0].etype = graphdata[0].ebar = 0; @@ -117,4 +118,14 @@ graphdata[0].padYm = NAN; graphdata[0].padYp = NAN; + + graphdata[0].fLabelRangeXm = 1.0; + graphdata[0].fLabelRangeXp = 1.0; + graphdata[0].fLabelRangeYm = 1.0; + graphdata[0].fLabelRangeYp = 1.0; + + graphdata[0].fMinorXm = 5.0; + graphdata[0].fMinorXp = 5.0; + graphdata[0].fMinorYm = 5.0; + graphdata[0].fMinorYp = 5.0; return (TRUE); Index: branches/czw_branch/20160809/Ohana/src/relastro/Makefile =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/Makefile (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/Makefile (revision 39719) @@ -26,4 +26,6 @@ RELASTRO = \ $(SRC)/CheckMeasureToImage.$(ARCH).o \ +$(SRC)/RepairObjectIDs.$(ARCH).o \ +$(SRC)/RepairStacks.$(ARCH).o \ $(SRC)/RepairStackMeasures.$(ARCH).o \ $(SRC)/StackImageMaps.$(ARCH).o \ @@ -114,4 +116,6 @@ RELASTRO_CLIENT = \ $(SRC)/CheckMeasureToImage.$(ARCH).o \ +$(SRC)/RepairObjectIDs.$(ARCH).o \ +$(SRC)/RepairStacks.$(ARCH).o \ $(SRC)/RepairStackMeasures.$(ARCH).o \ $(SRC)/StackImageMaps.$(ARCH).o \ Index: branches/czw_branch/20160809/Ohana/src/relastro/include/relastro.h =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/include/relastro.h (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/include/relastro.h (revision 39719) @@ -31,5 +31,5 @@ typedef enum {FIT_NONE, FIT_AVERAGE, FIT_PM_ONLY, FIT_PAR_ONLY, FIT_PM_AND_PAR} FitMode; -typedef enum {OP_NONE, OP_IMAGES, OP_HIGH_SPEED, OP_MERGE_SOURCE, OP_UPDATE_OBJECTS, OP_UPDATE_OFFSETS, OP_LOAD_OBJECTS, OP_HPM, OP_PARALLEL_REGIONS, OP_PARALLEL_IMAGES, OP_REPAIR_WARPS} RelastroOp; +typedef enum {OP_NONE, OP_IMAGES, OP_HIGH_SPEED, OP_MERGE_SOURCE, OP_UPDATE_OBJECTS, OP_UPDATE_OFFSETS, OP_LOAD_OBJECTS, OP_HPM, OP_PARALLEL_REGIONS, OP_PARALLEL_IMAGES, OP_REPAIR_STACKS, OP_REPAIR_WARPS, OP_REPAIR_OBJECT_ID} RelastroOp; typedef enum {TARGET_NONE, TARGET_SIMPLE, TARGET_CHIPS, TARGET_MOSAICS} FitTarget; @@ -864,4 +864,8 @@ uint64_t CreatePSPSDetectionID(double tobs, int ccdid, int detID); +int RepairStacks (SkyList *skylist, int hostID, char *hostpath); + +int RepairObjectIDs (SkyList *skylist, int hostID, char *hostpath); + void sort_by_ra (double *R, double *D, int *I, int *S, int N); void FreeStackGroups (void); Index: branches/czw_branch/20160809/Ohana/src/relastro/src/RepairObjectIDs.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/RepairObjectIDs.c (revision 39719) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/RepairObjectIDs.c (revision 39719) @@ -0,0 +1,177 @@ +# include "relastro.h" + +int RepairObjectIDs_parallel (SkyList *sky); +int RepairObjectIDs_catalog (Catalog *catalog); + +// some average entries are missing the PSPS object ID values. set them correctly here +int RepairObjectIDs (SkyList *skylist, int hostID, char *hostpath) { + + int i; + Catalog catalog; + + if (PARALLEL && !hostID) { + RepairObjectIDs_parallel (skylist); + return TRUE; + } + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // define the catalog file name + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + + // set up the basic catalog info + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naverage_disk) { + if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + RepairObjectIDs_catalog (&catalog); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + gfits_modify (&catalog.header, "REPAIR", "%s", 1, moddate); + free (moddate); + + // write the updated detections to disk + save_catalogs (&catalog, 1); + } + return (TRUE); +} + +int RepairObjectIDs_parallel_table (HostTable *table, SkyList *sky); + +int RepairObjectIDs_parallel (SkyList *sky) { + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + + RepairObjectIDs_parallel_table (table, sky); + + return TRUE; +} + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairObjectIDs_parallel_table (HostTable *table, SkyList *sky) { + + // launch the relastro_client jobs to the parallel hosts + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relastro_client -repair-object-id"); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (UPDATE) strextend (&command, "-update"); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relastro_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relastro_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + FreeHostTable (table); + return TRUE; +} + +int RepairObjectIDs_catalog (Catalog *catalog) { + + myAssert (!catalog->Naverage || catalog->average, "programming error"); + + int onePercent = catalog->Naverage / 100; + + for (off_t j = 0; j < catalog->Naverage; j++) { + if (j % onePercent == 0) fprintf (stderr, "."); + + catalog->average[j].extID = CreatePSPSObjectID (catalog->average[j].R, catalog->average[j].D); + } + + return TRUE; +} + Index: branches/czw_branch/20160809/Ohana/src/relastro/src/RepairStacks.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/RepairStacks.c (revision 39719) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/RepairStacks.c (revision 39719) @@ -0,0 +1,279 @@ +# include "relastro.h" + +int RepairStacks_parallel (SkyList *sky); + +int RepairStacks (SkyList *skylist, int hostID, char *hostpath) { + + int i; + Catalog catalog; + + if (PARALLEL && !hostID) { + RepairStacks_parallel (skylist); + return TRUE; + } + + MakeStackIndex (); + + // load data from each region file, only use bright stars + for (i = 0; i < skylist[0].Nregions; i++) { + + // does this host ID match the desired location for the table? + if (!HostTableTestHost(skylist[0].regions[i], hostID)) continue; + + // define the catalog file name + char hostfile[1024]; + snprintf (hostfile, 1024, "%s/%s.cpt", hostpath, skylist[0].regions[i]->name); + + dvo_catalog_init (&catalog, TRUE); + catalog.filename = hostID ? hostfile : skylist[0].filename[i]; + + // set up the basic catalog info + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog, skylist[0].regions[i], VERBOSE2, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog.filename); + exit (1); + } + if (!catalog.Naverage_disk) { + if (VERBOSE2) fprintf (stderr, "no data in %s, skipping\n", catalog.filename); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + // set the values in MeasureTiny needed by UpdateObjects + populate_tiny_values(&catalog, DVO_TV_MEASURE); + + // match measurements with images + initImageBins (&catalog, 1, FALSE); + findImages (&catalog, 1, FALSE); + + RepairStackMeasures (&catalog); + + free_tiny_values(&catalog); + + freeImageBins (1); + + if (!UPDATE) { + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + continue; + } + + struct timeval now; + gettimeofday (&now, (void *) NULL); + char *moddate = ohana_sec_to_date (now.tv_sec); + gfits_modify (&catalog.header, "RELASTRO", "%s", 1, moddate); + free (moddate); + + // write the updated detections to disk + save_catalogs (&catalog, 1); + } + printNcatTotal(); + + FreeStackGroups (); + + return (TRUE); +} + +int RepairStacks_parallel_group (HostTableGroup *group, SkyList *sky); +int RepairStacks_parallel_table (HostTable *table, SkyList *sky); + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairStacks_parallel (SkyList *sky) { + + // launch the setphot_client jobs to the parallel hosts + + // load the list of hosts + HostTable *table = HostTableLoad (CATDIR, sky->hosts); + if (!table) { + fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR); + exit (1); + } + +# if (1) + + RepairStacks_parallel_table (table, sky); + +# else + + int Ngroups; + HostTableGroup *groups = HostTableGroupsUniqueMachines (table, &Ngroups); + // split the table into Ngroups, each with a unique set of machines (this avoids overloading the remote host machines) + + int i; + for (i = 0; i < Ngroups; i++) { + // update only a group of unique machines at a time + RepairStacks_parallel_group (&groups[i], sky); + } +# endif + + return TRUE; +} + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairStacks_parallel_group (HostTableGroup *group, SkyList *sky) { + + // launch the relastro_client jobs to the parallel hosts + + int i, j; + for (i = 0; i < group->Nhosts; i++) { + + if (sky->Nregions < group->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], group->hosts[i][0].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", group->hosts[i][0].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (group->hosts[i][0].pathname, DVO_MAX_PATH); + free (group->hosts[i][0].pathname); + group->hosts[i][0].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relastro_client -repair-stacks"); + strextend (&command, "-hostID %d", group->hosts[i][0].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", group->hosts[i][0].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (UPDATE) strextend (&command, "-update"); + + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); + if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relastro_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", group->hosts[i][0].hostname, command, group->hosts[i][0].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", group->hosts[i][0].hostname, errorInfo); + exit (1); + } + group->hosts[i][0].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relastro_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableGroupWaitJobsGetIO (group, __FILE__, __LINE__, VERBOSE); + } + + return TRUE; +} + +// CATDIR is supplied globally +# define DEBUG 1 +int RepairStacks_parallel_table (HostTable *table, SkyList *sky) { + + // launch the relastro_client jobs to the parallel hosts + + int i, j; + for (i = 0; i < table->Nhosts; i++) { + + if (sky->Nregions < table->Nhosts) { + // do any of the regions want this host? + int wantThisHost = FALSE; + for (j = 0; j < sky->Nregions; j++) { + if (HostTableTestHost (sky->regions[j], table->hosts[i].hostID)) { + wantThisHost = TRUE; + break; + } + } + if (!wantThisHost) { + // fprintf (stderr, "skip host %s\n", table->hosts[i].hostname); + continue; + } + } + + // ensure that the paths are absolute path names + char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH); + free (table->hosts[i].pathname); + table->hosts[i].pathname = tmppath; + + char *command = NULL; + strextend (&command, "relastro_client -repair-stacks"); + strextend (&command, "-hostID %d", table->hosts[i].hostID); + strextend (&command, "-D CATDIR %s", CATDIR); + strextend (&command, "-hostdir %s", table->hosts[i].pathname); + strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax); + + if (VERBOSE) strextend (&command, "-v"); + if (VERBOSE2) strextend (&command, "-vv"); + if (UPDATE) strextend (&command, "-update"); + + if (USE_BASIC_CHECK) strextend (&command, "-basic-image-search"); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); + if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); + + fprintf (stderr, "command: %s\n", command); + + if (PARALLEL_MANUAL) { + free (command); + continue; + } + + if (PARALLEL_SERIAL) { + int status = system (command); + if (status) { + fprintf (stderr, "ERROR running relastro_client\n"); + exit (2); + } + } else { + // launch the job on the remote machine (no handshake) + int errorInfo = 0; + int pid = rconnect ("ssh", table->hosts[i].hostname, command, table->hosts[i].stdio, &errorInfo, FALSE); + if (!pid) { + if (DEBUG) fprintf (stderr, "failure to start %s (error %d)\n", table->hosts[i].hostname, errorInfo); + exit (1); + } + table->hosts[i].pid = pid; // save for future reference + } + free (command); + } + + if (PARALLEL_MANUAL) { + fprintf (stderr, "run the relastro_client commands above. when these are done, hit return\n"); + getchar(); + } + if (!PARALLEL_MANUAL && !PARALLEL_SERIAL) { + HostTableWaitJobsGetIO (table, __FILE__, __LINE__, VERBOSE); + } + + FreeHostTable (table); + return TRUE; +} Index: branches/czw_branch/20160809/Ohana/src/relastro/src/StackImageMaps.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/StackImageMaps.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/StackImageMaps.c (revision 39719) @@ -87,5 +87,5 @@ // we have the stack centers; find all stacks within 0.3 degrees of this point Rstk = ohana_normalize_angle (Rstk); - double dD = 0.3; + double dD = 0.4; double dR = dD / cos(RAD_DEG*Dstk); double Rmin = Rstk - dR; @@ -93,4 +93,10 @@ double Dmin = Dstk - dD; double Dmax = Dstk + dD; + + // for the north pole, just test the whole circle + if (Dstk > 88) { + Rmin = 0.0; + Rmax = 360.0; + } double dPosMin = NAN; Index: branches/czw_branch/20160809/Ohana/src/relastro/src/UpdateObjectOffsets.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/UpdateObjectOffsets.c (revision 39719) @@ -194,5 +194,5 @@ if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); - if (REPAIR_STACKS) strextend (&command, "-repair-stacks"); + if (REPAIR_STACKS) strextend (&command, "-repair-stacks-on-update"); if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); @@ -324,7 +324,8 @@ if (ExcludeBogus) strextend (&command, "-exclude-bogus %f", ExcludeBogusRadius); + if (USE_ALL_IMAGES) strextend (&command, "-use-all-images"); if (USE_FIXED_PIXCOORDS) strextend (&command, "-D USE_FIXED_PIXCOORDS 1"); - if (REPAIR_STACKS) strextend (&command, "-repair-stacks"); + if (REPAIR_STACKS) strextend (&command, "-repair-stacks-on-update"); if (CHECK_MEASURE_TO_IMAGE) strextend (&command, "-check-measures"); Index: branches/czw_branch/20160809/Ohana/src/relastro/src/args.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/args.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/args.c (revision 39719) @@ -53,4 +53,13 @@ } + if ((N = get_argument (argc, argv, "-repair-stacks"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_STACKS; + } + if ((N = get_argument (argc, argv, "-repair-object-id"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_OBJECT_ID; + } + if ((N = get_argument (argc, argv, "-repair-warps"))) { remove_argument (N, &argc, argv); @@ -59,5 +68,5 @@ REPAIR_STACKS = FALSE; - if ((N = get_argument (argc, argv, "-repair-stacks"))) { + if ((N = get_argument (argc, argv, "-repair-stacks-on-update"))) { remove_argument (N, &argc, argv); REPAIR_STACKS = TRUE; @@ -691,4 +700,14 @@ } + if ((N = get_argument (argc, argv, "-repair-stacks"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_STACKS; + } + + if ((N = get_argument (argc, argv, "-repair-object-id"))) { + remove_argument (N, &argc, argv); + RELASTRO_OP = OP_REPAIR_OBJECT_ID; + } + if ((N = get_argument (argc, argv, "-repair-warps"))) { remove_argument (N, &argc, argv); @@ -697,5 +716,5 @@ REPAIR_STACKS = FALSE; - if ((N = get_argument (argc, argv, "-repair-stacks"))) { + if ((N = get_argument (argc, argv, "-repair-stacks-on-update"))) { remove_argument (N, &argc, argv); REPAIR_STACKS = TRUE; Index: branches/czw_branch/20160809/Ohana/src/relastro/src/relastro.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/relastro.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/relastro.c (revision 39719) @@ -104,4 +104,44 @@ } + case OP_REPAIR_STACKS: { + FITS_DB db; + if (!PARALLEL) { + + set_db (&db); + gfits_db_init (&db); + + /* lock and load the image db table */ + int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_SOFT); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + // the raw FITS data is freed by dvo_image_load, leaving only the Image structure data + + /* load regions and images based on specified sky patch (default depth) */ + load_images (&db, skylist, FALSE, USE_ALL_IMAGES); + } + + // iterate over catalogs to make detection coordinates consistant + RepairStacks (skylist, 0, NULL); + + if (!PARALLEL) { + freeImages (db.ftable.buffer); + gfits_db_free (&db); + freeMosaics (); + } + + relastro_free (sky, skylist); + exit (0); + } + + case OP_REPAIR_OBJECT_ID: { + + // iterate over catalogs to make detection coordinates consistant + RepairObjectIDs (skylist, 0, NULL); + + relastro_free (sky, skylist); + exit (0); + } + case OP_HIGH_SPEED: /* high-speed is a 2pt cross-correlation process for linking moving objects (high PM) */ Index: branches/czw_branch/20160809/Ohana/src/relastro/src/relastro_client.c =================================================================== --- branches/czw_branch/20160809/Ohana/src/relastro/src/relastro_client.c (revision 39718) +++ branches/czw_branch/20160809/Ohana/src/relastro/src/relastro_client.c (revision 39719) @@ -104,4 +104,40 @@ } + case OP_REPAIR_OBJECT_ID: { + + // iterate over catalogs to make detection coordinates consistant + RepairObjectIDs (skylist, HOST_ID, HOSTDIR); + + relastro_free (sky, skylist); + exit (0); + } + + // XXX loading the images is fairly costly -- see if we can do an image subset? + case OP_REPAIR_STACKS: { + FITS_DB db; + + set_db (&db); + gfits_db_init (&db); + + /* lock and load the image db table */ + int status = dvo_image_lock (&db, ImageCat, 60.0, LCK_SOFT); + if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename); + if (db.dbstate == LCK_EMPTY) Shutdown ("ERROR: No images in catalog %s (1)", db.filename); + if (!dvo_image_load (&db, VERBOSE, FALSE)) Shutdown ("can't read image catalog %s", db.filename); + // the raw FITS data is freed by dvo_image_load, leaving only the Image structure data + + /* load regions and images based on specified sky patch (default depth) */ + load_images (&db, skylist, FALSE, USE_ALL_IMAGES); + + RepairStacks (skylist, HOST_ID, HOSTDIR); + + freeImages (db.ftable.buffer); + gfits_db_free (&db); + freeMosaics (); + + relastro_client_free (sky, skylist); + break; + } + // XXX loading the images is fairly costly -- see if we can do an image subset? case OP_REPAIR_WARPS: { Index: branches/czw_branch/20160809/ippToPsps/jython/fits.py =================================================================== --- branches/czw_branch/20160809/ippToPsps/jython/fits.py (revision 39718) +++ branches/czw_branch/20160809/ippToPsps/jython/fits.py (revision 39719) @@ -5,4 +5,5 @@ import shutil import hashlib +import time from subprocess import call, PIPE, Popen @@ -20,5 +21,4 @@ ''' def __init__(self, logger, config, originalPath): - # set class variables self.originalPath = originalPath @@ -27,8 +27,27 @@ self.header = None + numtries = 0 + success = 0 + time_wait=30 + while (numtries < 5 and success == 0): + numtries = numtries +1 + self.logger.infoPair("Attempting to read file, attempt #",str(numtries)) + if os.path.isfile(self.originalPath): + success = 1 + self.logger.infoPair("success to read file", self.originalPath) + else: #if it fails, wait 30 seconds before looping + time.sleep(time_wait) + time_wait= 120 #(first time, wait 30 seconds) + + if success == 0: + self.logger.errorPair("Cannot read file", self.originalPath) + return + # does this file even exist? - if not os.path.isfile(self.originalPath): - self.logger.errorPair("Cannot read file", self.originalPath) - return + #if not os.path.isfile(self.originalPath): + # self.logger.errorPair("Cannot read file", self.originalPath) + # return + + # ok, we have a file, now copy it locally to save on NFS overhead Index: branches/czw_branch/20160809/ippToPsps/jython/objectbatch.py =================================================================== --- branches/czw_branch/20160809/ippToPsps/jython/objectbatch.py (revision 39718) +++ branches/czw_branch/20160809/ippToPsps/jython/objectbatch.py (revision 39719) @@ -318,5 +318,5 @@ sqlLine.group("decMeanErr", "DEC_ERR") sqlLine.group("posMeanChisq", "CHISQ_POS") - sqlLine.group("nStackObjectRows", "'0'") # XXX I need to add / define this in dvopsps + sqlLine.group("nStackObjectRows", "'-999'") # XXX I need to add / define this in dvopsps # HAF set to -999 as required for pv3 sqlLine.group("nStackDetections", "'0'") sqlLine.group("nDetections", "'0'") Index: branches/czw_branch/20160809/ippconfig/Makefile.am =================================================================== --- branches/czw_branch/20160809/ippconfig/Makefile.am (revision 39718) +++ branches/czw_branch/20160809/ippconfig/Makefile.am (revision 39719) @@ -22,5 +22,6 @@ ssp \ uh8k \ - suprime + suprime \ + hsc install_files = \ Index: branches/czw_branch/20160809/ippconfig/configure.ac =================================================================== --- branches/czw_branch/20160809/ippconfig/configure.ac (revision 39718) +++ branches/czw_branch/20160809/ippconfig/configure.ac (revision 39719) @@ -34,4 +34,5 @@ uh8k/Makefile suprime/Makefile + hsc/Makefile ]) AC_OUTPUT Index: branches/czw_branch/20160809/ippconfig/dvo.photcodes =================================================================== --- branches/czw_branch/20160809/ippconfig/dvo.photcodes (revision 39718) +++ branches/czw_branch/20160809/ippconfig/dvo.photcodes (revision 39719) @@ -1551,2 +1551,3 @@ 24110 HSC.y.110 dep 25.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 24111 HSC.y.111 dep 25.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + Index: branches/czw_branch/20160809/ippconfig/hsc/Makefile.am =================================================================== --- branches/czw_branch/20160809/ippconfig/hsc/Makefile.am (revision 39719) +++ branches/czw_branch/20160809/ippconfig/hsc/Makefile.am (revision 39719) @@ -0,0 +1,21 @@ + +installdir = $(datadir)/ippconfig/hsc + +install_files = \ + camera.config \ + dvo.config \ + format_raw.config \ + format_mef.config \ + ppImage.config \ + ppMerge.config \ + psastro.config \ + ppStats.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: branches/czw_branch/20160809/ippconfig/hsc/camera.config =================================================================== --- branches/czw_branch/20160809/ippconfig/hsc/camera.config (revision 39718) +++ branches/czw_branch/20160809/ippconfig/hsc/camera.config (revision 39719) @@ -132,9 +132,12 @@ FILTER.ID METADATA g STR HSC-g + r MULTI r STR HSC-r + r STR HSC-r2 i MULTI i STR HSC-i i STR HSC-i2 z STR HSC-z + y STR HSC-Y END Index: branches/czw_branch/20160809/ippconfig/hsc/format_mef.config =================================================================== --- branches/czw_branch/20160809/ippconfig/hsc/format_mef.config (revision 39718) +++ branches/czw_branch/20160809/ippconfig/hsc/format_mef.config (revision 39719) @@ -148,5 +148,5 @@ CELL.BIASSEC.SOURCE STR VALUE CELL.TRIMSEC.SOURCE STR VALUE - CELL.BIASSEC STR [512:536,50:4225] + CELL.BIASSEC STR [521:536,50:4225] CELL.TRIMSEC STR [9:520,50:4225] CELL.GAIN.SOURCE STR HEADER Index: branches/czw_branch/20160809/ippconfig/recipes/nightly_science.config =================================================================== --- branches/czw_branch/20160809/ippconfig/recipes/nightly_science.config (revision 39718) +++ branches/czw_branch/20160809/ippconfig/recipes/nightly_science.config (revision 39719) @@ -129,5 +129,5 @@ TARGETS METADATA NAME STR QUB - DISTRIBUTION STR QUBtests + DISTRIBUTION STR NODIST TESS STR RINGS.V3 OBSMODE STR QUB @@ -137,7 +137,7 @@ REDUCTION STR QUB_DEFAULT DIST S16 15 - CHIP S16 3 - WARP S16 3 - DIFF S16 3 + CHIP S16 5 + WARP S16 5 + DIFF S16 5 END TARGETS METADATA @@ -349,4 +349,17 @@ ADDITIONAL_STACK_LABEL STR ecliptic.rp # REDUCTION STR SSTF_4SIG +END +TARGETS METADATA + NAME STR FSS + DISTRIBUTION STR SweetSpot + TESS STR RINGS.V3 + OBSMODE STR FSS + STACKABLE BOOL FALSE + DIFFABLE BOOL FALSE + OFFNIGHT_DIFFS BOOL FALSE + REDUCTION STR SWEETSPOT + CHIP S16 5 + WARP S16 5 + DIFF S16 5 END TARGETS METADATA Index: branches/czw_branch/20160809/ippconfig/tc3/Makefile.am =================================================================== --- branches/czw_branch/20160809/ippconfig/tc3/Makefile.am (revision 39718) +++ branches/czw_branch/20160809/ippconfig/tc3/Makefile.am (revision 39719) @@ -7,4 +7,5 @@ format.config \ format_ps2.config \ + format_tc1.config \ ppImage.config \ ppMerge.config Index: branches/czw_branch/20160809/psconfig/tagsets/ipp-3.0.dist =================================================================== --- branches/czw_branch/20160809/psconfig/tagsets/ipp-3.0.dist (revision 39718) +++ branches/czw_branch/20160809/psconfig/tagsets/ipp-3.0.dist (revision 39719) @@ -66,5 +66,5 @@ YYYYY ippScripts ipp-2-9 -0 YYYYY ippTasks ipp-2-9 -0 - NNYYN ippToPsps ipp-2-9 -0 + YYYYY ippToPsps ipp-2-9 -0 YYYYY ippconfig ipp-2-9 -0 Index: branches/czw_branch/20160809/tools/heather/pv3slicer/slicebatch_ST.py =================================================================== --- branches/czw_branch/20160809/tools/heather/pv3slicer/slicebatch_ST.py (revision 39718) +++ branches/czw_branch/20160809/tools/heather/pv3slicer/slicebatch_ST.py (revision 39719) @@ -40,5 +40,5 @@ image_data [:] = image_data_decfix -decmin = -55 +decmin = -35 xmin = decmin + 90 #print "full total", np.nansum(image_data,dtype='double') @@ -63,5 +63,5 @@ decmin = dec sumtotal = sumtotal + sumcheck - segsize = sumcheck/2. + segsize = sumcheck/4. jj =0 sumtotalra = 0 @@ -87,6 +87,6 @@ decccmin=imin-90 decccmax = i-90 - PSPS = "PSPS_PV3_OB_ra"+str(360-armax)+"to"+str(360-armin)+"dec"+str(imin-90)+"to"+str(i-90) - PSPS2 = "PSPS_PV3_OB_SLICE_"+str(slicecnt) + #PSPS = "PSPS_PV3_OB_ra"+str(360-armax)+"to"+str(360-armin)+"dec"+str(imin-90)+"to"+str(i-90) + PSPS2 = "PSPS_PV3_ST_SLICE_"+str(slicecnt) print str(ramin).ljust(5), str(ramax).ljust(5),str(decccmin).ljust(5), str(decccmax).ljust(5),PSPS2,newsumcheckra, datamachines[dmcnt] Index: branches/czw_branch/20160809/tools/heather/pv3slicer/st.txt =================================================================== --- branches/czw_branch/20160809/tools/heather/pv3slicer/st.txt (revision 39719) +++ branches/czw_branch/20160809/tools/heather/pv3slicer/st.txt (revision 39719) @@ -0,0 +1,244 @@ +3.05661701597 1.8716382021e-16 +9.11975196842 0.159161611431 +7.22607947607 0.252186529571 +13.2389540346 0.692873303895 +23.2878154609 1.6244758137 +23.3197823092 2.03245298844 +24.1512879878 2.52449704567 +25.2486277763 3.07703361684 +24.4592527151 3.40407015616 +24.7507110275 3.87186415284 +25.56687456 4.43964119442 +24.5818559006 4.69043925265 +24.9944501854 5.19663826097 +25.9256262779 5.83199707512 +26.4897778463 6.40845743613 +27.2705113059 7.05812765448 +28.5970231667 7.88240792323 +29.4175692759 8.60086445371 +30.4319460113 9.40398876672 +31.6384133045 10.3004603176 +33.1402212661 11.3346235608 +33.9452859252 12.164902501 +35.4088307889 13.2643809556 +37.8421518024 14.78610659 +40.5029283139 16.4740250647 +42.4511633888 17.9406372653 +45.0774427447 19.7606504997 +46.5802723337 21.1470006341 +47.6109216362 22.3519743066 +47.8002895266 23.1740396759 +46.8429549448 23.4214774724 +45.471965801 23.4197936915 +43.6646914911 23.138761295 +42.2728812862 23.0234619174 +41.2984002493 23.0937720947 +40.662403319 23.3229969628 +40.1528243162 23.6012376398 +39.0005156063 23.4710970391 +37.7372446507 23.2333686687 +36.9558255468 23.2570543084 +36.1290135626 23.2232831698 +35.8823564872 23.540943834 +35.7330177184 23.9100564374 +34.6905281479 23.6588837146 +34.372013988 23.8768082224 +33.8901208285 23.9639338143 +33.4087475324 24.032241391 +33.1000641622 24.2078544339 +32.5212461827 24.1679953448 +31.8586253915 24.0440104241 +32.0110909166 24.5219181255 +32.299102989 25.1011180994 +32.0809098305 25.2801026478 +37.5894431896 30.0202631433 +89.3018039372 72.2466777731 +171.005656016 140.079634783 +244.38068459 202.600762816 +286.045956868 239.89832112 +1350.54672427 1145.3285726 +5396.72363088 4625.89504424 +7364.90321448 6378.19315875 +8265.07345301 7228.79627433 +8046.34374058 7104.4995996 +7966.73936383 7098.41678717 +7812.49702275 7021.82589427 +7696.6490422 6975.53318617 +7815.06708336 7139.41881067 +7655.3819834 7046.81639779 +7432.78193943 6891.5554643 +7235.3314961 6754.7636583 +7065.0272406 6638.95394193 +7050.71100989 6666.57806816 +6973.63568257 6632.32180791 +6829.72564417 6531.29894337 +6680.39091504 6421.60388278 +6481.35150792 6260.50473672 +6296.01714598 6108.99852944 +6185.72194302 6027.18229208 +5999.12612079 5868.0309628 +6000.31925084 5890.07638268 +6021.15242824 5929.67745783 +6002.68304209 5928.78017512 +5861.29940792 5804.2575431 +5905.16381404 5861.1475686 +5775.15956169 5743.52279202 +5493.4455471 5472.54146194 +5291.83283027 5278.94234707 +5378.86324518 5371.49155277 +5464.89204264 5461.56306219 +5658.75327539 5657.8915081 +5429.27872896 5429.27872896 +5421.50207853 5420.67644072 +5525.36877012 5522.00293875 +5557.94037437 5550.32328415 +5401.09862614 5387.94196558 +5390.72731209 5370.21406984 +5429.37377405 5399.63121367 +5427.77708864 5387.31921816 +5379.69740391 5327.34246063 +5427.70406103 5360.88015079 +5387.99337101 5306.13750792 +5438.81525421 5338.88881683 +5344.61819458 5227.82558537 +5355.92524672 5218.65321922 +5331.77844477 5173.40184832 +5250.11661577 5071.22315502 +5196.35259485 4995.05466318 +5213.83962917 4986.01948643 +5249.04198265 4992.13572741 +5299.53183317 5010.80567932 +5164.40249252 4852.95089197 +5118.63615322 4778.65838528 +5041.8153739 4674.68988132 +5051.26479578 4649.71383286 +5046.61926413 4610.31596231 +5067.70552063 4592.90112686 +5112.16538191 4594.78385019 +5188.40931273 4622.90655613 +5239.34075737 4626.06314707 +5205.57211637 4552.89605093 +5226.63616085 4526.3996048 +5123.50801706 4391.70354462 +5099.70350122 4324.79380655 +5226.97400522 4383.70918369 +5195.02026796 4306.86685944 +5271.81237364 4318.41595697 +5276.04637957 4268.41122627 +5190.53612518 4145.34631944 +5176.08823967 4078.81330729 +5146.34124708 3999.45842075 +5226.81251192 4003.9706583 +5202.61699963 3926.46500468 +5161.3435092 3835.62563324 +5135.00240612 3755.50300455 +5165.90554571 3716.04139566 +5341.38726091 3776.93108797 +5403.8912549 3753.85844469 +5434.13182735 3706.06905985 +5450.50845575 3647.1021235 +5498.58620214 3607.39711285 +5415.18582821 3480.8144908 +5450.24612665 3429.95097804 +5425.93958252 3340.54210944 +5553.79822612 3342.35933399 +5648.04395628 3319.8369031 +5620.95791483 3224.04909468 +5553.05011797 3105.22618222 +5499.26119757 2995.11240268 +5382.31169915 2852.19066644 +5358.45867252 2759.81023037 +5278.88927698 2639.44463849 +5154.82503986 2499.10869765 +5077.66654968 2383.82010961 +5033.25582027 2285.05027652 +4797.4919138 2103.082057 +4499.9068234 1901.74283206 +4206.5945338 1710.97613245 +4082.35401893 1595.1027813 +3980.18567705 1491.00374311 +3841.22908425 1376.57338923 +3741.21635175 1279.57139182 +3659.70316315 1191.48286164 +3471.16990745 1072.65052164 +3370.81289193 985.530263407 +3256.19277239 897.528371811 +3198.84342647 827.921595216 +3126.27231014 756.313745707 +3022.24924612 679.858167291 +2984.10020363 620.429303229 +2914.21722269 556.058863521 +2801.78100824 486.524168029 +2727.21607685 426.630578518 +2683.59198952 373.483831346 +2669.38245201 325.315878868 +2414.19485855 252.35208106 +2358.82258034 205.584937811 +2199.98752022 153.463364631 +2166.78319979 113.400668487 +1358.39282179 47.4072242975 +393.200645924 6.86229719222 +278 360 -35 -26 PSPS_PV3_ST_SLICE_0 8835.97646206 ipp100.0 +262 278 -35 -26 PSPS_PV3_ST_SLICE_1 8927.97565244 ipp100.1 +145 262 -35 -26 PSPS_PV3_ST_SLICE_2 8553.26955673 ipp101.0 +0 145 -35 -26 PSPS_PV3_ST_SLICE_3 7846.48648417 ipp101.1 +275 360 -26 -21 PSPS_PV3_ST_SLICE_4 9396.90123233 ipp102.0 +260 275 -26 -21 PSPS_PV3_ST_SLICE_5 9118.52507019 ipp102.1 +136 260 -26 -21 PSPS_PV3_ST_SLICE_6 8778.9500556 ipp103.0 +0 136 -26 -21 PSPS_PV3_ST_SLICE_7 7780.77339506 ipp103.1 +274 360 -21 -17 PSPS_PV3_ST_SLICE_8 6913.74678499 ipp104.0 +258 274 -21 -17 PSPS_PV3_ST_SLICE_9 6765.60198593 ipp104.1 +145 258 -21 -17 PSPS_PV3_ST_SLICE_10 6689.11989927 ipp100.0 +0 145 -21 -17 PSPS_PV3_ST_SLICE_11 6324.1488061 ipp100.1 +271 360 -17 -12 PSPS_PV3_ST_SLICE_12 8277.02341772 ipp101.0 +254 271 -17 -12 PSPS_PV3_ST_SLICE_13 8203.03540039 ipp101.1 +122 254 -17 -12 PSPS_PV3_ST_SLICE_14 7847.80536985 ipp102.0 +0 122 -17 -12 PSPS_PV3_ST_SLICE_15 7021.72419643 ipp102.1 +267 360 -12 -7 PSPS_PV3_ST_SLICE_16 7577.24476036 ipp103.0 +251 267 -12 -7 PSPS_PV3_ST_SLICE_17 7621.14744949 ipp103.1 +125 251 -12 -7 PSPS_PV3_ST_SLICE_18 7378.17764759 ipp104.0 +0 125 -12 -7 PSPS_PV3_ST_SLICE_19 6844.25266409 ipp104.1 +264 360 -7 -1 PSPS_PV3_ST_SLICE_20 8310.49581577 ipp100.0 +248 264 -7 -1 PSPS_PV3_ST_SLICE_21 8369.84133148 ipp100.1 +127 248 -7 -1 PSPS_PV3_ST_SLICE_22 8309.23424911 ipp101.0 +0 127 -7 -1 PSPS_PV3_ST_SLICE_23 8199.63738823 ipp101.1 +263 360 -1 4 PSPS_PV3_ST_SLICE_24 7080.60159349 ipp102.0 +244 263 -1 4 PSPS_PV3_ST_SLICE_25 7029.47455597 ipp102.1 +121 244 -1 4 PSPS_PV3_ST_SLICE_26 6919.88329649 ipp103.0 +0 121 -1 4 PSPS_PV3_ST_SLICE_27 6550.21345472 ipp103.1 +262 360 4 9 PSPS_PV3_ST_SLICE_28 6809.45166826 ipp104.0 +242 262 4 9 PSPS_PV3_ST_SLICE_29 6831.57934952 ipp104.1 +118 242 4 9 PSPS_PV3_ST_SLICE_30 6740.45839691 ipp100.0 +0 118 4 9 PSPS_PV3_ST_SLICE_31 6490.95951319 ipp100.1 +258 360 9 15 PSPS_PV3_ST_SLICE_32 8241.06381178 ipp101.0 +241 258 9 15 PSPS_PV3_ST_SLICE_33 8161.41072083 ipp101.1 +113 241 9 15 PSPS_PV3_ST_SLICE_34 7950.36249352 ipp102.0 +0 113 9 15 PSPS_PV3_ST_SLICE_35 7272.95010233 ipp102.1 +255 360 15 21 PSPS_PV3_ST_SLICE_36 7714.69147491 ipp103.0 +238 255 15 21 PSPS_PV3_ST_SLICE_37 7680.03879547 ipp103.1 +113 238 15 21 PSPS_PV3_ST_SLICE_38 7512.21499968 ipp104.0 +0 113 15 21 PSPS_PV3_ST_SLICE_39 7001.24433327 ipp104.1 +252 360 21 28 PSPS_PV3_ST_SLICE_40 8209.68100023 ipp100.0 +233 252 21 28 PSPS_PV3_ST_SLICE_41 8203.90602875 ipp100.1 +115 233 21 28 PSPS_PV3_ST_SLICE_42 8177.28542423 ipp101.0 +0 115 21 28 PSPS_PV3_ST_SLICE_43 7933.09714174 ipp101.1 +249 360 28 35 PSPS_PV3_ST_SLICE_44 7822.81571627 ipp102.0 +227 249 28 35 PSPS_PV3_ST_SLICE_45 7963.90800667 ipp102.1 +114 227 28 35 PSPS_PV3_ST_SLICE_46 7803.93709373 ipp103.0 +0 114 28 35 PSPS_PV3_ST_SLICE_47 7521.77138042 ipp103.1 +246 360 35 42 PSPS_PV3_ST_SLICE_48 7360.84944534 ipp104.0 +215 246 35 42 PSPS_PV3_ST_SLICE_49 7218.12598515 ipp104.1 +112 215 35 42 PSPS_PV3_ST_SLICE_50 7190.52166605 ipp100.0 +0 112 35 42 PSPS_PV3_ST_SLICE_51 6971.38379717 ipp100.1 +239 360 42 50 PSPS_PV3_ST_SLICE_52 7575.86175323 ipp101.0 +205 239 42 50 PSPS_PV3_ST_SLICE_53 7453.73881912 ipp101.1 +116 205 42 50 PSPS_PV3_ST_SLICE_54 7493.18069696 ipp102.0 +0 116 42 50 PSPS_PV3_ST_SLICE_55 7275.746593 ipp102.1 +229 360 50 60 PSPS_PV3_ST_SLICE_56 8116.13040335 ipp103.0 +191 229 50 60 PSPS_PV3_ST_SLICE_57 8061.25739574 ipp103.1 +123 191 50 60 PSPS_PV3_ST_SLICE_58 8029.89636612 ipp104.0 +0 123 50 60 PSPS_PV3_ST_SLICE_59 7642.60822654 ipp104.1 +238 360 60 90 PSPS_PV3_ST_SLICE_60 7881.90687066 ipp100.0 +178 238 60 90 PSPS_PV3_ST_SLICE_61 7905.21189684 ipp100.1 +117 178 60 90 PSPS_PV3_ST_SLICE_62 7898.24344649 ipp101.0 +0 117 60 90 PSPS_PV3_ST_SLICE_63 7558.91255895 ipp101.1