Index: branches/czw_branch/20160809/DataStoreServer/scripts/dsprodindex
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/scripts/dsprodindex (revision 39651)
+++ branches/czw_branch/20160809/DataStoreServer/scripts/dsprodindex (revision 39719)
@@ -75,6 +75,8 @@
#
# XXX: the spec doesn't say what should happen in this case.
- # Here we follow the conductor implementation and return the whole list.
- # This may not be the right thing to do. It might be better to throw an error
+ # We used to follow the conductor implementation and return the whole list.
+ # but that is definitely not the right thing to do in our context.
+ # Exit with an error
+ exit 404;
}
}
Index: branches/czw_branch/20160809/DataStoreServer/scripts/dsrootindex
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/scripts/dsrootindex (revision 39651)
+++ branches/czw_branch/20160809/DataStoreServer/scripts/dsrootindex (revision 39719)
@@ -14,7 +14,24 @@
my $PS_EXIT_CONFIG_ERROR = 3;
+# set this variable to redirect listings of the root datastore directory
+# to the password protected php page on the postage stamp server web site.
+# Since the data store is restricted by IP address now this is no longer necessary
+my $redirect_root_to_pstamp = 0;
+if ($redirect_root_to_pstamp) {
+
+ print '
+
+
+
+
+
+ ';
+
+ exit 0;
+}
+
my $dbh = getDBHandle();
-my $stmt = $dbh->prepare("SELECT * FROM dsProduct");
+my $stmt = $dbh->prepare("SELECT * FROM dsProduct ORDER BY type");
$stmt->execute();
@@ -36,4 +53,5 @@
$row->{type}, $row->{description};
+ # XXX EAM : security by obfuscation
print $line;
}
Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/dsgetindex
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/cgi/dsgetindex (revision 39651)
+++ branches/czw_branch/20160809/DataStoreServer/web/cgi/dsgetindex (revision 39719)
@@ -20,4 +20,6 @@
my $PS_EXIT_CONFIG_ERROR = 3;
my $PS_EXIT_DATA_ERROR = 5;
+
+my $redirect_root_to_pstamp = 0;
my $uri = shift;
@@ -132,4 +134,5 @@
print @$stdout_buf;
}
+ $error_code = 0 if !defined $error_code;
} else {
if (0 && $html_mode) {
@@ -199,12 +202,24 @@
# return link
- print a({-href=>"./index.txt"}, "Text Version");
+ if (!$redirect_root_to_pstamp || $program ne "dsrootindex") {
+ print a({-href=>"./index.txt"}, "Text Version");
+ }
if ($#path >= 1) {
- print " \n";
-
- my $up = join('/', @path[0..$#path-1]);
- print a({-href=>".."}, "Up to $up");
-
+ # including the up links in the data store display causes
+ # wget -r to follow them up which makes a mess
+ # Turn them off if the user agent looks like wget
+ my $display_up_links = (lc($ENV{HTTP_USER_AGENT}) =~ /wget/) ? 0 : 1;
+ if ($display_up_links) {
+ print " \n";
+ my $up = join('/', @path[0..$#path-1]);
+ if ($redirect_root_to_pstamp and $program eq "dsprodindex") {
+ print a({-href=>"http://pstamp.ipp.ifa.hawaii.edu/dsroot.php"}, "Up to $up");
+ } elsif ($program eq 'dsrootindex') {
+ # no up link in dsroot
+ } else {
+ print a({-href=>".."}, "Up to $up");
+ }
+ }
}
print "\n";
@@ -244,23 +259,47 @@
if ($nolink) {
print $toks[0];
- }
-
- else {
- # assumes id is always first field
- my $wpath = "./$toks[0]";
-
- # if this is a file request, use a download cgi
- if ($wpath =~ /\.fits\s*$/) {
- # This doesn't work through the proxy
- # The /ds-cgi gets tried on alala
- # $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/'
+ } else {
+ # First column is link to an entity down one level in data store (product, fileset, file)
+ # assumes id is always first field
+ my $wpath = "./$toks[0]";
+
+ # if this is a file request, use a download cgi
+ my $fits = 0;
+ if ($wpath =~ /\.fits\s*$/) {
+ # This doesn't work through the proxy
+ # The /ds-cgi gets tried on alala
+ # $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/'
+ $fits = 1;
+ }
+
+ if ($fits) {
+ print a({-href=>$wpath, -type=>'image/x-fits'}, $toks[0]);
+ } else {
+ print a({-href=>$wpath}, $toks[0]);
+ }
+
+ if ($program eq 'dsrootindex') {
+ # for root listing make most recent fileset a link as well unless there are no filesets
+ my $fileset = $toks[1];
+ $fileset =~ s/^\s+//;
+ $fileset =~ s/\s+$//;
+ if ($fileset ne 'none') {
+ print "";
+ print "<$celltag>";
+ # drop the product from the list of tokens
+ my $product = shift @toks;
+ # elmiminate any whitesapace
+ $product =~ s/^\s+//;
+ $product =~ s/\s+$//;
+ print a({-href=>"./$product/$fileset"}, $fileset);
}
-
- print a({-href=>$wpath, -type=>'image/x-fits'}, $toks[0]);
- }
+ }
+ }
+ shift @toks;
print "";
- foreach my $val (@toks[1..$#toks]) {
+ # foreach my $val (@toks[1..$#toks]) {
+ foreach my $val (@toks) {
print "<$celltag>";
print $val;
Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/findskycalcmf.pl
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/cgi/findskycalcmf.pl (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/cgi/findskycalcmf.pl (revision 39719)
@@ -0,0 +1,167 @@
+#!/usr/bin/env perl
+
+# findsskycalcmf.pl Locate a skycal cmf file for a skycell.
+
+use strict;
+use warnings;
+
+use DBI;
+use File::Basename;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my $verbose;
+
+my $skycal_id;
+my $skycell_id;
+my $tess_id;
+my $filter;
+my $release;
+
+# NOTE: We do not use the ipp configuration to simplify running from a CGI script
+my $dbname = "gpc1";
+my $dbserver = "scidbm";
+my $dbuser = "ippuser";
+my $dbpassword = "ippuser";
+
+
+GetOptions(
+ 'skycell_id=s' => \$skycell_id,
+ 'tess_id=s' => \$tess_id,
+ 'filter=s' => \$filter,
+ 'skycal_id=s' => \$skycal_id,
+ 'release=s' => \$release,
+ 'dbname=s' => \$dbname,
+ 'verbose|v' => \$verbose,
+) or pod2usage (2);
+
+pod2usage( -msg => "Required options: (--tess_id and --skycell_id and --filter) or --skycal_id",
+ -exitval => 3)
+ unless $skycal_id or ($filter and $tess_id and $skycell_id);
+
+my ($filename, $cmf);
+if (defined $skycal_id) {
+ ($filename, $cmf) = find_cmf_by_skycal_id($skycal_id)
+} elsif (defined $tess_id and defined $skycell_id and defined $filter) {
+ ($filename, $cmf) = find_cmf_by_skycell($tess_id, $skycell_id, $filter, $release);
+} else {
+ # should have been trapped above
+ die "not enough parameters\n";
+}
+
+
+# FIX THESE PATHS!
+# set up the environment
+$ENV{NEB_SERVER}="http://nebserver.ipp.ifa.hawaii.edu:80/nebulous";
+$ENV{PERL5LIB} .= ":/home/panstarrs/bills/psconfig/debug.lin64/lib:/home/panstarrs/bills/psconfig/debug.lin64/lib/perl5";
+
+my $neb_locate = "/home/panstarrs/bills/psconfig/debug.lin64/bin/neb-locate -p";
+if ($cmf) {
+ my $resolved = `$neb_locate $cmf`;
+ if (!$resolved) {
+ print STDERR "failed to resolve $cmf\n";
+ exit 2 if (!$resolved)
+ }
+ # HERE IS THE OUTPUT FOR A SUCCESSFUL LOOKUP
+ print "$filename $resolved";
+} else {
+ # the find function prints an appropriate error if it fails
+ exit 2;
+}
+
+exit 0;
+
+sub open_db {
+ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+ my $dbh = DBI->connect($dsn, $dbuser, $dbpassword)
+ or die "Cannot connect to database.\n";
+
+ return $dbh;
+}
+
+sub find_cmf_by_skycal_id {
+ my $skycal_id = shift;
+
+ my $query = "SELECT * from skycalRun JOIN skycalResult USING(skycal_id) WHERE skycal_id = $skycal_id";
+
+ my $dbh = open_db();
+ my $stmt = $dbh->prepare($query);
+ if (!$stmt->execute()) {
+ print "DBI error\n";
+ return undef;
+ }
+ my $results = $stmt->fetchrow_hashref();
+ if (!$results) {
+ print "results for skycal_id: $skycal_id not found\n";
+ return undef;
+ }
+
+ if ($results->{fault}) {
+ my $fault = $results->{fault};
+ $fault .= " (GONE)" if $fault == 26;
+ print "camRun $skycal_id has fault $fault\n";
+ return undef;
+ }
+
+ my $path_base = $results->{path_base};
+
+ my $cmf = $path_base . ".cmf";
+
+ return parse_filename($cmf);
+}
+
+sub find_cmf_by_skycell {
+ my ($tess_id, $skycell_id, $filter, $release) = @_;
+
+ my $where_clause = " (tess_id = '$tess_id' AND skycell_id = '$skycell_id' and filter LIKE '$filter%')";
+
+ # prioritize by release priority values
+ my $order = "priority DESC";
+ if ($release) {
+ $where_clause .= " AND ippRelease.release_name = '$release'";
+ }
+
+ my $query = "SELECT skycal_id, skycalResult.quality, skycalResult.fault, skycalResult.path_base, priority\n"
+ . " FROM ippRelease JOIN relStack using(rel_id) JOIN skycalResult using(skycal_id) "
+ . " JOIN stackRun USING(stack_id, tess_id, skycell_id, filter)"
+ . " WHERE $where_clause \n ORDER by $order limit 1;";
+
+ print STDERR "$query\n" if $verbose;
+
+ my $dbh = open_db();
+ my $stmt = $dbh->prepare($query);
+ if (!$stmt->execute()) {
+ print "DBI error\n";
+ return undef;
+ }
+ my $results = $stmt->fetchrow_hashref();
+ if (!$results) {
+ print "results not found\n";
+ return undef;
+ }
+ my $skycal_id = $results->{skycal_id};
+ if ($results->{fault}) {
+ my $fault = $results->{fault};
+ if ($fault == 26) {
+ $fault .= " (GONE)";
+ }
+ print "skycalResult $skycal_id has fault $fault\n";
+ return undef;
+ } elsif ($results->{quality} != 0) {
+ print "skycalRun $skycal_id has poor quality $results->{quality}\n";
+ return undef;
+ }
+
+
+ my $path_base = $results->{path_base};
+
+ my $cmf = $path_base . ".cmf";
+ return parse_filename($cmf);
+}
+
+sub parse_filename {
+ my $cmf = shift;
+ my $filename = basename($cmf);
+ return ($filename, $cmf);
+}
Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/findsmf.pl
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/cgi/findsmf.pl (revision 39651)
+++ branches/czw_branch/20160809/DataStoreServer/web/cgi/findsmf.pl (revision 39719)
@@ -35,8 +35,8 @@
# NOTE: We do not use the ipp configuration to simplify running from a CGI script
-my $dbname = "XXX";
-my $dbserver = "XXX";
-my $dbuser = "XXX";
-my $dbpassword = "XXX";
+my $dbname = "gpc1";
+my $dbserver = "scidbm";
+my $dbuser = "ippuser";
+my $dbpassword = "ippuser";
@@ -77,4 +77,5 @@
exit 2 if (!$resolved)
}
+ # HERE IS THE OUTPUT FOR A SUCCESSFUL LOOKUP
print "$filename $resolved";
} else {
Index: branches/czw_branch/20160809/DataStoreServer/web/cgi/listsmfs.pl
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/cgi/listsmfs.pl (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/cgi/listsmfs.pl (revision 39719)
@@ -0,0 +1,172 @@
+#!/usr/bin/env perl
+
+# listsmf.pl Print a listing of information about smf files for processing meeting
+# certain selection criteria.
+
+use strict;
+use warnings;
+
+use DBI;
+use File::Basename;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my $verbose;
+
+my $cam_id;
+my $exp_id;
+my $exp_name;
+my $filter;
+my $label;
+my $data_group;
+my $dateobs_min;
+my $dateobs_max;
+my $release;
+my $getMagicked;
+
+# NOTE: We do not use the ipp configuration to get this information in order to simplify running from a CGI script
+# XXX: probably should be using some local file though.
+# XXX: take a look at my module dsdbh which has the function getDBHandle();
+#
+my $dbname = "gpc1";
+my $dbserver = "scidbm";
+my $dbuser = "ippuser";
+my $dbpassword = "ippuser";
+
+
+my $defaultToToday = 1;
+
+GetOptions(
+ 'label=s' => \$label,
+ 'data_group=s' => \$data_group,
+ 'dateobs_min=s' => \$dateobs_min,
+ 'dateobs_max=s' => \$dateobs_max,
+ 'release=s' => \$release,
+ 'filter=s' => \$filter,
+ 'dbname=s' => \$dbname,
+
+ 'exp_name=s' => \$exp_name,
+ 'exp_id=s' => \$exp_id,
+ 'cam_id=s' => \$cam_id,
+ 'verbose|v' => \$verbose,
+) or pod2usage (2);
+
+
+# if data_group is supplied we don't need to default the date
+if ($data_group) {
+ $defaultToToday = 0;
+}
+
+# XXX: since this is going to be used in a cgi script we should carefully vet the supplied
+# date format.
+if (!$dateobs_min) {
+ # default to today
+ if ($dateobs_max) {
+ print "Must supply begin date if max date is supplied.\n";
+ exit 2;
+ }
+ if ($defaultToToday) {
+ my ($day, $month, $year) = (gmtime)[3,4,5];
+
+ $dateobs_min = sprintf "%4d-%02d-%02dT00:00:00Z", 1900+$year, 1+$month, $day;
+ print "$dateobs_min\n" if $verbose;
+ }
+} else {
+ # add times if not supplied
+ if (!($dateobs_min =~ 'T')) {
+ $dateobs_min .= 'T00:00:00Z';
+ print "$dateobs_min\n" if $verbose;
+ }
+ if (!($dateobs_max =~ 'T')) {
+ $dateobs_max .= 'T23:59:59Z';
+ print "$dateobs_max\n" if $verbose;
+ }
+}
+
+
+my $dbh = open_db();
+
+my $query = 'SELECT cam_id, camRun.label, camRun.state, quality, release_name, camProcessedExp.path_base,'
+ . ' exp_name, exp_id, filter, dateobs, comment'
+ . ' FROM camRun JOIN camProcessedExp USING(cam_id) JOIN chipRun using(chip_id) JOIN rawExp USING(exp_id)'
+ . ' LEFT JOIN relExp USING(cam_id, exp_id) LEFT JOIN ippRelease USING(rel_id)'
+ . " WHERE ";
+
+my $and = '';;
+if ($data_group) {
+ $query .= " camRun.data_group = '$data_group'";
+ $and = ' AND';
+}
+
+if ($dateobs_min) {
+ $query .= "$and dateobs > '$dateobs_min'";
+}
+
+if ($dateobs_max) {
+ $query .= " AND dateobs <= '$dateobs_max'";
+}
+
+# Should I use "LIKE" qualifiers here?
+if ($label) {
+ $query .= " AND camRun.label = '$label'";
+}
+
+if ($release) {
+ if ((uc($release) eq 'NONE') or (uc($release) eq 'NULL')) {
+ $query .= " AND release_name IS NULL";
+ } else {
+ $query .= " AND release_name = '$release'";
+ }
+}
+
+if ($filter) {
+ # if single character filter is supplied append wild card
+ # character to the filter string
+ if (length($filter) == 1) {
+ $filter .= '%';
+ }
+ $query .= " AND filter LIKE '$filter'";
+}
+
+print "$query\n" if $verbose;
+
+my $statement = $dbh->prepare($query);
+
+$statement->execute();
+
+my $numRows = 0;
+
+# Print Header line as a comment.
+# XXX: rather than fine tuning the spacing on this I should have used the
+# same format line that I use for the data below.
+
+print
+'#cam_id smf_name quality state exp_name label release filter obs_date obs_time comment' . "\n";
+#1780291 o7616g0016o.1131500.cm.1780291.smf 0 full o7616g0016o QUB.nightlyscience none z.00000 2016-08-16 05:57:32 'Transient PS16cgx z band cell 45 65'
+
+while ( my $row = $statement->fetchrow_hashref() ) {
+ $numRows++;
+
+ my $smf_name = basename($row->{path_base}) . '.smf';
+ my $release_name = $row->{release_name};
+ $release_name = 'none' unless $release_name;
+
+ printf "%ld %s %5d %-7s %s %-20s %-14s %-11s %s '%s'\n",
+ $row->{cam_id}, $smf_name, $row->{quality}, $row->{state},
+ $row->{exp_name}, $row->{label}, $release_name, $row->{filter},
+ $row->{dateobs}, $row->{comment};
+}
+
+print "$numRows found\n" if $verbose;
+
+exit 0;
+
+sub open_db {
+ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+ my $dbh = DBI->connect($dsn, $dbuser, $dbpassword)
+ or die "Cannot connect to database $dbname.\n";
+
+ return $dbh;
+}
+
Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/getsmf.html
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/getsmf.html (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/getsmf.html (revision 39719)
@@ -0,0 +1,117 @@
+
+
+IPP SMF Access Tools
+
+
+
+
+Access to IPP smf files
+
+
+Go back up.
+
+
+getsmf
+
+IPP smf files contain the results of camera stage processing.
+They are FITS format files containg tables with the photometric and
+astronometric measurements for the
+sources detected in a single exposure.
+
+
+getsmf.php is a tool that may be used to retrieve an IPP smf file
+given certain parameters.
+
The url of the tool is
+ http://misc.ipp.ifa.hawaii.edu/getsmf.php
+
+When this page is accessed without parameters it simply displays an
+error message.
+With valid parameters the tool returns the contents of the selected
+smf file.
+
+Note: Obviously displaying the contents of a FITS file in a web browser is
+not very useful.
+getsmf.php is intended to be accessed directly using HTTP.
+It is easy to write scripts that use the command line tools curl or wget
+to retrieve smf files using this tool.
+
+For example, the following two equivalent unix commands copy the contents of
+the smf containing the results from the cameraRun with cam_id=1780283)
+to a local files:
+
+ curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -o 1780283.smf
+
+ wget 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -O 1780283-wget.smf
+
+
+The previous method allows the user to control the name of the local copy.
+If one would like to use the original file name supplied by the server
+the commands may be changed to:
+
+ curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -O --remote-header-name
+
+ wget 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' --content-disposition
+
+
+In this example the name of the downloaded file is "o7616g0008o.1131492.cm.1780283.smf".
+
+Note: The quote characters in these commands are needed to prevent
+the '?' characters from being interpreted by user's shell.
+
+parameters
+There are a few parameters that may be used to select specific smf files.
+Parameters are supplied by appending them to the base url following
+a '?' character.
+
+If multiple parameters are needed
+they should be separated by a '&' character.
+
+cam_id=value
+The examples above demonstrated selection by cam_id. When a
+cam_id is supplied any other parameters supplied are ignored.
+
+exp_name=value
+Selects an smf file for an exposure with the given name.
+For example supplying the parameter: exp_name=o7616g0008o
+gets the same result as the examples listed above. (As of the time
+of this writing).
+
+Note that exposures may be processed multiple times. If no other selection
+parameters are found, the smf for the most recent processing is selected.
+release=value
+When used with exp_name, the release parameter specifies the
+ipp release (processing version) for the smf file.
+For example: exp_name=o5275g0401o&release=3PI.PV3
retrieves the
+smf processed for exposure o5275g0401o in 3PI.PV3.
+If the release were changed to 3PI.nightly
+the smf from the original nightly procecessing would be retrieved.
+(At the time of this writing, if the release parameter were omitted
+the 3PI.PV3 version would also be retrieved since it is currently
+the latest processing.)
+data_group=value
+This is a rarely used parameter.
+With exp_name, chooses smfs from camRuns processed as part of
+the supplied data group.
+
+Errors and Exceptions
+If an smf matching the supplied parameters is not found in the IPP, a message
+is included in the response. The http request succeeds.
+
+After a user retrieves a file in this manner it should be checked
+to insure that it
+is actually a FITS format file.
+
+For example the following getsmf request asks for a non-existent smf:
+
+ curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=99999999999' -o test.smf
+
+After this command succeeds, the file test.smf is not a FITS file.
+Instead it contains text. This fact and its contents may be found
+with the following
+commands.
+
+ $ file test.smf
+ test.smf: ASCII text
+ $ cat test.smf
+ Could not find smf for cam_id: 99999999999.
+
Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/index.html
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/index.html (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/index.html (revision 39719)
@@ -0,0 +1,29 @@
+
+
+IPP-Misc: SMF Access
+
+
+
+
+IPP-Misc: SMF Access
+
+
+This page contains links to information about accessing
+IPP SMF files
+which contain the results of IPP camera stage processing.
+
+These files are FITS format files containing multiple binary table extension
+sections. These which contain the photometric and astronometric measurements
+for the sources detected in a single Pan-STARRS exposure.
+
+These files are the inital inputs to the PS calibration process.
+
+The following links describe how to use the IPP-MISC SMF Access tools
+to discover and retrive SMF files.
+
+
+
Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/listsmfs.html
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/listsmfs.html (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/listsmfs.html (revision 39719)
@@ -0,0 +1,141 @@
+
+
+
+listsmf
+
+
+
+
+
+listsmfs
+
+
+Go back up.
+
+listsmf.php is a tool that may be used to retrieve certain information
+about SMF files from the IPP database.
+
The url of the tool is
+http://misc.ipp.ifa.hawaii.edu/listsmfs.php
+
+
+The list returned is controlled by parameters supplied
+as part of the http request used to access the page.
+
+By default listsmfs displays the list of
+smfs corresponding to the PS1 science exposures collected on
+the current date (UTC). The output may be fine tuned using parameters.
+The use of parameters is discussed below.
+
+Here is some sample output.
+
+#cam_id smf_name quality state exp_name label release filter obs_date obs_time comment
+1783121 o7622g0062o.1134677.cm.1783121.smf 0 full o7622g0062o OSS.nightlyscience SSS.nightly i.00000 2016-08-22 07:43:12 'OSSR.R19S6.16.Q.i ps1_32_2599 visit 1'
+1783120 o7622g0061o.1134678.cm.1783120.smf 0 full o7622g0061o OSS.nightlyscience SSS.nightly i.00000 2016-08-22 07:42:16 'OSSR.R19S6.16.Q.i ps1_32_2493 visit 1'
+1783122 o7622g0063o.1134679.cm.1783122.smf 0 full o7622g0063o OSS.nightlyscience SSS.nightly i.00000 2016-08-22 07:44:08 'OSSR.R19S6.16.Q.i ps1_32_2614 visit 1'
+... and so on.
+
+
+
+The first line, which begins with the "comment" character '#', is a
+header line. This line lists the names of the columns shown
+in the subsequent lines.
+
+If no exposures are found that match the supplied parameters (and their
+defaults) only the header line is displayed.
+
The first 4 columns are the most important. They are very useful
+for constructing requests to retrieve individual SMFs using
+getsmf.php.
+
+
List of Columns
+
+cam_id
+The IPP camera run identifier.
+smf_name
+The base name of the SMF file for the exposure (if any).
+quality
+The quality value for the camera processing. A non-zero value indicates
+a problem with the processing. For example: unsuccessful astrometric
+solution. In the event of a run with non zero quality
+no SMF file will be available.
+state
+The current state of the camera run.
+Runs that have completed processing will have
+the state "full". SMF files will not be available for runs with any
+state other than full. Runs with state "new" may finish processing
+at a later time.
+exp_name
+The exposure name of the source exposure. (Also known as "Frame Name"
+in PSPS and other contexts.)
+label
+The processing label for the camRun.
+release
+The Release that contains the exposures. Examples might be
+3PI.nightly or SSS.nightly.
+If the exposure has not been assigned to a release this column will have
+the value "none".
+filter
+The filter used by the camera for the exposure.
+obs_date
+Date of the observation (UTC).
+obs_time
+Time of the observation (UTC).
+comment
+The comment string for the exposure. Note that single quote characters
+have been added around the comment string as delimiters to
+facilitate parsing.
+
+
+Parameters
+
+As mentioned above, listsmfs gives a list of the
+exposures for the current (UTC) day by default.
+Parameters may be used to
+control the selection. Parameters are provided by appending
+an HTML query string to the request URL.
+
+For example the sample output shown above was generated by a request
+with the parameter string '?release=SSS.nightly' appended to the URL
+(on the date that this page was written of course).
+
+If multiple parameters are to be supplied they should be separated
+by a '&' character. For example '?release=SSS.nightly&filter=r'
+would list exposures with the named release in the r band filter..
+
+Here is the list of the parameters.
+
+
+
date_min and date_max
+The date/time limits for observations to be included by the listing.
+The format for the datetime is
+
+ YYYY-MM-DDTHH:MM:SS
+where the '-', 'T', and ':' characters are literal and the other characters are
+numeric values for year, month, day, hours, minutes, and seconds.
+
+For example 'date_min=2016-08-22T08:00:00&date_max=2016-08-22T09:00:00'
+would select exposures observed between 0800 UTC and before 0900 UTC
+on Augst 22.
+
+Notes on date limits.
+The time portion may be omitted. For example the parameter pair
+'date_min=2016-08-21&date_max=2016-08-22' will cause all exposures observed
+on 2016-08-21 or 2016-08-22 to be listed.
+
+If date_min is not supplied, any date_max value supplied is silently ignored.
+Instead page uses the defaults for the date parameter: exposures for the current date are listed (if any).
+
+
filter
+Only exposures with the corresponding filter are listed. If a single
+character value is supplied for filter, exposures from all filters that
+begin with that letter are listed.
+
+label
+The list is restricted to camRuns with the supplied label.
+
+release
+The list is restrictred to exposures with supplied release name.
+
+data_group
+The list is restrictred to camRuns with the supplied data group.
+Note: This parameter is expected to be rarely used.
+
Index: branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/scripting.html
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/scripting.html (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/doc/smfs/scripting.html (revision 39719)
@@ -0,0 +1,151 @@
+
+
+Scripting IPP SMF Access
+
+
+
+
+Scripting IPP SMF Access
+
+
+Go back up.
+
+
+The process of retrieving SMF files is straightforward.
+
+- Discover the list of files of interest.
+
- Build valid commands to retrieve them from the IPP to local storage.
+
- Check the results for validity.
+
- Start working on your science :).
+
+
+
+This page is primarily concerned with the first two items on this list.
+
+
+The two pages listsmfs.php and getsmf.php have a relatively straightforward
+interface. UNIX command line tools such as curl or wget may be used to
+access the interface.
+
+
Sample PERL script
+
+The perl script below gives an example of how to use curl to get
+a list of the current day's smf files from listsmf.php and print curl
+commands that may be used to download the files.
+
+
+ #!/usr/bin/env perl
+
+ use strict;
+ use warnings;
+
+ my $verbose = 0;
+
+ my $misc_url = 'http://misc.ipp.ifa.hawaii.edu';
+ my $list_url = "$misc_url/listsmfs.php";
+ my $get_url = "$misc_url/getsmf.php";
+
+ my $silent = ' --silent';
+ if ($verbose) {
+ $silent = '';
+ }
+
+ my $list_cmd = "curl $silent http://misc.ipp.ifa.hawaii.edu/listsmfs.php";
+
+ my $output = `$list_cmd`;
+
+ if ($? != 0) {
+ my $rc = $?;
+ print STDERR "Ooops something went wrong with the command $list_cmd\n";
+ print STDERR "command returned $rc\n";
+ exit $rc >> 8;
+ }
+
+ my @lines = split "\n", $output;
+
+ foreach my $line (@lines) {
+ # skip the header line and any others that begin with '#'
+ next if $line =~ '#';
+
+ # split the line at the single quote character that preceeds the
+ # comment string
+ my ($left, $right) = split "'", $line;
+
+ # the right side of this split is the comment string ...
+ my $comment = $right;
+
+ # ... and the left side of this split contains a string
+ # with the other parameters.
+ # The other parameters are separated by whitespace
+
+ # NOTE: One might have thought we could parse the header string
+ # and find the column names but that implies more than the
+ # listsmf "interface" commits to at this time.
+
+ my ($cam_id, $smf_name, $quality, $state, $exp_name, $label, $release, $filter, $obs_date, $obs_time) = split " ", $left;
+
+ if ($quality != 0) {
+ print STDERR "camRun $cam_id had bad quality $quality\n";
+ next;
+ }
+
+ if ($state ne 'full') {
+ print STDERR "camRun is in state $state\n";
+ next;
+ }
+
+ # generate and print out a curl command that could be used to
+ # download the smf file to the current directory
+ my $this_cmd = "curl $silent '$get_url?cam_id=$cam_id' --output $smf_name";
+
+ print "$this_cmd\n";
+ }
+ # all done
+
+
+When executed the resulting output will look something like this:
+
+ # 1783100 o7622g0010o.1134626.cm.1783100.smf 0 ICECUBE-160731A.01 i band dither 1 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783100' --output o7622g0010o.1134626.cm.1783100.smf
+ # 1783101 o7622g0011o.1134627.cm.1783101.smf 0 ICECUBE-160731A.01 i band dither 2 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783101' --output o7622g0011o.1134627.cm.1783101.smf
+ # 1783102 o7622g0012o.1134628.cm.1783102.smf 0 ICECUBE-160731A.01 i band dither 3 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783102' --output o7622g0012o.1134628.cm.1783102.smf
+ # 1783103 o7622g0013o.1134629.cm.1783103.smf 0 ICECUBE-160731A.01 i band dither 4 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783103' --output o7622g0013o.1134629.cm.1783103.smf
+ # 1783104 o7622g0014o.1134630.cm.1783104.smf 0 ICECUBE-160731A.01 i band dither 5 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783104' --output o7622g0014o.1134630.cm.1783104.smf
+
+
+
+There are two lines for each exposure. The first (which begins with
+ a '#' character) lists prints selected metadata
+for the smf and its source exposure. The second line gives
+a curl command that would copy the file to the user's current directory.
+
Note, while not suggested, one could take the output of listsmfs.php
+and source it into a UNIX shell to execute the commands.
+
+This script demonstrates how to
+
+- Use curl to get a list of the smfs for the current day's exposures
+
- Split the lines of the output into its columns (including the comment string)
+
- Examining the state and quality values to determine whether or not an SMF exists for the exposure
+
- Printing a curl command line which when exectuted with copy the SMF to the current directory
+
+
+This simple script shows the most important steps necessary for successful
+smf retrieval.
+
+
What about errors?
+
+For the sake of speed we chose to not include any information that
+might be useful for detecting errors in the download process. The database
+does contain anything (a checksum for example)
+which could be used to gain confidence that a file was transmitted
+successfully.
+
+
+In practice so far this has not shown to be an issue. getsmf.php has been
+used to download hundreds of thousands of smf files. When users have
+detected problem files, simply trying the download again seems to correct
+the problem.
Index: branches/czw_branch/20160809/DataStoreServer/web/php/getskycalcmf.php
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/php/getskycalcmf.php (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/php/getskycalcmf.php (revision 39719)
@@ -0,0 +1,132 @@
+$command\n
";
+ }
+ $command = escapeshellcmd($command);
+ $output = array();
+
+ exec($command, $output, $command_status);
+
+ if ($command_status == 0) {
+ // we only expect one line of output
+ $len = count($output);
+ if ($len == 1) {
+ list($filename, $pathname) = explode(" ", $output[0]);
+ if ($filename && $pathname) {
+ if (!$debug) {
+ $gotFile = 1;
+ } else {
+ echo "$filename $pathname\n";
+ echo "$command\n";
+ }
+ }
+ } else {
+ echo "
unexpected output from $command: $output[0] $output[1]\n";
+ }
+ } else {
+ if ($debug) {
+ echo "
command failed $command_status\n";
+ }
+ }
+} else {
+ // something went wrong with the paramters. Error message produced below.
+}
+
+if ($gotFile) {
+ if ($rvar_list) {
+ echo "cmf : $filename\n";
+ // echo "
path: $pathname\n
";
+ } else {
+ // All systems are go. Time to write the output.
+ // First set up the header
+ header('Content-type: application/fits');
+ header("Content-Disposition: attachment; filename=\"$filename\"");
+ $filesize = filesize($pathname);
+ header("Content-Length: $filesize");
+ header('Expires: now');
+
+ // copy the contents of the file to the stream
+ readfile($pathname);
+ }
+} else {
+ // XXX: Figure out how to stop wget from redirecting these error
+ // messages to the nasty filename
+ $error_string="Could not find cmf";
+ if ($rvar_skycal_id) {
+ $error_string .= " for skycal_id: $rvar_skycal_id";
+ } elseif ($rvar_skycell_id and $rvar_tess_id and $rvar_filter) {
+ $error_string .= " for tess_id $rvar_tess_id skycell_id: $rvar_skycell_id filter: $rvar_filter";
+ } else {
+ $error_string .= ".
Not enough parameters supplied";
+ }
+
+ echo "$error_string.\n";
+}
+
+function getVar($var) {
+ if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+ $rvar = $_POST[$var];
+ } else {
+ $rvar = $_GET[$var];
+ }
+ $rvar = stripslashes($rvar);
+ $rvar = htmlentities($rvar);
+ $rvar = strip_tags($rvar);
+ return $rvar;
+}
+
+
+if ($list) {
+ // print lots of information from the PHP installation
+ // phpinfo(-1);
+
+ // print the most useful variables
+ // phpinfo(32);
+}
+
+?>
Index: branches/czw_branch/20160809/DataStoreServer/web/php/getsmf.php
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/php/getsmf.php (revision 39651)
+++ branches/czw_branch/20160809/DataStoreServer/web/php/getsmf.php (revision 39719)
@@ -5,16 +5,7 @@
// To just list use wget 'http://ippc17/ipp-misc/getsmf.php?exp_name=o5732g0036o&list=1'
-// Only configuration variable here, the location of the cgi script to find smf files
-$command = "/data/ippc17.0/datastore/ds-cgi/findsmf.pl";
-
-$rvar_exp_name = "";
-$rvar_cam_id = "";
-$rvar_data_group = "";
-$rvar_list = 0;
$error_string = "";
$debug = 0;
-
-# import_request_variables("g", "rvar_");
$rvar_exp_name = getVar('exp_name');
@@ -24,4 +15,6 @@
$rvar_data_group = getVar('data_group');
$rvar_list = getVar('list');
+
+$command = "/data/ippc17.0/datastore/ds-cgi/findsmf.pl";
# cam_id takes priority
@@ -38,5 +31,8 @@
if ($command) {
if ($rvar_release) {
- $command .= " --release $release";
+ if ($rvar_release == "3PI.GR1") {
+ $rvar_release = "3PI.PV1";
+ }
+ $command .= " --release $rvar_release";
}
if ($rvar_data_group) {
@@ -69,9 +65,9 @@
}
} else {
- echo "unexpected output from $command: $output[0] $output[1]\n";
+ echo "
unexpected output from $command: $output[0] $output[1]\n";
}
} else {
if ($debug) {
- echo "command failed $command_status\n";
+ echo "
command failed $command_status\n";
}
}
@@ -92,7 +88,7 @@
readfile($pathname);
} else {
- echo "smf file name is $filename
\nfile is $pathname\n";
+ echo "smf : $filename\n";
+ # echo "path: $pathname\n";
}
-
} else {
// XXX: Figure out how to stop wget from redirecting these error
Index: branches/czw_branch/20160809/DataStoreServer/web/php/listsmfs.php
===================================================================
--- branches/czw_branch/20160809/DataStoreServer/web/php/listsmfs.php (revision 39719)
+++ branches/czw_branch/20160809/DataStoreServer/web/php/listsmfs.php (revision 39719)
@@ -0,0 +1,149 @@
+
smf list\n";
+}
+
+if ($debug) {
+ echo "pos_curl: $pos_curl pos_wget: $pos_wget\n";
+}
+
+
+if ($command) {
+ $command = escapeshellcmd($command);
+ if ($debug) {
+ $agent = $_SERVER['HTTP_USER_AGENT'];
+ echo "HTTP_USER_AGENT is: $agent\n
";
+ echo "
$command\n
";
+ }
+ $output = array();
+
+ # RUN the command
+ exec($command, $output, $command_status);
+
+ if ($command_status == 0) {
+ if (!$not_browser) {
+ # when viewing in a web browser without this the newlines get lost
+ echo "";
+ }
+
+ # output the list
+ foreach( $output as $line) {
+ echo "$line\n";
+ }
+ } else {
+ if ($debug) {
+ echo "
command failed $command_status\n";
+ }
+ }
+} else {
+}
+
+
+function getVar($var) {
+ if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+ $rvar = $_POST[$var];
+ } else {
+ $rvar = $_GET[$var];
+ }
+ $rvar = stripslashes($rvar);
+ $rvar = htmlentities($rvar);
+ $rvar = strip_tags($rvar);
+ return $rvar;
+}
+
+
+if ($list) {
+ // print lots of information
+ // phpinfo(-1);
+ print "\n";
+ print "Submitter IP ADDR: $submitter_ip_addr\n";
+ print "
Remote IP ADDR: $remote_ip_addr\n";
+
+ // print the most useful variables
+ phpinfo(32);
+}
+
+?>