);
+
+close F;
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsdbh.pm
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsdbh.pm (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsdbh.pm (revision 41111)
@@ -0,0 +1,20 @@
+# create a database handle based on the configuration given
+# in environment variables
+
+sub getDBHandle {
+ my $dbserver = $ENV{DBSERVER};
+ my $dbuser = $ENV{DBUSER};
+ my $dbpass = $ENV{DBPASSWORD};
+ my $dbname = $ENV{DBNAME};
+
+ die "database enviornment not set up" unless defined($dbserver) and defined($dbuser)
+ and defined($dbpass) and defined($dbname);
+
+ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+ my $dbh = DBI->connect($dsn, $dbuser, $dbpass) or die "Cannot connect to server\n";
+
+ return $dbh;
+}
+
+return 1;
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsfits.cgi
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsfits.cgi (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsfits.cgi (revision 41111)
@@ -0,0 +1,3 @@
+#!/bin/bash
+. ./dsshellconfig
+./dodsfits $*
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsgetindex
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsgetindex (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsgetindex (revision 41111)
@@ -0,0 +1,310 @@
+#!/usr/bin/env perl
+#
+# dsgetindex: Generate response to http GET for a DataStore index
+#
+# The only required argument is the REQUEST_URI.
+# The number of elements in the URI tell us whether we are listing
+# the data store root, a product, or a fileset.
+#
+# if a second argument -html is provided, the output is in HTML format
+# otherwise plain text is used
+#
+# The only thing specific to the ipp used here is the exit codes.
+
+use warnings;
+use strict;
+
+use CGI ':standard';
+use IPC::Cmd 0.36 qw( can_run run );
+
+my $PS_EXIT_CONFIG_ERROR = 3;
+my $PS_EXIT_DATA_ERROR = 5;
+
+my $redirect_root_to_pstamp = 0;
+
+my $uri = shift;
+
+if (!$uri) {
+ warn("need uri");
+ exit ($PS_EXIT_CONFIG_ERROR);
+}
+
+my $DS_ROOT = $ENV{DS_ROOT};
+
+die("DS_ROOT not found in the environment") unless defined($DS_ROOT);
+
+my @path;
+
+my $html_mode;
+my $arg = shift;
+if ($arg && ($arg eq "-html")) {
+ $html_mode = 1;
+}
+
+#
+# split up the uri into componnts.
+#
+# XXX: Find a simpler way to do this
+
+# collapse any multipe slashes in uri
+$uri = File::Spec->canonpath($uri);
+
+# split the directories from the file
+my $directories;
+my $file = "";
+my $query_str;
+
+if ($html_mode) {
+ # there is no file on the url
+
+ # print STDERR "html mode\n";
+
+ ($directories, $query_str) = split/\?/, $uri;
+
+ # used by the pretty printer
+ @path = split(/\//, $directories);
+ # shift off the empty bit
+ shift @path;
+
+} else {
+ my $volume;
+ ( $volume, $directories, $file) = File::Spec->splitpath($uri);
+
+ # find the query string if any
+ # (This is only relevant for fileset queries but we follow conductor and do not enforse that restriction)
+ ($file, $query_str) = split/\?/, $file;
+
+ if ($file ne "index.txt") {
+ warn("unexepected index file name: $file");
+ exit ($PS_EXIT_CONFIG_ERROR);
+ }
+}
+
+if (!$query_str) {
+ $query_str = "";
+}
+
+my @dirs = split /\//, $directories;
+
+# shift out the empty string at left of the first /
+shift @dirs;
+# shift out 'ds'
+shift @dirs;
+
+# now we're left with either nothing, a product, or a product and a fileset
+my $product = shift @dirs;
+my $fileset = shift @dirs;
+
+if (@dirs) {
+ # XXX need to output an error page
+ warn("unexpected extra directories: @dirs in uri: $uri");
+ exit ($PS_EXIT_CONFIG_ERROR);
+}
+
+my $program;
+if ($fileset) {
+ $program = "dsfsindex";
+} elsif ($product) {
+ $program = "dsprodindex";
+ $fileset .= " $query_str";
+} else {
+ $program = "dsrootindex";
+ $fileset = "";
+ $product = "";
+}
+
+my $missing_tools;
+my $index_program = can_run($program) or (warn "Can't find $program" and $missing_tools = 1);
+if ($missing_tools) {
+ #warn("Can't find required tools.");
+ exit ($PS_EXIT_CONFIG_ERROR);
+}
+
+{
+ my $command = "$index_program $product $fileset";
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => 0);
+
+ if ($success) {
+ if ($html_mode) {
+ html_index($stdout_buf);
+ } else {
+ print header('text/plain', '200 OK');
+ print @$stdout_buf;
+ }
+ $error_code = 0 if !defined $error_code;
+ } else {
+ if (0 && $html_mode) {
+ XXX: TODO
+
+ } else {
+ # since we're invoked as a redirect from a text file browsers are expecting
+ # 'text/plain' format
+ print header('text/plain', '404 ERROR');
+# print header('text/plain', '410 GONE');
+# print header('text/plain', '200 OK');
+ print @$stderr_buf;
+ print STDERR @$stderr_buf;
+# exit 410;
+ }
+ }
+ exit $error_code >> 8;
+}
+
+
+sub html_index {
+
+ my $out_array_ref = shift;
+ my $outBuf = join( "", @$out_array_ref);
+
+ #print header('text/plain', '200 OK');
+
+ #print "outputting from html_index\n";
+ #print @$out_buf;
+ pprint_pre();
+ my @lines = split /^/, $outBuf;
+ foreach my $line (@lines) {
+ pprint($line);
+ }
+ pprint_post();
+}
+
+#
+# XXX:
+# quick hacky pretty printer
+# Adapted from Eric's Data Store mock up
+#
+
+#sub print_free_space {
+# use Filesys::Df;
+# # get free space
+# my $ref = df($DS_ROOT);
+# printf 'Free space: %.2f G (%.1f%%)
',
+# $ref->{bfree}/(1024*1024),
+# 100.0*$ref->{bfree}/$ref->{blocks};
+# print "\n";
+#}
+sub pprint_pre {
+
+ print header('text/html', '200 OK');
+ my $name;
+ # note $fileset is not so we look at the name of the index script to determine the level
+ if ($program eq "dsfsindex") {
+ $name = $fileset;
+ } elsif ($program eq "dsprodindex") {
+ $name = "$product";
+ } else {
+ $name = "";
+ }
+ print start_html("IPP Data Store $name");
+
+
+ # return link
+ if (!$redirect_root_to_pstamp || $program ne "dsrootindex") {
+ print a({-href=>"./index.txt"}, "Text Version");
+ }
+
+ if ($#path >= 1) {
+ # including the up links in the data store display causes
+ # wget -r to follow them up which makes a mess
+ # Turn them off if the user agent looks like wget
+ my $display_up_links = (lc($ENV{HTTP_USER_AGENT}) =~ /wget/) ? 0 : 1;
+ if ($display_up_links) {
+ print " \n";
+ my $up = join('/', @path[0..$#path-1]);
+ if ($redirect_root_to_pstamp and $program eq "dsprodindex") {
+ print a({-href=>"http://pstamp.ipp.ifa.hawaii.edu/dsroot.php"}, "Up to $up");
+ } elsif ($program eq 'dsrootindex') {
+ # no up link in dsroot
+ } else {
+ print a({-href=>".."}, "Up to $up");
+ }
+ }
+ }
+ print "\n";
+
+
+ print '';
+}
+
+
+sub pprint_post {
+ print '
';
+ print p();
+
+# print_free_space();
+ print end_html();
+}
+
+
+sub pprint {
+ my $line = shift;
+
+ my $nolink;
+ my $celltag = 'td';
+
+ # assume a comment line means its the header
+ if ($line =~ /^\s*#/) {
+ $celltag = 'th';
+ $nolink = 1;
+ }
+
+ my @toks = split(/\|/, $line);
+
+ print '';
+
+ print "<$celltag>";
+
+ if ($nolink) {
+ print $toks[0];
+ } else {
+ # First column is link to an entity down one level in data store (product, fileset, file)
+ # assumes id is always first field
+ my $wpath = "./$toks[0]";
+
+ # if this is a file request, use a download cgi
+ my $fits = 0;
+ if ($wpath =~ /\.fits\s*$/) {
+ # This doesn't work through the proxy
+ # The /ds-cgi gets tried on alala
+ # $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/'
+ $fits = 1;
+ }
+
+ if ($fits) {
+ print a({-href=>$wpath, -type=>'image/x-fits'}, $toks[0]);
+ } else {
+ print a({-href=>$wpath}, $toks[0]);
+ }
+
+ if ($program eq 'dsrootindex') {
+ # for root listing make most recent fileset a link as well unless there are no filesets
+ my $fileset = $toks[1];
+ $fileset =~ s/^\s+//;
+ $fileset =~ s/\s+$//;
+ if ($fileset ne 'none') {
+ print "";
+ print "<$celltag>";
+ # drop the product from the list of tokens
+ my $product = shift @toks;
+ # elmiminate any whitesapace
+ $product =~ s/^\s+//;
+ $product =~ s/\s+$//;
+ print a({-href=>"./$product/$fileset"}, $fileset);
+ }
+ }
+ }
+ shift @toks;
+
+ print "";
+
+ # foreach my $val (@toks[1..$#toks]) {
+ foreach my $val (@toks) {
+ print "<$celltag>";
+ print $val;
+ print "";
+ }
+
+ print '
';
+}
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsindex.cgi
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsindex.cgi (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsindex.cgi (revision 41111)
@@ -0,0 +1,3 @@
+#!/bin/bash
+. ./dsshellconfig
+dsgetindex $REQUEST_URI -html
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsshellconfig
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsshellconfig (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dsshellconfig (revision 41111)
@@ -0,0 +1,15 @@
+# set up some environment needed by the scripts
+export PERL5LIB=INSERT_VALUE_HERE
+
+# IPC::Cmd::can_run routine doesn't find programs in ds-cgi
+# unless . is in the path
+PATH=.:${PATH}
+
+# setting these here allow the data store to run independnt of the ipp configuration
+export DBSERVER=INSERT_VALUE_HERE
+export DBNAME=INSERT_VALUE_HERE
+export DBPASSWORD=INSERT_VALUE_HERE
+export DBUSER=INSERT_VALUE_HERE
+
+export DS_ROOT=INSERT_VALUE_HERE
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dstxtindex.cgi
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dstxtindex.cgi (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/dstxtindex.cgi (revision 41111)
@@ -0,0 +1,3 @@
+#!/bin/bash
+. ./dsshellconfig
+./dsgetindex $REQUEST_URI;
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/findskycalcmf.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/findskycalcmf.pl (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/findskycalcmf.pl (revision 41111)
@@ -0,0 +1,167 @@
+#!/usr/bin/env perl
+
+# findsskycalcmf.pl Locate a skycal cmf file for a skycell.
+
+use strict;
+use warnings;
+
+use DBI;
+use File::Basename;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my $verbose;
+
+my $skycal_id;
+my $skycell_id;
+my $tess_id;
+my $filter;
+my $release;
+
+# NOTE: We do not use the ipp configuration to simplify running from a CGI script
+my $dbname = "gpc1";
+my $dbserver = "scidbm";
+my $dbuser = "ippuser";
+my $dbpassword = "ippuser";
+
+
+GetOptions(
+ 'skycell_id=s' => \$skycell_id,
+ 'tess_id=s' => \$tess_id,
+ 'filter=s' => \$filter,
+ 'skycal_id=s' => \$skycal_id,
+ 'release=s' => \$release,
+ 'dbname=s' => \$dbname,
+ 'verbose|v' => \$verbose,
+) or pod2usage (2);
+
+pod2usage( -msg => "Required options: (--tess_id and --skycell_id and --filter) or --skycal_id",
+ -exitval => 3)
+ unless $skycal_id or ($filter and $tess_id and $skycell_id);
+
+my ($filename, $cmf);
+if (defined $skycal_id) {
+ ($filename, $cmf) = find_cmf_by_skycal_id($skycal_id)
+} elsif (defined $tess_id and defined $skycell_id and defined $filter) {
+ ($filename, $cmf) = find_cmf_by_skycell($tess_id, $skycell_id, $filter, $release);
+} else {
+ # should have been trapped above
+ die "not enough parameters\n";
+}
+
+
+# FIX THESE PATHS!
+# set up the environment
+$ENV{NEB_SERVER}="http://nebserver.ipp.ifa.hawaii.edu:80/nebulous";
+$ENV{PERL5LIB} .= ":/home/panstarrs/bills/psconfig/debug.lin64/lib:/home/panstarrs/bills/psconfig/debug.lin64/lib/perl5";
+
+my $neb_locate = "/home/panstarrs/bills/psconfig/debug.lin64/bin/neb-locate -p";
+if ($cmf) {
+ my $resolved = `$neb_locate $cmf`;
+ if (!$resolved) {
+ print STDERR "failed to resolve $cmf\n";
+ exit 2 if (!$resolved)
+ }
+ # HERE IS THE OUTPUT FOR A SUCCESSFUL LOOKUP
+ print "$filename $resolved";
+} else {
+ # the find function prints an appropriate error if it fails
+ exit 2;
+}
+
+exit 0;
+
+sub open_db {
+ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+ my $dbh = DBI->connect($dsn, $dbuser, $dbpassword)
+ or die "Cannot connect to database.\n";
+
+ return $dbh;
+}
+
+sub find_cmf_by_skycal_id {
+ my $skycal_id = shift;
+
+ my $query = "SELECT * from skycalRun JOIN skycalResult USING(skycal_id) WHERE skycal_id = $skycal_id";
+
+ my $dbh = open_db();
+ my $stmt = $dbh->prepare($query);
+ if (!$stmt->execute()) {
+ print "DBI error\n";
+ return undef;
+ }
+ my $results = $stmt->fetchrow_hashref();
+ if (!$results) {
+ print "results for skycal_id: $skycal_id not found\n";
+ return undef;
+ }
+
+ if ($results->{fault}) {
+ my $fault = $results->{fault};
+ $fault .= " (GONE)" if $fault == 26;
+ print "camRun $skycal_id has fault $fault\n";
+ return undef;
+ }
+
+ my $path_base = $results->{path_base};
+
+ my $cmf = $path_base . ".cmf";
+
+ return parse_filename($cmf);
+}
+
+sub find_cmf_by_skycell {
+ my ($tess_id, $skycell_id, $filter, $release) = @_;
+
+ my $where_clause = " (tess_id = '$tess_id' AND skycell_id = '$skycell_id' and filter LIKE '$filter%')";
+
+ # prioritize by release priority values
+ my $order = "priority DESC";
+ if ($release) {
+ $where_clause .= " AND ippRelease.release_name = '$release'";
+ }
+
+ my $query = "SELECT skycal_id, skycalResult.quality, skycalResult.fault, skycalResult.path_base, priority\n"
+ . " FROM ippRelease JOIN relStack using(rel_id) JOIN skycalResult using(skycal_id) "
+ . " JOIN stackRun USING(stack_id, tess_id, skycell_id, filter)"
+ . " WHERE $where_clause \n ORDER by $order limit 1;";
+
+ print STDERR "$query\n" if $verbose;
+
+ my $dbh = open_db();
+ my $stmt = $dbh->prepare($query);
+ if (!$stmt->execute()) {
+ print "DBI error\n";
+ return undef;
+ }
+ my $results = $stmt->fetchrow_hashref();
+ if (!$results) {
+ print "results not found\n";
+ return undef;
+ }
+ my $skycal_id = $results->{skycal_id};
+ if ($results->{fault}) {
+ my $fault = $results->{fault};
+ if ($fault == 26) {
+ $fault .= " (GONE)";
+ }
+ print "skycalResult $skycal_id has fault $fault\n";
+ return undef;
+ } elsif ($results->{quality} != 0) {
+ print "skycalRun $skycal_id has poor quality $results->{quality}\n";
+ return undef;
+ }
+
+
+ my $path_base = $results->{path_base};
+
+ my $cmf = $path_base . ".cmf";
+ return parse_filename($cmf);
+}
+
+sub parse_filename {
+ my $cmf = shift;
+ my $filename = basename($cmf);
+ return ($filename, $cmf);
+}
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/findsmf.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/findsmf.pl (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/findsmf.pl (revision 41111)
@@ -0,0 +1,213 @@
+#!/usr/bin/env perl
+
+# findsmf.pl Locate an smf file for an exposure either by cam_id, exp_id, or exp_name
+#
+# If cam_id is used, the smf for that run is selected.
+#
+# If selecting by exposure the camRun is selected by the following rules
+# If a release name is supplied the smf from that release is selected
+# If a data_group is supplied the value is used in the lookup (LIKE comparison)
+# Otherwise all CamRuns for the exposure are considered.
+# If multiple camRuns are found the one selected is the first row in the list formed with
+# the following SQL ordering
+# ORDER BY quality, priority DESC, camRun.cam_id DESC
+# runs with zero quality
+# runs with an entry in the released exposure table ordered by priority
+# the latest camRun
+# For best results use the release tables.
+
+use strict;
+use warnings;
+
+use DBI;
+use File::Basename;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my $verbose;
+
+my $cam_id;
+my $exp_id;
+my $exp_name;
+my $data_group;
+my $release;
+my $getMagicked;
+
+# NOTE: We do not use the ipp configuration to simplify running from a CGI script
+my $dbname = "gpc1";
+my $dbserver = "scidbm";
+my $dbuser = "ippuser";
+my $dbpassword = "ippuser";
+
+
+GetOptions(
+ 'exp_name=s' => \$exp_name,
+ 'exp_id=s' => \$exp_id,
+ 'cam_id=s' => \$cam_id,
+ 'release=s' => \$release,
+ 'data_group=s' => \$data_group,
+ 'dbname=s' => \$dbname,
+ 'verbose|v' => \$verbose,
+) or pod2usage (2);
+
+pod2usage( -msg => "Required options: --exp_name --exp_id or --cam_id",
+ -exitval => 3)
+ unless $exp_name or $exp_id or $cam_id;
+
+my ($filename, $smf, $fallback);
+if (defined $cam_id) {
+ ($filename, $smf, $fallback) = find_smf_by_cam_id($cam_id, $getMagicked);
+} else {
+ ($filename, $smf, $fallback) = find_smf_by_exposure($exp_name, $exp_id, $release, $data_group, $getMagicked);
+}
+
+# set up the environment
+$ENV{NEB_SERVER}="http://nebserver.ipp.ifa.hawaii.edu:80/nebulous";
+$ENV{PERL5LIB} .= ":/home/panstarrs/bills/psconfig/debug.lin64/lib:/home/panstarrs/bills/psconfig/debug.lin64/lib/perl5";
+
+my $neb_locate = "/home/panstarrs/bills/psconfig/debug.lin64/bin/neb-locate -p";
+if ($smf) {
+ my $resolved = `$neb_locate $smf`;
+ if (!$resolved) {
+ print STDERR "failed to resolve $smf\n";
+ if ($fallback) {
+ print STDERR " falling back to $fallback\n";
+ $resolved = `$neb_locate $fallback`;
+ }
+ exit 2 if (!$resolved)
+ }
+ # HERE IS THE OUTPUT FOR A SUCCESSFUL LOOKUP
+ print "$filename $resolved";
+} else {
+ # the find function prints an appropriate error if it fails
+ exit 2;
+}
+
+exit 0;
+
+sub open_db {
+ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+ my $dbh = DBI->connect($dsn, $dbuser, $dbpassword)
+ or die "Cannot connect to database.\n";
+
+ return $dbh;
+}
+
+sub find_smf_by_cam_id {
+ my $cam_id = shift;
+ my $getMagicked = shift;
+
+ my $query = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE cam_id = $cam_id";
+
+ my $dbh = open_db();
+ my $stmt = $dbh->prepare($query);
+ if (!$stmt->execute()) {
+ print "DBI error\n";
+ return undef;
+ }
+ my $results = $stmt->fetchrow_hashref();
+ if (!$results) {
+ print "results for cam_id: $cam_id not found\n";
+ return undef;
+ }
+ my $magicked = $results->{magicked};
+
+ if ($results->{fault}) {
+ my $fault = $results->{fault};
+ $fault .= " (GONE)" if $fault == 26;
+ print "camRun $cam_id has fault $fault\n";
+ return undef;
+ }
+
+ my $path_base = $results->{path_base};
+
+ my $smf = $path_base . ".smf";
+
+ return parse_filename($smf, ($magicked and !$getMagicked));
+}
+
+sub find_smf_by_exposure {
+ my $exp_name = shift;
+ my $exp_id = shift;
+ my $release = shift;
+ my $data_group = shift;
+ my $getMagicked = shift;
+
+ my $where_clause = "";
+ if ($exp_id) {
+ $where_clause .= "exp_id = $exp_id";
+ } elsif ($exp_name) {
+ $where_clause .= "exp_name = '$exp_name'";
+ }
+
+ my $joinrelease = "";
+ # prioritize good quality runs, with higher release priority values, and finally newer cam_id
+ my $order = "quality, priority DESC, camRun.cam_id DESC";
+ my $selectmore = "";
+ if ($release) {
+ $where_clause .= " AND ippRelease.release_name = '$release'";
+ } elsif ($data_group) {
+ # ignore data_group if release is supplied
+ $where_clause .= " AND camRun.data_group LIKE '$data_group'";
+ } else {
+ # since we have no qualifiers on the selection omit faulted runs so that we have a chance
+ # to find a good one
+ $where_clause .= " AND camProcessedExp.fault = 0";
+ }
+
+ my $query = "SELECT cam_id, quality, camRun.state, camProcessedExp.fault, camRun.magicked, camProcessedExp.path_base, IFNULL(priority, 0)\n"
+ . " FROM camRun JOIN camProcessedExp USING(cam_id) join chipRun using(chip_id) join rawExp using(exp_id)\n"
+ . " LEFT JOIN relExp using(exp_id, cam_id) LEFT JOIN ippRelease USING(rel_id)\n"
+ . " WHERE $where_clause \n ORDER by $order limit 1;";
+
+ print STDERR "$query\n" if $verbose;
+
+ my $dbh = open_db();
+ my $stmt = $dbh->prepare($query);
+ if (!$stmt->execute()) {
+ print "DBI error\n";
+ return undef;
+ }
+ my $results = $stmt->fetchrow_hashref();
+ if (!$results) {
+ if ($exp_name) {
+ print "results for exp_name: $exp_name not found\n";
+ } else {
+ print "results for exp_id: $exp_id not found\n";
+ }
+ return undef;
+ }
+ my $cam_id = $results->{cam_id};
+ my $magicked = $results->{magicked};
+ if ($results->{fault}) {
+ my $fault = $results->{fault};
+ if ($fault == 26) {
+ $fault .= " (GONE)";
+ }
+ print "camRun $cam_id has fault $fault\n";
+ return undef;
+ } elsif ($results->{quality} != 0) {
+ print "camRun $cam_id has poor quality $results->{quality}\n";
+ return undef;
+ }
+
+
+ my $path_base = $results->{path_base};
+
+ my $smf = $path_base . ".smf";
+ return parse_filename($smf, ($magicked and !$getMagicked));
+}
+
+sub parse_filename {
+ my $smf = shift;
+ my $getUncensored = shift;
+ my $filename = basename($smf);
+ my $fallback;
+ if ($getUncensored) {
+ my $dirname = dirname($smf);
+ $fallback = $smf;
+ $smf = "$dirname/SR_$filename";
+ }
+ return ($filename, $smf, $fallback);
+}
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/index.txt.template
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/index.txt.template (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/index.txt.template (revision 41111)
@@ -0,0 +1,1 @@
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/listsmfs.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/listsmfs.pl (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/cgi/listsmfs.pl (revision 41111)
@@ -0,0 +1,172 @@
+#!/usr/bin/env perl
+
+# listsmf.pl Print a listing of information about smf files for processing meeting
+# certain selection criteria.
+
+use strict;
+use warnings;
+
+use DBI;
+use File::Basename;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my $verbose;
+
+my $cam_id;
+my $exp_id;
+my $exp_name;
+my $filter;
+my $label;
+my $data_group;
+my $dateobs_min;
+my $dateobs_max;
+my $release;
+my $getMagicked;
+
+# NOTE: We do not use the ipp configuration to get this information in order to simplify running from a CGI script
+# XXX: probably should be using some local file though.
+# XXX: take a look at my module dsdbh which has the function getDBHandle();
+#
+my $dbname = "gpc1";
+my $dbserver = "scidbm";
+my $dbuser = "ippuser";
+my $dbpassword = "ippuser";
+
+
+my $defaultToToday = 1;
+
+GetOptions(
+ 'label=s' => \$label,
+ 'data_group=s' => \$data_group,
+ 'dateobs_min=s' => \$dateobs_min,
+ 'dateobs_max=s' => \$dateobs_max,
+ 'release=s' => \$release,
+ 'filter=s' => \$filter,
+ 'dbname=s' => \$dbname,
+
+ 'exp_name=s' => \$exp_name,
+ 'exp_id=s' => \$exp_id,
+ 'cam_id=s' => \$cam_id,
+ 'verbose|v' => \$verbose,
+) or pod2usage (2);
+
+
+# if data_group is supplied we don't need to default the date
+if ($data_group) {
+ $defaultToToday = 0;
+}
+
+# XXX: since this is going to be used in a cgi script we should carefully vet the supplied
+# date format.
+if (!$dateobs_min) {
+ # default to today
+ if ($dateobs_max) {
+ print "Must supply begin date if max date is supplied.\n";
+ exit 2;
+ }
+ if ($defaultToToday) {
+ my ($day, $month, $year) = (gmtime)[3,4,5];
+
+ $dateobs_min = sprintf "%4d-%02d-%02dT00:00:00Z", 1900+$year, 1+$month, $day;
+ print "$dateobs_min\n" if $verbose;
+ }
+} else {
+ # add times if not supplied
+ if (!($dateobs_min =~ 'T')) {
+ $dateobs_min .= 'T00:00:00Z';
+ print "$dateobs_min\n" if $verbose;
+ }
+ if (!($dateobs_max =~ 'T')) {
+ $dateobs_max .= 'T23:59:59Z';
+ print "$dateobs_max\n" if $verbose;
+ }
+}
+
+
+my $dbh = open_db();
+
+my $query = 'SELECT cam_id, camRun.label, camRun.state, quality, release_name, camProcessedExp.path_base,'
+ . ' exp_name, exp_id, filter, dateobs, comment'
+ . ' FROM camRun JOIN camProcessedExp USING(cam_id) JOIN chipRun using(chip_id) JOIN rawExp USING(exp_id)'
+ . ' LEFT JOIN relExp USING(cam_id, exp_id) LEFT JOIN ippRelease USING(rel_id)'
+ . " WHERE ";
+
+my $and = '';;
+if ($data_group) {
+ $query .= " camRun.data_group = '$data_group'";
+ $and = ' AND';
+}
+
+if ($dateobs_min) {
+ $query .= "$and dateobs > '$dateobs_min'";
+}
+
+if ($dateobs_max) {
+ $query .= " AND dateobs <= '$dateobs_max'";
+}
+
+# Should I use "LIKE" qualifiers here?
+if ($label) {
+ $query .= " AND camRun.label = '$label'";
+}
+
+if ($release) {
+ if ((uc($release) eq 'NONE') or (uc($release) eq 'NULL')) {
+ $query .= " AND release_name IS NULL";
+ } else {
+ $query .= " AND release_name = '$release'";
+ }
+}
+
+if ($filter) {
+ # if single character filter is supplied append wild card
+ # character to the filter string
+ if (length($filter) == 1) {
+ $filter .= '%';
+ }
+ $query .= " AND filter LIKE '$filter'";
+}
+
+print "$query\n" if $verbose;
+
+my $statement = $dbh->prepare($query);
+
+$statement->execute();
+
+my $numRows = 0;
+
+# Print Header line as a comment.
+# XXX: rather than fine tuning the spacing on this I should have used the
+# same format line that I use for the data below.
+
+print
+'#cam_id smf_name quality state exp_name label release filter obs_date obs_time comment' . "\n";
+#1780291 o7616g0016o.1131500.cm.1780291.smf 0 full o7616g0016o QUB.nightlyscience none z.00000 2016-08-16 05:57:32 'Transient PS16cgx z band cell 45 65'
+
+while ( my $row = $statement->fetchrow_hashref() ) {
+ $numRows++;
+
+ my $smf_name = basename($row->{path_base}) . '.smf';
+ my $release_name = $row->{release_name};
+ $release_name = 'none' unless $release_name;
+
+ printf "%ld %s %5d %-7s %s %-20s %-14s %-11s %s '%s'\n",
+ $row->{cam_id}, $smf_name, $row->{quality}, $row->{state},
+ $row->{exp_name}, $row->{label}, $release_name, $row->{filter},
+ $row->{dateobs}, $row->{comment};
+}
+
+print "$numRows found\n" if $verbose;
+
+exit 0;
+
+sub open_db {
+ my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+ my $dbh = DBI->connect($dsn, $dbuser, $dbpassword)
+ or die "Cannot connect to database $dbname.\n";
+
+ return $dbh;
+}
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/conf/httpd-datastore.conf
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/conf/httpd-datastore.conf (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/conf/httpd-datastore.conf (revision 41111)
@@ -0,0 +1,41 @@
+#
+# Root of the data store (where the directory hierarchy lives
+#
+
+
+ DirectoryIndex /ds-cgi/dsindex.cgi
+
+ #Options None
+
+ # turn on server side includes
+ Options +Includes
+
+ # files will be processed for server side includes if they have the
+ # executable bit set
+ XBitHack on
+
+ AddType text/plain .txt
+ AddOutputFilter INCLUDES .txt
+
+ AllowOverride None
+
+ Order allow,deny
+ Allow from all
+
+
+
+#
+# Location of the cgi scripts
+#
+
+
+ Options ExecCGI
+
+
+ Order allow,deny
+ Allow from all
+
+
+Alias /ds /data/ipp000.0/datastore/dsroot/
+ScriptAlias /ds-cgi /data/ipp000.0/datastore/ds-cgi/
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/getsmf.html
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/getsmf.html (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/getsmf.html (revision 41111)
@@ -0,0 +1,117 @@
+
+
+IPP SMF Access Tools
+
+
+
+
+Access to IPP smf files
+
+
+Go back up.
+
+
+getsmf
+
+IPP smf files contain the results of camera stage processing.
+They are FITS format files containg tables with the photometric and
+astronometric measurements for the
+sources detected in a single exposure.
+
+
+getsmf.php is a tool that may be used to retrieve an IPP smf file
+given certain parameters.
+
The url of the tool is
+ http://misc.ipp.ifa.hawaii.edu/getsmf.php
+
+When this page is accessed without parameters it simply displays an
+error message.
+With valid parameters the tool returns the contents of the selected
+smf file.
+
+Note: Obviously displaying the contents of a FITS file in a web browser is
+not very useful.
+getsmf.php is intended to be accessed directly using HTTP.
+It is easy to write scripts that use the command line tools curl or wget
+to retrieve smf files using this tool.
+
+For example, the following two equivalent unix commands copy the contents of
+the smf containing the results from the cameraRun with cam_id=1780283)
+to a local files:
+
+ curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -o 1780283.smf
+
+ wget 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -O 1780283-wget.smf
+
+
+The previous method allows the user to control the name of the local copy.
+If one would like to use the original file name supplied by the server
+the commands may be changed to:
+
+ curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' -O --remote-header-name
+
+ wget 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1780283' --content-disposition
+
+
+In this example the name of the downloaded file is "o7616g0008o.1131492.cm.1780283.smf".
+
+Note: The quote characters in these commands are needed to prevent
+the '?' characters from being interpreted by user's shell.
+
+parameters
+There are a few parameters that may be used to select specific smf files.
+Parameters are supplied by appending them to the base url following
+a '?' character.
+
+If multiple parameters are needed
+they should be separated by a '&' character.
+
+cam_id=value
+The examples above demonstrated selection by cam_id. When a
+cam_id is supplied any other parameters supplied are ignored.
+
+exp_name=value
+Selects an smf file for an exposure with the given name.
+For example supplying the parameter: exp_name=o7616g0008o
+gets the same result as the examples listed above. (As of the time
+of this writing).
+
+Note that exposures may be processed multiple times. If no other selection
+parameters are found, the smf for the most recent processing is selected.
+release=value
+When used with exp_name, the release parameter specifies the
+ipp release (processing version) for the smf file.
+For example: exp_name=o5275g0401o&release=3PI.PV3
retrieves the
+smf processed for exposure o5275g0401o in 3PI.PV3.
+If the release were changed to 3PI.nightly
+the smf from the original nightly procecessing would be retrieved.
+(At the time of this writing, if the release parameter were omitted
+the 3PI.PV3 version would also be retrieved since it is currently
+the latest processing.)
+data_group=value
+This is a rarely used parameter.
+With exp_name, chooses smfs from camRuns processed as part of
+the supplied data group.
+
+Errors and Exceptions
+If an smf matching the supplied parameters is not found in the IPP, a message
+is included in the response. The http request succeeds.
+
+After a user retrieves a file in this manner it should be checked
+to insure that it
+is actually a FITS format file.
+
+For example the following getsmf request asks for a non-existent smf:
+
+ curl 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=99999999999' -o test.smf
+
+After this command succeeds, the file test.smf is not a FITS file.
+Instead it contains text. This fact and its contents may be found
+with the following
+commands.
+
+ $ file test.smf
+ test.smf: ASCII text
+ $ cat test.smf
+ Could not find smf for cam_id: 99999999999.
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/index.html
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/index.html (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/index.html (revision 41111)
@@ -0,0 +1,29 @@
+
+
+IPP-Misc: SMF Access
+
+
+
+
+IPP-Misc: SMF Access
+
+
+This page contains links to information about accessing
+IPP SMF files
+which contain the results of IPP camera stage processing.
+
+These files are FITS format files containing multiple binary table extension
+sections. These which contain the photometric and astronometric measurements
+for the sources detected in a single Pan-STARRS exposure.
+
+These files are the inital inputs to the PS calibration process.
+
+The following links describe how to use the IPP-MISC SMF Access tools
+to discover and retrive SMF files.
+
+
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/listsmfs.html
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/listsmfs.html (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/listsmfs.html (revision 41111)
@@ -0,0 +1,141 @@
+
+
+
+listsmf
+
+
+
+
+
+listsmfs
+
+
+Go back up.
+
+listsmf.php is a tool that may be used to retrieve certain information
+about SMF files from the IPP database.
+
The url of the tool is
+http://misc.ipp.ifa.hawaii.edu/listsmfs.php
+
+
+The list returned is controlled by parameters supplied
+as part of the http request used to access the page.
+
+By default listsmfs displays the list of
+smfs corresponding to the PS1 science exposures collected on
+the current date (UTC). The output may be fine tuned using parameters.
+The use of parameters is discussed below.
+
+Here is some sample output.
+
+#cam_id smf_name quality state exp_name label release filter obs_date obs_time comment
+1783121 o7622g0062o.1134677.cm.1783121.smf 0 full o7622g0062o OSS.nightlyscience SSS.nightly i.00000 2016-08-22 07:43:12 'OSSR.R19S6.16.Q.i ps1_32_2599 visit 1'
+1783120 o7622g0061o.1134678.cm.1783120.smf 0 full o7622g0061o OSS.nightlyscience SSS.nightly i.00000 2016-08-22 07:42:16 'OSSR.R19S6.16.Q.i ps1_32_2493 visit 1'
+1783122 o7622g0063o.1134679.cm.1783122.smf 0 full o7622g0063o OSS.nightlyscience SSS.nightly i.00000 2016-08-22 07:44:08 'OSSR.R19S6.16.Q.i ps1_32_2614 visit 1'
+... and so on.
+
+
+
+The first line, which begins with the "comment" character '#', is a
+header line. This line lists the names of the columns shown
+in the subsequent lines.
+
+If no exposures are found that match the supplied parameters (and their
+defaults) only the header line is displayed.
+
The first 4 columns are the most important. They are very useful
+for constructing requests to retrieve individual SMFs using
+getsmf.php.
+
+
List of Columns
+
+cam_id
+The IPP camera run identifier.
+smf_name
+The base name of the SMF file for the exposure (if any).
+quality
+The quality value for the camera processing. A non-zero value indicates
+a problem with the processing. For example: unsuccessful astrometric
+solution. In the event of a run with non zero quality
+no SMF file will be available.
+state
+The current state of the camera run.
+Runs that have completed processing will have
+the state "full". SMF files will not be available for runs with any
+state other than full. Runs with state "new" may finish processing
+at a later time.
+exp_name
+The exposure name of the source exposure. (Also known as "Frame Name"
+in PSPS and other contexts.)
+label
+The processing label for the camRun.
+release
+The Release that contains the exposures. Examples might be
+3PI.nightly or SSS.nightly.
+If the exposure has not been assigned to a release this column will have
+the value "none".
+filter
+The filter used by the camera for the exposure.
+obs_date
+Date of the observation (UTC).
+obs_time
+Time of the observation (UTC).
+comment
+The comment string for the exposure. Note that single quote characters
+have been added around the comment string as delimiters to
+facilitate parsing.
+
+
+Parameters
+
+As mentioned above, listsmfs gives a list of the
+exposures for the current (UTC) day by default.
+Parameters may be used to
+control the selection. Parameters are provided by appending
+an HTML query string to the request URL.
+
+For example the sample output shown above was generated by a request
+with the parameter string '?release=SSS.nightly' appended to the URL
+(on the date that this page was written of course).
+
+If multiple parameters are to be supplied they should be separated
+by a '&' character. For example '?release=SSS.nightly&filter=r'
+would list exposures with the named release in the r band filter..
+
+Here is the list of the parameters.
+
+
+
date_min and date_max
+The date/time limits for observations to be included by the listing.
+The format for the datetime is
+
+ YYYY-MM-DDTHH:MM:SS
+where the '-', 'T', and ':' characters are literal and the other characters are
+numeric values for year, month, day, hours, minutes, and seconds.
+
+For example 'date_min=2016-08-22T08:00:00&date_max=2016-08-22T09:00:00'
+would select exposures observed between 0800 UTC and before 0900 UTC
+on Augst 22.
+
+Notes on date limits.
+The time portion may be omitted. For example the parameter pair
+'date_min=2016-08-21&date_max=2016-08-22' will cause all exposures observed
+on 2016-08-21 or 2016-08-22 to be listed.
+
+If date_min is not supplied, any date_max value supplied is silently ignored.
+Instead page uses the defaults for the date parameter: exposures for the current date are listed (if any).
+
+
filter
+Only exposures with the corresponding filter are listed. If a single
+character value is supplied for filter, exposures from all filters that
+begin with that letter are listed.
+
+label
+The list is restricted to camRuns with the supplied label.
+
+release
+The list is restrictred to exposures with supplied release name.
+
+data_group
+The list is restrictred to camRuns with the supplied data group.
+Note: This parameter is expected to be rarely used.
+
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/scripting.html
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/scripting.html (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/doc/smfs/scripting.html (revision 41111)
@@ -0,0 +1,151 @@
+
+
+Scripting IPP SMF Access
+
+
+
+
+Scripting IPP SMF Access
+
+
+Go back up.
+
+
+The process of retrieving SMF files is straightforward.
+
+- Discover the list of files of interest.
+
- Build valid commands to retrieve them from the IPP to local storage.
+
- Check the results for validity.
+
- Start working on your science :).
+
+
+
+This page is primarily concerned with the first two items on this list.
+
+
+The two pages listsmfs.php and getsmf.php have a relatively straightforward
+interface. UNIX command line tools such as curl or wget may be used to
+access the interface.
+
+
Sample PERL script
+
+The perl script below gives an example of how to use curl to get
+a list of the current day's smf files from listsmf.php and print curl
+commands that may be used to download the files.
+
+
+ #!/usr/bin/env perl
+
+ use strict;
+ use warnings;
+
+ my $verbose = 0;
+
+ my $misc_url = 'http://misc.ipp.ifa.hawaii.edu';
+ my $list_url = "$misc_url/listsmfs.php";
+ my $get_url = "$misc_url/getsmf.php";
+
+ my $silent = ' --silent';
+ if ($verbose) {
+ $silent = '';
+ }
+
+ my $list_cmd = "curl $silent http://misc.ipp.ifa.hawaii.edu/listsmfs.php";
+
+ my $output = `$list_cmd`;
+
+ if ($? != 0) {
+ my $rc = $?;
+ print STDERR "Ooops something went wrong with the command $list_cmd\n";
+ print STDERR "command returned $rc\n";
+ exit $rc >> 8;
+ }
+
+ my @lines = split "\n", $output;
+
+ foreach my $line (@lines) {
+ # skip the header line and any others that begin with '#'
+ next if $line =~ '#';
+
+ # split the line at the single quote character that preceeds the
+ # comment string
+ my ($left, $right) = split "'", $line;
+
+ # the right side of this split is the comment string ...
+ my $comment = $right;
+
+ # ... and the left side of this split contains a string
+ # with the other parameters.
+ # The other parameters are separated by whitespace
+
+ # NOTE: One might have thought we could parse the header string
+ # and find the column names but that implies more than the
+ # listsmf "interface" commits to at this time.
+
+ my ($cam_id, $smf_name, $quality, $state, $exp_name, $label, $release, $filter, $obs_date, $obs_time) = split " ", $left;
+
+ if ($quality != 0) {
+ print STDERR "camRun $cam_id had bad quality $quality\n";
+ next;
+ }
+
+ if ($state ne 'full') {
+ print STDERR "camRun is in state $state\n";
+ next;
+ }
+
+ # generate and print out a curl command that could be used to
+ # download the smf file to the current directory
+ my $this_cmd = "curl $silent '$get_url?cam_id=$cam_id' --output $smf_name";
+
+ print "$this_cmd\n";
+ }
+ # all done
+
+
+When executed the resulting output will look something like this:
+
+ # 1783100 o7622g0010o.1134626.cm.1783100.smf 0 ICECUBE-160731A.01 i band dither 1 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783100' --output o7622g0010o.1134626.cm.1783100.smf
+ # 1783101 o7622g0011o.1134627.cm.1783101.smf 0 ICECUBE-160731A.01 i band dither 2 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783101' --output o7622g0011o.1134627.cm.1783101.smf
+ # 1783102 o7622g0012o.1134628.cm.1783102.smf 0 ICECUBE-160731A.01 i band dither 3 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783102' --output o7622g0012o.1134628.cm.1783102.smf
+ # 1783103 o7622g0013o.1134629.cm.1783103.smf 0 ICECUBE-160731A.01 i band dither 4 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783103' --output o7622g0013o.1134629.cm.1783103.smf
+ # 1783104 o7622g0014o.1134630.cm.1783104.smf 0 ICECUBE-160731A.01 i band dither 5 visit 1
+ curl --silent 'http://misc.ipp.ifa.hawaii.edu/getsmf.php?cam_id=1783104' --output o7622g0014o.1134630.cm.1783104.smf
+
+
+
+There are two lines for each exposure. The first (which begins with
+ a '#' character) lists prints selected metadata
+for the smf and its source exposure. The second line gives
+a curl command that would copy the file to the user's current directory.
+
Note, while not suggested, one could take the output of listsmfs.php
+and source it into a UNIX shell to execute the commands.
+
+This script demonstrates how to
+
+- Use curl to get a list of the smfs for the current day's exposures
+
- Split the lines of the output into its columns (including the comment string)
+
- Examining the state and quality values to determine whether or not an SMF exists for the exposure
+
- Printing a curl command line which when exectuted with copy the SMF to the current directory
+
+
+This simple script shows the most important steps necessary for successful
+smf retrieval.
+
+
What about errors?
+
+For the sake of speed we chose to not include any information that
+might be useful for detecting errors in the download process. The database
+does contain anything (a checksum for example)
+which could be used to gain confidence that a file was transmitted
+successfully.
+
+
+In practice so far this has not shown to be an issue. getsmf.php has been
+used to download hundreds of thousands of smf files. When users have
+detected problem files, simply trying the download again seems to correct
+the problem.
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/getskycalcmf.php
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/getskycalcmf.php (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/getskycalcmf.php (revision 41111)
@@ -0,0 +1,132 @@
+$command\n
";
+ }
+ $command = escapeshellcmd($command);
+ $output = array();
+
+ exec($command, $output, $command_status);
+
+ if ($command_status == 0) {
+ // we only expect one line of output
+ $len = count($output);
+ if ($len == 1) {
+ list($filename, $pathname) = explode(" ", $output[0]);
+ if ($filename && $pathname) {
+ if (!$debug) {
+ $gotFile = 1;
+ } else {
+ echo "$filename $pathname\n";
+ echo "$command\n";
+ }
+ }
+ } else {
+ echo "
unexpected output from $command: $output[0] $output[1]\n";
+ }
+ } else {
+ if ($debug) {
+ echo "
command failed $command_status\n";
+ }
+ }
+} else {
+ // something went wrong with the paramters. Error message produced below.
+}
+
+if ($gotFile) {
+ if ($rvar_list) {
+ echo "cmf : $filename\n";
+ // echo "
path: $pathname\n
";
+ } else {
+ // All systems are go. Time to write the output.
+ // First set up the header
+ header('Content-type: application/fits');
+ header("Content-Disposition: attachment; filename=\"$filename\"");
+ $filesize = filesize($pathname);
+ header("Content-Length: $filesize");
+ header('Expires: now');
+
+ // copy the contents of the file to the stream
+ readfile($pathname);
+ }
+} else {
+ // XXX: Figure out how to stop wget from redirecting these error
+ // messages to the nasty filename
+ $error_string="Could not find cmf";
+ if ($rvar_skycal_id) {
+ $error_string .= " for skycal_id: $rvar_skycal_id";
+ } elseif ($rvar_skycell_id and $rvar_tess_id and $rvar_filter) {
+ $error_string .= " for tess_id $rvar_tess_id skycell_id: $rvar_skycell_id filter: $rvar_filter";
+ } else {
+ $error_string .= ".
Not enough parameters supplied";
+ }
+
+ echo "$error_string.\n";
+}
+
+function getVar($var) {
+ if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+ $rvar = $_POST[$var];
+ } else {
+ $rvar = $_GET[$var];
+ }
+ $rvar = stripslashes($rvar);
+ $rvar = htmlentities($rvar);
+ $rvar = strip_tags($rvar);
+ return $rvar;
+}
+
+
+if ($list) {
+ // print lots of information from the PHP installation
+ // phpinfo(-1);
+
+ // print the most useful variables
+ // phpinfo(32);
+}
+
+?>
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/getsmf.php
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/getsmf.php (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/getsmf.php (revision 41111)
@@ -0,0 +1,129 @@
+$command\n
";
+ }
+ $command = escapeshellcmd($command);
+ $output = array();
+
+ exec($command, $output, $command_status);
+
+ if ($command_status == 0) {
+ // we only expect one line of output
+ $len = count($output);
+ if ($len == 1) {
+ list($filename, $pathname) = explode(" ", $output[0]);
+ if ($filename && $pathname) {
+ if (!$debug) {
+ $gotFile = 1;
+ } else {
+ echo "$filename $pathname\n";
+ echo "$command\n";
+ }
+ }
+ } else {
+ echo "
unexpected output from $command: $output[0] $output[1]\n";
+ }
+ } else {
+ if ($debug) {
+ echo "
command failed $command_status\n";
+ }
+ }
+} else {
+}
+
+if ($gotFile) {
+ if (!$rvar_list) {
+ // All systems are go. Time to write the output.
+ // First set up the header
+ header('Content-type: application/fits');
+ header("Content-Disposition: attachment; filename=\"$filename\"");
+ $filesize = filesize($pathname);
+ header("Content-Length: $filesize");
+ header('Expires: now');
+
+ // copy the contents of the file to the stream
+ readfile($pathname);
+ } else {
+ echo "smf : $filename\n";
+ # echo "path: $pathname\n";
+ }
+} else {
+ // XXX: Figure out how to stop wget from redirecting these error
+ // messages to the nasty filename
+ $error_string="Could not find smf";
+ if ($rvar_cam_id) {
+ $error_string .= " for cam_id: $rvar_cam_id";
+ } elseif ($rvar_exp_name) {
+ $error_string .= " for exposure: $rvar_exp_name";
+ } else {
+ $error_string .= ". No cam_id or exp_name provided";
+ }
+
+ echo "$error_string.\n";
+}
+
+function getVar($var) {
+ if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+ $rvar = $_POST[$var];
+ } else {
+ $rvar = $_GET[$var];
+ }
+ $rvar = stripslashes($rvar);
+ $rvar = htmlentities($rvar);
+ $rvar = strip_tags($rvar);
+ return $rvar;
+}
+
+
+if ($list) {
+ // print lots of information
+ // phpinfo(-1);
+
+ // print the most useful variables
+ // phpinfo(32);
+}
+
+?>
Index: /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/listsmfs.php
===================================================================
--- /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/listsmfs.php (revision 41111)
+++ /branches/ccl_branches/ipponly-20191108/DataStoreServer/web/php/listsmfs.php (revision 41111)
@@ -0,0 +1,149 @@
+
smf list\n";
+}
+
+if ($debug) {
+ echo "pos_curl: $pos_curl pos_wget: $pos_wget\n";
+}
+
+
+if ($command) {
+ $command = escapeshellcmd($command);
+ if ($debug) {
+ $agent = $_SERVER['HTTP_USER_AGENT'];
+ echo "HTTP_USER_AGENT is: $agent\n
";
+ echo "
$command\n
";
+ }
+ $output = array();
+
+ # RUN the command
+ exec($command, $output, $command_status);
+
+ if ($command_status == 0) {
+ if (!$not_browser) {
+ # when viewing in a web browser without this the newlines get lost
+ echo "";
+ }
+
+ # output the list
+ foreach( $output as $line) {
+ echo "$line\n";
+ }
+ } else {
+ if ($debug) {
+ echo "
command failed $command_status\n";
+ }
+ }
+} else {
+}
+
+
+function getVar($var) {
+ if ($_SERVER['REQUEST_METHOD'] == 'POST') {
+ $rvar = $_POST[$var];
+ } else {
+ $rvar = $_GET[$var];
+ }
+ $rvar = stripslashes($rvar);
+ $rvar = htmlentities($rvar);
+ $rvar = strip_tags($rvar);
+ return $rvar;
+}
+
+
+if ($list) {
+ // print lots of information
+ // phpinfo(-1);
+ print "\n";
+ print "Submitter IP ADDR: $submitter_ip_addr\n";
+ print "
Remote IP ADDR: $remote_ip_addr\n";
+
+ // print the most useful variables
+ phpinfo(32);
+}
+
+?>