Index: /branches/ccl_branches/ipponly-20191108/pstamp/Makefile.am
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/Makefile.am (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/Makefile.am (revision 41103)
@@ -0,0 +1,9 @@
+SUBDIRS = src scripts
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA= ppstamp.pc
+
+EXTRA_DIST = \
+ ppstamp.pc.in
+
+CLEANFILES = *~ core core.*
Index: /branches/ccl_branches/ipponly-20191108/pstamp/autogen.sh
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/autogen.sh (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/autogen.sh (revision 41103)
@@ -0,0 +1,103 @@
+#!/bin/sh
+# Run this to generate all the initial makefiles, etc.
+
+srcdir=`dirname $0`
+test -z "$srcdir" && srcdir=.
+
+ORIGDIR=`pwd`
+cd $srcdir
+
+PROJECT=pstamp
+TEST_TYPE=-f
+# change this to be a unique filename in the top level dir
+FILE=autogen.sh
+
+DIE=0
+
+LIBTOOLIZE=libtoolize
+ACLOCAL="aclocal $ACLOCAL_FLAGS"
+AUTOHEADER=autoheader
+AUTOMAKE=automake
+AUTOCONF=autoconf
+
+($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || {
+ echo
+ echo "You must have $LIBTOOLIZE installed to compile $PROJECT."
+ echo "Download the appropriate package for your distribution,"
+ echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/"
+ DIE=1
+}
+
+($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || {
+ echo
+ echo "You must have $ACLOCAL installed to compile $PROJECT."
+ echo "Download the appropriate package for your distribution,"
+ echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/"
+ DIE=1
+}
+
+($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || {
+ echo
+ echo "You must have $AUTOHEADER installed to compile $PROJECT."
+ echo "Download the appropriate package for your distribution,"
+ echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/"
+ DIE=1
+}
+
+($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || {
+ echo
+ echo "You must have $AUTOMAKE installed to compile $PROJECT."
+ echo "Download the appropriate package for your distribution,"
+ echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/"
+ DIE=1
+}
+
+($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || {
+ echo
+ echo "You must have $AUTOCONF installed to compile $PROJECT."
+ echo "Download the appropriate package for your distribution,"
+ echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/"
+ DIE=1
+}
+
+if test "$DIE" -eq 1; then
+ exit 1
+fi
+
+test $TEST_TYPE $FILE || {
+ echo "You must run this script in the top-level $PROJECT directory"
+ exit 1
+}
+
+if test -z "$*"; then
+ echo "I am going to run ./configure with no arguments - if you wish "
+ echo "to pass any to it, please specify them on the $0 command line."
+fi
+
+$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed"
+$ACLOCAL || echo "$ACLOCAL failed"
+$AUTOHEADER || echo "$AUTOHEADER failed"
+$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed"
+$AUTOCONF || echo "$AUTOCONF failed"
+
+cd $ORIGDIR
+
+run_configure=true
+for arg in $*; do
+ case $arg in
+ --no-configure)
+ run_configure=false
+ ;;
+ *)
+ ;;
+ esac
+done
+
+if $run_configure; then
+ $srcdir/configure --enable-maintainer-mode "$@"
+ echo
+ echo "Now type 'make' to compile $PROJECT."
+else
+ echo
+ echo "Now run 'configure' and 'make' to compile $PROJECT."
+fi
Index: /branches/ccl_branches/ipponly-20191108/pstamp/configure.ac
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/configure.ac (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/configure.ac (revision 41103)
@@ -0,0 +1,65 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_PREREQ(2.61)
+
+AC_INIT([ppstamp], [0.1.1], [ipp-support@ifa.hawaii.edu])
+AC_CONFIG_SRCDIR([src])
+AC_CONFIG_SRCDIR([scripts])
+
+AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2])
+AM_CONFIG_HEADER([src/config.h])
+AM_MAINTAINER_MODE
+
+IPP_STDLDFLAGS
+
+AC_LANG(C)
+AC_GNU_SOURCE
+AC_PROG_CC_C99
+AC_PROG_INSTALL
+AC_PROG_LIBTOOL
+AC_SYS_LARGEFILE
+
+dnl ------------------------------------------------------------
+
+AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing])
+if test "$ERRORCODES" = "missing" ; then
+ AC_MSG_ERROR([psParseErrorCodes is required])
+fi
+
+
+PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0])
+PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0])
+
+OHANA_CONFIG=`which ohana-config`
+
+dnl get CFLAGS for ohana
+AC_MSG_NOTICE([libohana cflags info supplied by ohana-config])
+AC_MSG_CHECKING([libohana cflags])
+OHANA_CFLAGS="`${OHANA_CONFIG} --cflags`"
+AC_MSG_RESULT([${OHANA_CFLAGS}])
+
+dnl get LDFLAGS for ohana
+AC_MSG_NOTICE([libohana ldflags info supplied by ohana-config])
+AC_MSG_CHECKING([libohana ldflags])
+OHANA_LIBS="`${OHANA_CONFIG} --libs`"
+AC_MSG_RESULT([${OHANA_LIBS}])
+
+dnl Set CFLAGS for build
+IPP_STDOPTS
+IPP_STDCFLAGS
+
+PPSTAMP_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULE_CFLAGS=} ${OHANA_CFLAGS}"
+PPSTAMP_LIBS="${PSLIB_LIBS=} ${PSMODULE_LIBS=} ${OHANA_LIBS}"
+echo "PPSTAMP_CFLAGS: $PPSTAMP_CFLAGS"
+echo "PPSTAMP_LIBS: $PPSTAMP_LIBS"
+
+AC_SUBST([PPSTAMP_CFLAGS])
+AC_SUBST([PPSTAMP_LIBS])
+
+AC_CONFIG_FILES([
+ Makefile
+ src/Makefile
+ scripts/Makefile
+ ppstamp.pc
+])
+
+AC_OUTPUT
Index: /branches/ccl_branches/ipponly-20191108/pstamp/doc/dsinstall.txt
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/doc/dsinstall.txt (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/doc/dsinstall.txt (revision 41103)
@@ -0,0 +1,417 @@
+Installing the data store sample implementation
+XXX: This text was found lying around in my workspace. It needs to be
+checked for accuarcy
+
+Overview
+--------
+
+This note describes the installation and operation of the IPP Postage Stamp Server.
+We assume that IPP source for examination and that the user has the standard
+IPP configuration environment set up.
+
+This component of the IPP system is not intended for general distribution to IPP users
+and does not have streamlined installation procedures at this time.
+
+The PSS is a system which responds to requests to retrieve images from an IPP Image Server.
+The images may be either complete images or subsets of images known as "Postage Stamps".
+
+The internal mechanisms of the PSS are also used to service MOPS detectabiltiy
+query requests.
+
+Given this more general utility it should perhaps be called the "IPP Request Server".
+
+
+Data Store
+----------
+The PSS uses the Data Store mechanism to find incoming requests and to
+provide results to the end user.
+
+A DataStore is simply a web server with URLS that supports an interface
+that allows collections of data called File Sets to be located and accessed.
+
+A Data Store is organized into 3 levels. These are described in the following sections.
+
+Data Store Root - a list of Products
+-------------------------------------
+
+Assume that a datastore is located on the server WEBSERVERHOST.
+An HTTP get to the url http://WEBSERVERHOST/ds/index.txt would give a list of the
+products in the Data Store.
+
+ # productID |Most recent |Time registered |type |Description |
+ pstampresults|web204 |2008-10-16T00:35:16Z|psresults|Postage Stamp Results |
+ pstamprequests|TEST0012 |2008-08-29T03:26:40Z|psrequest|Postage Stamp Requests |
+ mops-pstamp-results|MOPSSTAMPREQ20080813T214918Z_00|2008-08-13T21:49:42Z|psresults|Postage Stamp Results for MOPS |
+
+Each line is divided into fields separated by a 'pipe' character | .
+
+Data Store Product - a list of filesets
+---------------------------------------
+
+Similarly the url http://WEBSERVERHOST/ds/pstampresults/index.txt gives the list of filesets
+in the product pstampresults:
+
+ # filesetID|time registered |type |
+ TEST0011 |2008-08-29T03:05:03Z|PSRESULTS|
+ TEST0012 |2008-08-29T03:31:36Z|PSRESULTS|
+ web190 |2008-10-15T02:29:11Z|PSRESULTS|
+ web191 |2008-10-15T02:32:16Z|PSRESULTS|
+ web192 |2008-10-15T02:38:50Z|PSRESULTS|
+ web193 |2008-10-15T03:08:57Z|PSRESULTS|
+ web194 |2008-10-15T23:32:11Z|PSRESULTS|
+ web195 |2008-10-15T23:35:39Z|PSRESULTS|
+ web196 |2008-10-15T23:52:40Z|PSRESULTS|
+ web197 |2008-10-15T23:59:05Z|PSRESULTS|
+ web198 |2008-10-16T00:04:17Z|PSRESULTS|
+ web200 |2008-10-16T00:17:46Z|PSRESULTS|
+ web204 |2008-10-16T00:35:16Z|PSRESULTS|
+
+Note that the fileset ID for the latest fileset (web204) in the list appears in the root listing in the last
+section.
+
+The product list protocol accepts a query string which can be used to limit the listing to all filesets
+registered after a given fileset.
+
+For example, http://WEBSERVERHOST/ds/pstampresults/index.txt?web198 yields
+
+ # filesetID|time registered |type |
+ web200 |2008-10-16T00:17:46Z|PSRESULTS|
+ web204 |2008-10-16T00:35:16Z|PSRESULTS|
+
+
+File Set - a list of files
+--------------------------
+http://ippds.somwhere.net/ds/pstampresults/web205/index.txt
+
+ # fileID |bytes |md5sum |type |
+ results.fits |28800 |4e2048479e551844e31102a301d50740|table |
+ # fileID |bytes |md5sum |type |chipname|
+ 1_1_o4748g0259o.ota01.fits|19840320|19a6233ab966cf3e13b7da8271dead67|chip |XY01 |
+ 1_2_o4748g0259o.ota02.fits|20416320|625bf8198dd7aef7c892af03584ee0ed|chip |XY02 |
+
+ .... followed by 58 other files in this GPC1 exposure
+
+
+The IPP includes a number of tools for accessing a data store. An introduction to these tools may
+be seen by using the program perldoc.
+
+ dsrootls
+ dsproductls
+ dsfilesetls
+ dsget
+
+
+Installing the IPP Data Store Server Implementation
+---------------------------------------------------
+
+The data store implementation consists of a set of cgi scripts which are
+invoked by an apache web server. The indexes for the Data Store are stored in a mysql
+database. The cgi scripts use DBI perl module to query the database. No database updates
+are performed by the Data Store cgi scripts.
+
+The files for the Data Store Server located in the IPP source tree in the directory
+DataStoreServer (this directory may not be available in the distributed IPP
+tarballs).
+
+Mysql data base configuration
+-----------------------------
+
+The tables for the PSS are part of the regular IPP database tables.
+
+The Data Store implementation has been designed to not require the IPP for its
+operation so the Data Store's tables are not created as part of the IPP database setup.
+
+Currently however, the PostageStamp Server implementation assumes that the data
+store and postage stamp tables are in the same mysql database (this will be
+fixed soon).
+
+To set up the database go to the directory DataStoreServer/scripts and start up mysql.
+
+If you do not already have a current IPP database that you would like to use
+for the Postage Stamp Request tables issue the following commands
+
+ mysql> create database mydatabase;
+ mysql> use mydatabase;
+ mysql> source scripts/tabledefs.sql;
+
+Now exit mysql and run the command
+ pxadmin -create -dbname mydatabase
+
+If you already have a database set up, then just issue the commands
+ mysql> use mydatabase;
+ mysql> source scripts/tabledefs.sql;
+
+Web Server Configuration
+------------------------
+The apache configuration needs to have the following entries added to
+the appropriate configuration file. (Different apache distributions have
+different locations for these files so it's up to the installer to
+figure out where these configuration lines should go).
+
+
+
+ DirectoryIndex /ds-cgi/dsindex.cgi
+
+ #Options None
+
+ # turn on server side includes
+ Options +Includes
+
+ # files will be processed for server side includes if they have the
+ # executable bit set
+ XBitHack on
+
+ AddType text/plain .txt
+ AddOutputFilter INCLUDES .txt
+
+ AllowOverride None
+
+ Order allow,deny
+ Allow from all
+
+
+ #
+ # Location of the cgi scripts
+ #
+
+
+ Options ExecCGI
+
+
+ Order allow,deny
+ Allow from all
+
+
+ Alias /ds /DATASTOREDIR/dsroot/
+ ScriptAlias /ds-cgi /DATASTOREDIR/ds-cgi/
+
+(another sample of this configuration file may be found in
+ DataStoreServer/web/conf/httpd-datastore.conf
+
+Here we have assumed that the data store cgi scripts will be installed in
+/DATASTOREDIR/ds-cgi and the data store data files will be stored in
+/DATASTOREDIR/dsroot.
+
+To install the cgi scripts
+ cd your_ipp_src_directory
+ cd DataStoreServer
+ scripts/installscripts --dsroot /DATASTOREDIR/dsroot --cgidir /DATASTOREDIR/ds-cgi
+
+The file /DATASTOREDIR/ds-cgi/dsshellconfig is a shell script that sets
+several environment variables that are required by the scripts.
+
+This script should be editied to customize the values for the installation.
+(This file will not be overwritten by subsequent invocations of
+installscripts).
+
+There are some scripts that the Postage Stamp Server uses that need to be
+installed into the IPP build directory. These are not included in the current IPP
+build. To build them invoke the following commands from the DataStoreServer
+directory.
+ psautogen
+ make install
+
+
+Testing the DataStore
+---------------------
+At this point an empty data store should be up and running. This can be tested
+by going to the following url in a web browser
+ http://WEBSERVERHOST/ds/index.txt
+
+If things are working properlly you should see the following output
+
+# productID |Most recent |Time registered |type |Description |
+
+If an errror occurs check the web server error log and begin debugging.
+
+The following command may be used to create a test product in the data store
+
+dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --add testproduct --type psresults --description 'test data store product'
+
+Now if you point to http://WEBSERVERHOST/ds/index.txt with a web browser you
+should see
+
+# productID |Most recent |Time registered |type |Description |
+testproduct |none |2008-10-16T20:48:23Z|psresults|test data store product |
+
+(of course your timestamp will be different).
+
+As a final test we will add a fileset to the data store.
+
+echo this is a file > testfile.txt
+
+echo 'testfile.txt|||text|' | dsreg --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --product testproduct --add testfileset --type PSRESULTS --list - --copy --datapath .
+
+-list - tells dsreg to get the list of files for the fileset from stdin.
+(Usually a file is used). The input to the command gives the name of the file to be added to the
+fileset, blank entries for size and md5sum, and the file type 'text'.
+
+Now if you go to http://WEBSERVERHOST/ds/testfileset/index.txt you should see
+the fileset.
+
+If you leave off the index.txt in the urls you will get a more web browser
+friendly html pages with links.
+For example
+ http://WEBSERVERHOST/ds
+
+To delete the test product
+ dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ --del psresults --rm
+
+Set up the product for the postage stamp results with
+
+ dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --add pstampresults --type psresults --description 'Postage Stamp results'
+
+and a product for requests
+
+ dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --add pstamprequests --type psrequest --description 'Postage Stamp requests'
+
+Postage Stamp Server Set up
+---------------------------
+
+Now we are ready to set up the Postage Stamp Server.
+
+The programs that make up the PSS are controlled by a set of pantasks tasks.
+There are some configuration variables that must be added to the site.config file.
+
+ # root directory of the data store
+ # this must match the directory used above in the Data Store setup
+ DATA_STORE_ROOT STR /DATASTOREDIR/dsroot
+
+ # default data store product used for results
+ # (used by the web interface)
+ PSTAMP_DATA_STORE_PRODUCT STR pstampresults
+
+ # This directory is used for downloaded request files and other purposes
+ # it can go anywhere. The permissions on the directory must be such that
+ # that the web server program has permission to write there.
+ PSTAMP_WORKDIR STR /SOMEWHERE/pstamp-work
+
+PSS projects
+------------
+Each Postage stamp request selects images from a particular 'project'.
+Each project has a row in the table pstampProject has the following columns
+
+ column example
+ -------------------
+ name simtest
+ state enabled
+ dbname ps_simtest
+ dvodb NULL (currently not used)
+ camera SIMTEST
+ telescope SimScope
+ need_magic 0
+
+The column dbname contains the name of the database that is used to look up images.
+To create a project use pstamptool. In this example ps_simtest is a database
+refererencing images made by a simtest run.
+
+pstamptool -dbname mydatabase -addproject -name foo -imagedb ps_simtest -inst SIMTEST -telescope SimScope
+
+
+Postage Stamp Request Tables
+----------------------------
+
+Postage Stamp requests are submitted in the form of a FITS binary table. The
+format of this table is described in a document which may be found on the IPP wiki.
+http://kiawe.ifa.hawaii.edu/IPPwiki/index.php/PostageStampServer
+
+Only the first table extension found in the request file is processed.
+
+Each postage stamp request file submitted to the PSS generates a "Request".
+The name of the request is given by the value for the keyword REQ_NAME in
+the extension's fits header. Each postage stamp request must have a unique REQ_NAME must be unique.
+
+To process the request the PSS parses the request file.
+Each row in a request table contains a request specification. Each request specification
+causes zero or more "Jobs" to be queued for processing. Each Job operates on
+one input image.
+
+When all of the Jobs for a given Request complete, a Postage stamp results
+file is created and it along with the images produced by the jobs are
+registered in a fileset on the output Data Store.
+
+There are two job types specified by the value for the JOB_TYPE keyword in the
+request specification.
+
+ JOB_TYPE = "stamp" creates a "postage stamp" image.
+ JOB_TYPE = "get_image" builds a fileset containing copies of images
+
+There are two programs that may be used to create a postage stamp request.
+pstamprequest creates a request file with a single request specification.
+
+For example
+
+ pstamprequest -req_name REQ_00001 -project simtest /mydirectory/req.fits \
+ -byexp chip simtest.004.000 null \
+ -pixcenter 500 500 -pixrange 100 100
+
+creates a request file req.fits containing a request specification for a 100 x 100 pixel
+postage stamp image centered at (500,500) in the source.
+The source image is chip processed image for one of the object exposures from a simtest run.
+
+Another program that may be used to create a request file is located in
+pstamp/test/pstamp_req_create. This program takes a text file and creates a multi-row postage
+stamp request. A couple of example input files may be found in pstamp/test/*.txt
+
+Queueing a PostageStamp Request
+-------------------------------
+The postage stamp server looks for work to do by examining a collection of
+input Data Stores. For example, the MOPS team will have a datastore onto which
+they will submit their postage stamp requests.
+
+There will also be a web based interface. A prototype for this will be
+discussed in a later section of this document.
+
+For testing, we may queue a request by hand using the program pstamptool
+
+ pstamptool -dbname mydatabase -addreq -uri /mydirectory/req.fits
+
+This adds an entry to the table pstampRequests with state = 'new'
+
+Request Processing
+------------------
+
+Postage stamp requests are processed by a set of pantasks tasks.
+
+ Task Status
+ AV Name Njobs Ngood Nfail Ntime Command
+ ++ pstamp.request.find 921 921 0 0 pstamp_queue_requests.pl
+ ++ pstamp.request.run 13 13 0 0 pstamp_parser_run.pl
+ ++ request.finish.load 1839 1839 0 0 pstamptool
+ ++ request.finish.run 14 14 0 0 request_finish.pl
+ ++ pstamp.job.load 1840 1840 0 0 pstamptool
+ ++ pstamp.job.run 30 30 0 0 pstamp_job_run.pl
+ ++ pstamp.request.load 1566 1566 0 0 pstamptool
+
+pstamp.request.find examines incoming data stores for new requests and adds
+them if found
+
+pstamp.request.load dispatches pending requests for parsing
+
+pstamp.request.run runs the request parser which queues jobs to service the
+request specifications.
+
+pstamp.job.load dispatches pending jobs for processing
+
+pstamp.job.run runs a single job
+
+pstamp.finish.load looks for requests which have no pending jobs
+
+pstamp.finish.run builds the results fileset and registers it on the ouptut
+Data Store.
+
+Output Fileset
+-------------
+A successful request will generate a fileset with the name the REQ_NAME of the
+request in the output Data Store. The output product is determined by the
+source of the request as discussed in the next section. (If the request did
+not come from a data store the product is given by the value of PSTAMP_DATA_STORE_PRODUCT
+in the IPP site.config file.
+
+
+
+
Index: /branches/ccl_branches/ipponly-20191108/pstamp/doc/psnotes.txt
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/doc/psnotes.txt (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/doc/psnotes.txt (revision 41103)
@@ -0,0 +1,415 @@
+Notes on the IPP Postage Stamp Server
+
+Overview
+--------
+
+This note describes the installation and operation of the IPP Postage Stamp Server.
+We assume that IPP source for examination and that the user has the standard
+IPP configuration environment set up.
+
+This component of the IPP system is not intended for general distribution to IPP users
+and does not have streamlined installation procedures at this time.
+
+The PSS is a system which responds to requests to retrieve images from an IPP Image Server.
+The images may be either complete images or subsets of images known as "Postage Stamps".
+
+The internal mechanisms of the PSS are also used to service MOPS detectabiltiy
+query requests.
+
+Given this more general utility it should perhaps be called the "IPP Request Server".
+
+
+Data Store
+----------
+The PSS uses the Data Store mechanism to find incoming requests and to
+provide results to the end user.
+
+A DataStore is simply a web server with URLS that supports an interface
+that allows collections of data called File Sets to be located and accessed.
+
+A Data Store is organized into 3 levels. These are described in the following sections.
+
+Data Store Root - a list of Products
+-------------------------------------
+
+Assume that a datastore is located on the server WEBSERVERHOST.
+An HTTP get to the url http://WEBSERVERHOST/ds/index.txt would give a list of the
+products in the Data Store.
+
+ # productID |Most recent |Time registered |type |Description |
+ pstampresults|web204 |2008-10-16T00:35:16Z|psresults|Postage Stamp Results |
+ pstamprequests|TEST0012 |2008-08-29T03:26:40Z|psrequest|Postage Stamp Requests |
+ mops-pstamp-results|MOPSSTAMPREQ20080813T214918Z_00|2008-08-13T21:49:42Z|psresults|Postage Stamp Results for MOPS |
+
+Each line is divided into fields separated by a 'pipe' character | .
+
+Data Store Product - a list of filesets
+---------------------------------------
+
+Similarly the url http://WEBSERVERHOST/ds/pstampresults/index.txt gives the list of filesets
+in the product pstampresults:
+
+ # filesetID|time registered |type |
+ TEST0011 |2008-08-29T03:05:03Z|PSRESULTS|
+ TEST0012 |2008-08-29T03:31:36Z|PSRESULTS|
+ web190 |2008-10-15T02:29:11Z|PSRESULTS|
+ web191 |2008-10-15T02:32:16Z|PSRESULTS|
+ web192 |2008-10-15T02:38:50Z|PSRESULTS|
+ web193 |2008-10-15T03:08:57Z|PSRESULTS|
+ web194 |2008-10-15T23:32:11Z|PSRESULTS|
+ web195 |2008-10-15T23:35:39Z|PSRESULTS|
+ web196 |2008-10-15T23:52:40Z|PSRESULTS|
+ web197 |2008-10-15T23:59:05Z|PSRESULTS|
+ web198 |2008-10-16T00:04:17Z|PSRESULTS|
+ web200 |2008-10-16T00:17:46Z|PSRESULTS|
+ web204 |2008-10-16T00:35:16Z|PSRESULTS|
+
+Note that the fileset ID for the latest fileset (web204) in the list appears in the root listing in the last
+section.
+
+The product list protocol accepts a query string which can be used to limit the listing to all filesets
+registered after a given fileset.
+
+For example, http://WEBSERVERHOST/ds/pstampresults/index.txt?web198 yields
+
+ # filesetID|time registered |type |
+ web200 |2008-10-16T00:17:46Z|PSRESULTS|
+ web204 |2008-10-16T00:35:16Z|PSRESULTS|
+
+
+File Set - a list of files
+--------------------------
+http://ippds.somwhere.net/ds/pstampresults/web205/index.txt
+
+ # fileID |bytes |md5sum |type |
+ results.fits |28800 |4e2048479e551844e31102a301d50740|table |
+ # fileID |bytes |md5sum |type |chipname|
+ 1_1_o4748g0259o.ota01.fits|19840320|19a6233ab966cf3e13b7da8271dead67|chip |XY01 |
+ 1_2_o4748g0259o.ota02.fits|20416320|625bf8198dd7aef7c892af03584ee0ed|chip |XY02 |
+
+ .... followed by 58 other files in this GPC1 exposure
+
+
+The IPP includes a number of tools for accessing a data store. An introduction to these tools may
+be seen by using the program perldoc.
+
+ dsrootls
+ dsproductls
+ dsfilesetls
+ dsget
+
+
+Installing the IPP Data Store Server Implementation
+---------------------------------------------------
+
+The data store implementation consists of a set of cgi scripts which are
+invoked by an apache web server. The indexes for the Data Store are stored in a mysql
+database. The cgi scripts use DBI perl module to query the database. No database updates
+are performed by the Data Store cgi scripts.
+
+The files for the Data Store Server located in the IPP source tree in the directory
+DataStoreServer (this directory may not be available in the distributed IPP
+tarballs).
+
+Mysql data base configuration
+-----------------------------
+
+The tables for the PSS are part of the regular IPP database tables.
+
+The Data Store implementation has been designed to not require the IPP for its
+operation so the Data Store's tables are not created as part of the IPP database setup.
+
+Currently however, the PostageStamp Server implementation assumes that the data
+store and postage stamp tables are in the same mysql database (this will be
+fixed soon).
+
+To set up the database go to the directory DataStoreServer/scripts and start up mysql.
+
+If you do not already have a current IPP database that you would like to use
+for the Postage Stamp Request tables issue the following commands
+
+ mysql> create database mydatabase;
+ mysql> use mydatabase;
+ mysql> source scripts/tabledefs.sql;
+
+Now exit mysql and run the command
+ pxadmin -create -dbname mydatabase
+
+If you already have a database set up, then just issue the commands
+ mysql> use mydatabase;
+ mysql> source scripts/tabledefs.sql;
+
+Web Server Configuration
+------------------------
+The apache configuration needs to have the following entries added to
+the appropriate configuration file. (Different apache distributions have
+different locations for these files so it's up to the installer to
+figure out where these configuration lines should go).
+
+
+
+ DirectoryIndex /ds-cgi/dsindex.cgi
+
+ #Options None
+
+ # turn on server side includes
+ Options +Includes
+
+ # files will be processed for server side includes if they have the
+ # executable bit set
+ XBitHack on
+
+ AddType text/plain .txt
+ AddOutputFilter INCLUDES .txt
+
+ AllowOverride None
+
+ Order allow,deny
+ Allow from all
+
+
+ #
+ # Location of the cgi scripts
+ #
+
+
+ Options ExecCGI
+
+
+ Order allow,deny
+ Allow from all
+
+
+ Alias /ds /DATASTOREDIR/dsroot/
+ ScriptAlias /ds-cgi /DATASTOREDIR/ds-cgi/
+
+(another sample of this configuration file may be found in
+ DataStoreServer/web/conf/httpd-datastore.conf
+
+Here we have assumed that the data store cgi scripts will be installed in
+/DATASTOREDIR/ds-cgi and the data store data files will be stored in
+/DATASTOREDIR/dsroot.
+
+To install the cgi scripts
+ cd your_ipp_src_directory
+ cd DataStoreServer
+ scripts/installscripts --dsroot /DATASTOREDIR/dsroot --cgidir /DATASTOREDIR/ds-cgi
+
+The file /DATASTOREDIR/ds-cgi/dsshellconfig is a shell script that sets
+several environment variables that are required by the scripts.
+
+This script should be editied to customize the values for the installation.
+(This file will not be overwritten by subsequent invocations of
+installscripts).
+
+There are some scripts that the Postage Stamp Server uses that need to be
+installed into the IPP build directory. These are not included in the current IPP
+build. To build them invoke the following commands from the DataStoreServer
+directory.
+ psautogen
+ make install
+
+
+Testing the DataStore
+---------------------
+At this point an empty data store should be up and running. This can be tested
+by going to the following url in a web browser
+ http://WEBSERVERHOST/ds/index.txt
+
+If things are working properlly you should see the following output
+
+# productID |Most recent |Time registered |type |Description |
+
+If an errror occurs check the web server error log and begin debugging.
+
+The following command may be used to create a test product in the data store
+
+dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --add testproduct --type psresults --description 'test data store product'
+
+Now if you point to http://WEBSERVERHOST/ds/index.txt with a web browser you
+should see
+
+# productID |Most recent |Time registered |type |Description |
+testproduct |none |2008-10-16T20:48:23Z|psresults|test data store product |
+
+(of course your timestamp will be different).
+
+As a final test we will add a fileset to the data store.
+
+echo this is a file > testfile.txt
+
+echo 'testfile.txt|||text|' | dsreg --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --product testproduct --add testfileset --type PSRESULTS --list - --copy --datapath .
+
+-list - tells dsreg to get the list of files for the fileset from stdin.
+(Usually a file is used). The input to the command gives the name of the file to be added to the
+fileset, blank entries for size and md5sum, and the file type 'text'.
+
+Now if you go to http://WEBSERVERHOST/ds/testfileset/index.txt you should see
+the fileset.
+
+If you leave off the index.txt in the urls you will get a more web browser
+friendly html pages with links.
+For example
+ http://WEBSERVERHOST/ds
+
+To delete the test product
+ dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ --del psresults --rm
+
+Set up the product for the postage stamp results with
+
+ dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --add pstampresults --type psresults --description 'Postage Stamp results'
+
+and a product for requests
+
+ dsprodtool --dbname mydatabase --dsroot /DATASTOREDIR/dsroot/ \
+ --add pstamprequests --type psrequest --description 'Postage Stamp requests'
+
+Postage Stamp Server Set up
+---------------------------
+
+Now we are ready to set up the Postage Stamp Server.
+
+The programs that make up the PSS are controlled by a set of pantasks tasks.
+There are some configuration variables that must be added to the site.config file.
+
+ # root directory of the data store
+ # this must match the directory used above in the Data Store setup
+ DATA_STORE_ROOT STR /DATASTOREDIR/dsroot
+
+ # default data store product used for results
+ # (used by the web interface)
+ PSTAMP_DATA_STORE_PRODUCT STR pstampresults
+
+ # This directory is used for downloaded request files and other purposes
+ # it can go anywhere. The permissions on the directory must be such that
+ # that the web server program has permission to write there.
+ PSTAMP_WORKDIR STR /SOMEWHERE/pstamp-work
+
+PSS projects
+------------
+Each Postage stamp request selects images from a particular 'project'.
+Each project has a row in the table pstampProject has the following columns
+
+ column example
+ -------------------
+ name simtest
+ state enabled
+ dbname ps_simtest
+ dvodb NULL (currently not used)
+ camera SIMTEST
+ telescope SimScope
+ need_magic 0
+
+The column dbname contains the name of the database that is used to look up images.
+To create a project use pstamptool. In this example ps_simtest is a database
+refererencing images made by a simtest run.
+
+pstamptool -dbname mydatabase -addproject -name foo -imagedb ps_simtest -inst SIMTEST -telescope SimScope
+
+
+Postage Stamp Request Tables
+----------------------------
+
+Postage Stamp requests are submitted in the form of a FITS binary table. The
+format of this table is described in a document which may be found on the IPP wiki.
+http://kiawe.ifa.hawaii.edu/IPPwiki/index.php/PostageStampServer
+
+Only the first table extension found in the request file is processed.
+
+Each postage stamp request file submitted to the PSS generates a "Request".
+The name of the request is given by the value for the keyword REQ_NAME in
+the extension's fits header. Each postage stamp request must have a unique REQ_NAME must be unique.
+
+To process the request the PSS parses the request file.
+Each row in a request table contains a request specification. Each request specification
+causes zero or more "Jobs" to be queued for processing. Each Job operates on
+one input image.
+
+When all of the Jobs for a given Request complete, a Postage stamp results
+file is created and it along with the images produced by the jobs are
+registered in a fileset on the output Data Store.
+
+There are two job types specified by the value for the JOB_TYPE keyword in the
+request specification.
+
+ JOB_TYPE = "stamp" creates a "postage stamp" image.
+ JOB_TYPE = "get_image" builds a fileset containing copies of images
+
+There are two programs that may be used to create a postage stamp request.
+pstamprequest creates a request file with a single request specification.
+
+For example
+
+ pstamprequest -req_name REQ_00001 -project simtest /mydirectory/req.fits \
+ -byexp chip simtest.004.000 null \
+ -pixcenter 500 500 -pixrange 100 100
+
+creates a request file req.fits containing a request specification for a 100 x 100 pixel
+postage stamp image centered at (500,500) in the source.
+The source image is chip processed image for one of the object exposures from a simtest run.
+
+Another program that may be used to create a request file is located in
+pstamp/test/pstamp_req_create. This program takes a text file and creates a multi-row postage
+stamp request. A couple of example input files may be found in pstamp/test/*.txt
+
+Queueing a PostageStamp Request
+-------------------------------
+The postage stamp server looks for work to do by examining a collection of
+input Data Stores. For example, the MOPS team will have a datastore onto which
+they will submit their postage stamp requests.
+
+There will also be a web based interface. A prototype for this will be
+discussed in a later section of this document.
+
+For testing, we may queue a request by hand using the program pstamptool
+
+ pstamptool -dbname mydatabase -addreq -uri /mydirectory/req.fits
+
+This adds an entry to the table pstampRequests with state = 'new'
+
+Request Processing
+------------------
+
+Postage stamp requests are processed by a set of pantasks tasks.
+
+ Task Status
+ AV Name Njobs Ngood Nfail Ntime Command
+ ++ pstamp.request.find 921 921 0 0 pstamp_queue_requests.pl
+ ++ pstamp.request.run 13 13 0 0 pstamp_parser_run.pl
+ ++ request.finish.load 1839 1839 0 0 pstamptool
+ ++ request.finish.run 14 14 0 0 request_finish.pl
+ ++ pstamp.job.load 1840 1840 0 0 pstamptool
+ ++ pstamp.job.run 30 30 0 0 pstamp_job_run.pl
+ ++ pstamp.request.load 1566 1566 0 0 pstamptool
+
+pstamp.request.find examines incoming data stores for new requests and adds
+them if found
+
+pstamp.request.load dispatches pending requests for parsing
+
+pstamp.request.run runs the request parser which queues jobs to service the
+request specifications.
+
+pstamp.job.load dispatches pending jobs for processing
+
+pstamp.job.run runs a single job
+
+pstamp.finish.load looks for requests which have no pending jobs
+
+pstamp.finish.run builds the results fileset and registers it on the ouptut
+Data Store.
+
+Output Fileset
+-------------
+A successful request will generate a fileset with the name the REQ_NAME of the
+request in the output Data Store. The output product is determined by the
+source of the request as discussed in the next section. (If the request did
+not come from a data store the product is given by the value of PSTAMP_DATA_STORE_PRODUCT
+in the IPP site.config file.
+
+
+
+
Index: /branches/ccl_branches/ipponly-20191108/pstamp/ppstamp.pc.in
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/ppstamp.pc.in (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/ppstamp.pc.in (revision 41103)
@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@/@PACKAGE_NAME@
+
+Name: @PACKAGE_NAME@
+Description: Pan-STARRS Postage Stamper
+Version: @VERSION@
+Requires: pslib psmodules
+Libs: -L${libdir} @PPSTAMP_LIBS@
+Cflags: -I${includedir} @PPSTAMP_CFLAGS@
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/Makefile.am
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/Makefile.am (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/Makefile.am (revision 41103)
@@ -0,0 +1,41 @@
+
+installdir = $(bindir)
+
+install_files = \
+ pstamp_cleanup.pl \
+ pstamp_finish.pl \
+ pstamp_insert_request.pl \
+ pstamp_job_run.pl \
+ pstamp_listjobs.pl \
+ pstamp_new_request.sh \
+ pstampparse.pl \
+ pstamp_parser_run.pl \
+ pstamp_queue_cleanup.pl \
+ pstamp_queue_requests.pl \
+ pstamp_queue_update_cleanup.pl \
+ pstamp_request_file \
+ pstamp_results_file.pl \
+ pstamp_revert_request.pl \
+ pstamp_runcommand.sh \
+ pstamp_server_status \
+ pstamp_save_server_status.pl \
+ pstamp_webrequest.pl \
+ pstamp_get_image_job.pl \
+ psmkreq \
+ psgetcalibinfo \
+ psstatus \
+ pstampstopfaulted \
+ pstamp_checkdependent.pl \
+ request_finish.pl \
+ detectability_respond.pl \
+ detect_query_read \
+ detect_query_create \
+ detect_response_create \
+ dquery_job_run.pl \
+ dquery_finish.pl \
+ dqueryparse.pl \
+ fakedresponse.pl
+
+install_SCRIPTS = $(install_files)
+
+ACLOCAL_AMFLAGS = -I m4
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_query_create
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_query_create (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_query_create (revision 41103)
@@ -0,0 +1,374 @@
+#!/usr/bin/env perl
+
+# create a MOPS_DETECTABILITY_QUERY fits table from a text description file
+
+use warnings;
+use strict;
+
+use Astro::FITS::CFITSIO qw( :constants );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for output table
+
+my ( $input, # Name of input text file
+ $output, # Name of output table
+ $query_id,
+ $nostage,
+ $version,
+ );
+
+GetOptions(
+ 'input|i=s' => \$input,
+ 'output|o=s' => \$output,
+ 'query_id|q=s' => \$query_id,
+ 'version|v=s' => \$version,
+ 'nostage' => \$nostage,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --input --output",
+ -exitval => 3) unless defined $input and defined $output;
+
+# Read what we've been given
+my $in;
+if ($input eq '-') {
+ $in = \*STDIN;
+} else {
+ open $in, "<$input" or die "cannot open $input for reading";
+}
+my %colData;
+my %headerData;
+my $numRows = read_data_for_table($in,'\s+', \%colData, \%headerData);
+if (!$numRows) {
+ print STDERR "no data in $input\n";
+ exit 1;
+}
+
+# The header keywords
+my $header = [
+ { name => 'QUERY_ID',
+ writetype => TSTRING,
+ comment => 'MOPS Query ID for this batch query',
+ value => undef
+ },
+ { name => 'EXTVER',
+ writetype => TSTRING,
+ comment => 'Extension version',
+ value => undef
+ },
+ { name => 'FPA_ID',
+ writetype => TSTRING,
+ comment => 'orginal FPA_ID used at ingest',
+ value => undef
+ },
+ { name => 'MJD-OBS',
+ writetype => TDOUBLE,
+ comment => 'starting time of the exposure, MJD',
+ value => undef
+ },
+ { name => 'FILTER',
+ writetype => TSTRING,
+ comment => 'effective filter use for the exposure',
+ value => undef
+ },
+ { name => 'OBSCODE',
+ writetype => TSTRING,
+ comment => 'site identifier (MPC observatory code)',
+ value => undef
+ },
+ { name => 'STAGE',
+ writetype => TSTRING,
+ comment => 'processing stage to examine',
+ value => undef
+ }
+];
+
+# Validate header.
+
+if (defined($query_id)) {
+ $headerData{QUERY_ID} = $query_id;
+}
+unless (exists($headerData{QUERY_ID})) {
+ die "No QUERY_ID specified for header.";
+}
+
+unless (exists($headerData{EXTVER})) {
+ die "No EXTVER specified for header.";
+}
+if ($headerData{EXTVER} == 1) {
+ unless (exists($headerData{STAGE})) {
+ warn "No STAGE value specified in header. Assuming default value of 'diff'";
+ $headerData{STAGE} = 'diff';
+ }
+ foreach my $entry_ref (@{ $header }) {
+ my $name = $entry_ref->{name};
+ unless (exists($headerData{$name})) {
+ die "Required header value $name not specified (try EXTVER=2?).";
+ }
+ $entry_ref->{value} = $headerData{$name};
+ }
+}
+elsif ($headerData{EXTVER} == 2) {
+ unless (exists($headerData{STAGE})) {
+ warn "No STAGE value specified in header. Assuming default value of 'diff'";
+ $headerData{STAGE} = 'diff';
+ }
+ my $tmp_header;
+ foreach my $entry_ref (@{ $header }) {
+ my $name = $entry_ref->{name};
+ if (exists($headerData{$name})) {
+ $entry_ref->{value} = $headerData{$name};
+ push @{ $tmp_header }, $entry_ref;
+ }
+ }
+ $header = $tmp_header;
+}
+else {
+ die "Unknown EXTVER = $headerData{EXTVER}.";
+}
+
+# Specification of columns to write
+my $columns = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING },
+ # coordinate at start of exposure, in degrees
+ { name => 'RA1_DEG', type => 'D', writetype => TDOUBLE },
+ # coordinate at start of exposure, in degrees
+ { name => 'DEC1_DEG', type => 'D', writetype => TDOUBLE },
+ # coordinate at end of exposure, in degrees
+ { name => 'RA2_DEG', type => 'D', writetype => TDOUBLE },
+ # coordinate at end of exposure, in degrees
+ { name => 'DEC2_DEG', type => 'D', writetype => TDOUBLE },
+ # apparent magnitude
+ { name => 'MAG', type => 'D', writetype => TDOUBLE },
+];
+my $columns_v2 = [
+ { name => 'FPA_ID', type => '20A', writetype => TSTRING },
+ { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE },
+ { name => 'FILTER', type => '20A', writetype => TSTRING },
+ { name => 'OBSCODE', type => '20A', writetype => TSTRING },
+ { name => 'STAGE', type => '20A', writetype => TSTRING }
+ ];
+# Validate the data.
+if ($headerData{EXTVER} == 1) {
+ foreach my $entry_ref (@{ $columns }) {
+ my $name = $entry_ref->{name};
+ unless (exists($colData{$name})) {
+ die "Required data column $name not found (try EXTVER=2?).";
+ }
+ }
+}
+if ($headerData{EXTVER} == 2) {
+ my $tmp_columns;
+ foreach my $entry_ref (@{ $columns }) {
+ my $name = $entry_ref->{name};
+ if (exists($colData{$name})) {
+ push @{ $tmp_columns }, $entry_ref;
+ }
+ }
+ foreach my $entry_ref (@{ $columns_v2 }) {
+ my $name = $entry_ref->{name};
+ if (exists($colData{$name})) {
+ push @{ $tmp_columns }, $entry_ref;
+ }
+ }
+ $columns = $tmp_columns;
+}
+
+# Construct the array of arrays
+my @colDataAoA = ();
+foreach my $col_def (@{ $columns }) {
+ my $name = $col_def->{name};
+ push @colDataAoA, $colData{$name};
+}
+
+# foreach (@$columns) {
+# push @colData, [];
+# }
+
+
+
+my $status = make_fits_table($output, EXTNAME, $numRows, \@colDataAoA, $columns, $header);
+
+exit $status;
+
+# XXXXX: This should be in a module
+# two utility functions that may be used to create a FITS binary
+# table from hashes describing the header keywords and columns
+
+# read_table_description reads the data for a table from a simple text file
+# make_fits_table writes out the table to a named file
+
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+
+# A function to build a fits binary table from supplied data
+#
+sub make_fits_table {
+ my $output = shift; # name of output file
+ my $extname = shift; # extension name
+ my $numRows = shift; # number of rows in the table
+ my $colData = shift; # ref to array of arrays containing the data for each column
+ my $columns = shift; # ref to array of column descriptions (each a hash)
+ # with keys: name, type, and writetype
+ my $header = shift; # ref to array of header keyword descriptions - each a hash
+ # with keys: name, name, writetype, comment, and value
+ my $status = 0;
+
+ die "incorrect arguments" if !defined($columns);
+ # note $header can be nil
+
+ # build arrays for cfitsio
+ my @colNames; # Names of columns
+ my @colTypes; # Types of columns
+ my @colWriteType; # type to use to write
+
+ foreach my $colSpec ( @$columns) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ }
+
+ if (-e $output) {
+ unlink "$output" or die "failed to remove existing $output";
+ }
+
+ my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status ); # Output file handle
+ check_fitsio( $status );
+
+ $outFits->create_img( 16, 0, undef, $status );
+ check_fitsio( $status );
+
+ # Create the table
+
+ $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames,
+ \@colNames, \@colTypes, undef, $extname, $status );
+ check_fitsio( $status );
+
+ # if header keyword descriptions were provided add them
+ if ($header) {
+ foreach my $headerword ( @$header ) {
+ my $value = $headerword->{value};
+ unless (defined $value) {
+ print "Can't find header keyword $headerword\n";
+ next;
+ }
+ # zap quotation marks
+ $value =~ s/\'//g;
+ my $name = $headerword->{name};
+ my $type = $headerword->{writetype};
+ my $comment = $headerword->{comment};
+ $outFits->write_key( $type, $name, $value, $comment, $status );
+ check_fitsio( $status );
+ }
+ }
+
+
+ for (my $i = 0; $i < scalar @colNames; $i++) {
+ my $writeType = $colWriteType[$i];
+ $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status );
+ check_fitsio( $status );
+ }
+
+ $outFits->close_file( $status );
+
+ return 0;
+
+} # end of sub make_fits_table
+
+
+
+# read the table contents from a file
+#
+# input text file format:
+# lines that begin with '#' are comment lines and are skipped.
+# other lines are data. Each data line is split into fields with the
+# provided separator
+#
+# if $header is not null header the first non-commented line is read to
+# fill the value for each header keyword. The number of fields must match
+# the number of keywords.
+#
+# Following the optional header data, each data line contains data for each
+# row in the table. The number of fields must match the number of column
+# arrays provided.
+
+sub read_data_for_table {
+ my $in = shift; # input file handle
+ my $sep = shift; # string containing field separator
+ my $colData = shift; # reference to an array of arrays for the data
+ my $header = shift; # rerence to array of header keyword descriptions
+
+ my $line_num = 0;
+ my @keywords = ();
+ my $row_num;
+ while (<$in>) {
+ chomp;
+# print STDERR "$line_num $#keywords $_\n";
+ if ($_ =~ /^\s*$/) {
+ next;
+ }
+ if ($line_num == 0) {
+ if ($_ !~ /EXTVER/) {
+ next;
+ }
+ # Parse header information keywords
+ $_ =~ s/#//g;
+ @keywords = split /\s+/;
+ if ($keywords[0] eq '') {
+ shift(@keywords);
+ }
+ }
+ elsif ($line_num == 1) {
+ # Parse header information values
+ my @values = split /\s+/;
+ if ($#values != $#keywords) {
+ die "Number of header columns in input does not equal expected number of header words $#values $#keywords";
+ }
+ for (my $i = 0; $i <= $#values; $i++) {
+ $header->{$keywords[$i]} = $values[$i];
+ }
+ }
+ elsif ($line_num == 2) {
+ if ($_ !~ /ROWNUM/) {
+ next;
+ }
+ # Parse table information keywords, dumping old keywords
+ $_ =~ s/#//g;
+ @keywords = split /\s+/;
+ if ($keywords[0] eq '') {
+ shift(@keywords);
+ }
+ }
+ else {
+ # Parse table information values
+ unless ($_ =~ /^#/) {
+ my @values = split /\s+/;
+ if ($#values != $#keywords) {
+ die "Number of header columns in input does not equal expected number of header words $#values $#keywords";
+ }
+ for (my $i = 0; $i <= $#values; $i++) {
+ push @{ $colData{$keywords[$i]} }, $values[$i];
+ $row_num = $#{ $colData{$keywords[$i] } } + 1;
+ }
+ }
+ }
+ $line_num++;
+ }
+ # we return the number of rows read
+ return $row_num;
+}
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ die "CFITSIO error: $msg\n";
+ }
+}
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_query_read
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_query_read (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_query_read (revision 41103)
@@ -0,0 +1,408 @@
+#!/usr/bin/env perl
+
+# Read and parse a fits table containing a MOPS_DETECTABILITY_QUERY Extension
+# and optionally print out the contents
+
+# by default prints out the keywords in the exension header followed by the rows of the table
+# with labels
+# -l omits the labels
+# -h omits the header
+# -r omits the rows
+# so -lhr will print nothing
+
+use warnings;
+use strict;
+
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use Math::Trig;
+use Data::Dumper;
+use IPC::Cmd 0.36 qw( can_run run );
+
+use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for table
+
+my $no_print_label = 0; # omit the labels
+my $no_print_header = 0; # omit the header keywords
+my $no_print_rows = 0; # omit the rows
+my $version = 0;
+
+my ( $input, # Name of input text file
+ $output, # Name of output table
+ $save_temps, # Save temporary files?
+ $dbname, # needed to do camtool lookups.
+ );
+
+my $regtool = can_run('regtool') or (die "Can't find regtool");
+my $camtool = can_run('camtool') or (die "Can't find camtool");
+my $difftool = can_run('difftool') or (die "Can't find difftool");
+
+
+GetOptions(
+ 'input|i=s' => \$input,
+ 'output|o=s' => \$output,
+ 'dbname=s' => \$dbname,
+ 'nolabel|l' => \$no_print_label,
+ 'noheader|h' => \$no_print_header,
+ 'norows|r' => \$no_print_rows
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --input",
+ -exitval => 3) unless defined $input;
+
+if ($output && ($output ne "-")) {
+ die "cannot open $output for output\n" unless open OUTFILE, ">$output";
+ select OUTFILE;
+}
+
+my $status = 0;
+
+# Read the input file
+
+my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle
+check_fitsio($status);
+
+$inFits->movnam_hdu(BINARY_TBL, EXTNAME, 0, $status) and check_fitsio($status);
+
+my $inHeader = $inFits->read_header(); # Header for input
+
+my $numRows; # Number of rows in table
+$inFits->get_num_rows($numRows, $status) and check_fitsio($status);
+
+# The keywords found in the header
+
+my $parse_error = 0;
+# Parse the header to determine what we have to work with.
+# Clean up empty space and remove unneeded single quotes.
+foreach my $header_key (keys %{ $inHeader }) {
+ $inHeader->{$header_key} =~ s/\s+//g;
+ $inHeader->{$header_key} =~ s/\'//g;
+}
+my ($EXTVER,$headerEXTNAME,$QUERY_ID,
+ $FPA_ID,$MJD_OBS,$FILTER,
+ $OBSCODE,$STAGE) =
+ ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},
+ $inHeader->{FPA_ID},$inHeader->{'MJD-OBS'},$inHeader->{FILTER},
+ $inHeader->{OBSCODE},$inHeader->{STAGE});
+
+unless(defined($EXTVER) && defined($headerEXTNAME) &&
+ (($EXTVER == 1)||($EXTVER == 2)) &&
+ ($headerEXTNAME eq EXTNAME)&&
+ (defined($QUERY_ID))) {
+ $parse_error = 1;
+}
+unless(($EXTVER == 2)||
+ ((defined($QUERY_ID))&&(defined($FPA_ID))&&(defined($MJD_OBS))&&
+ (defined($FILTER))&&(defined($OBSCODE)))) {
+ $parse_error = 2;
+}
+
+# Specification of columns
+my $column_defs = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING, version => 1 },
+ # coordinate at start of exposure, in degrees
+ { name => 'RA1_DEG', type => 'D', writetype => TDOUBLE, version => 1 },
+ # coordinate at start of exposure, in degrees
+ { name => 'DEC1_DEG', type => 'D', writetype => TDOUBLE, version => 1 },
+ # coordinate at end of exposure, in degrees
+ { name => 'RA2_DEG', type => 'D', writetype => TDOUBLE, version => 1 },
+ # coordinate at end of exposure, in degrees
+ { name => 'DEC2_DEG', type => 'D', writetype => TDOUBLE, version => 1 },
+ # apparent magnitude
+ { name => 'MAG', type => 'D', writetype => TDOUBLE, version => 1 },
+ # v2 query_id: needs to be here.
+ { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID },
+ # v2 fpa_id: original FPA_ID used at ingest
+ { name => 'FPA_ID', type => '20A', writetype => TSTRING, version => 2, default => $FPA_ID },
+ # v2 mjd obs: starting time of the exposure, MJD
+ { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE, version => 2, default => $MJD_OBS },
+ # v2 filter: effective filter used for the exposure as a string, allowed values of g, r, i, z, y, B, V, w
+ { name => 'FILTER', type => '3A', writetype => TSTRING, version => 2, default => $FILTER },
+ # v2 obscode: site identifier (MPC observatory code)
+ { name => 'OBSCODE', type => '3A', writetype => TSTRING, version => 2, default => $OBSCODE },
+ # v2 stage: stage name to perform query on, allowed values of 'chip', 'warp', 'stack', and 'diff'
+ { name => 'STAGE', type => '20A', writetype => TSTRING, version => 2, default => $STAGE },
+];
+
+# Parse the list of columns
+my @colNames; # Names of columns
+my @colTypes; # Types of columns
+my @colWriteType; # type to use to write
+my %colData; # Data for each column referenced by name
+foreach my $colSpec ( @$column_defs) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+}
+
+foreach my $col (@$column_defs) {
+ my ($col_num, $col_type, $col_data);
+
+ if ($col->{version} > $EXTVER) {
+ @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
+ next;
+ }
+ else {
+ if (defined($col->{default})) {
+ @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
+ next;
+ }
+ }
+
+ $inFits->get_colnum(0, $col->{name}, $col_num, $status);
+ if ($status == 0) {
+ $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
+ $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
+ and check_fitsio($status);
+ $colData{$col->{name}} = $col_data;
+ if ($col->{name} eq 'QUERY_ID') {
+ print @{ $col_data } . "\n";
+ }
+
+ }
+ elsif ($status == 219) {
+ @{ $colData{$col->{name}} } = map { "Not_Set" } (0 .. $numRows - 1);
+ $status = 0;
+ }
+}
+
+# Set things that aren't set.
+for (my $i = 0; $i < $numRows; $i++) {
+# Simple stuff first.
+ my %known_filters = ();
+ if ($colData{STAGE}[$i] eq "Not_Set") {
+ $colData{STAGE}[$i] = 'diff';
+ }
+ if ($colData{OBSCODE}[$i] eq "Not_Set") {
+ $colData{OBSCODE}[$i] = 566;
+ }
+# Define filter and MJD from FPA_ID
+ if (($colData{FILTER}[$i] eq "Not_Set")&&($colData{FPA_ID}[$i] ne "Not_Set")) {
+ if (exists($known_filters{$colData{FPA_ID}[$i]})) {
+ $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]};
+ }
+ else {
+ my $cmd = "$regtool -processedexp -dbname $dbname -exp_name $colData{FPA_ID}[$i]";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $cmd, verbose => 0);
+ unless ($success) {
+ # This is a problem, because I'm not sure how we handle a failure to read something.
+ # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
+ # we can't send that response back.
+ die("Unable to perform $cmd error code: $error_code");
+ }
+ foreach my $entry (split /\n/, (join "", @$full_buf)) {
+ $entry =~ s/^\s+//;
+ my @line = split /\s+/, $entry;
+ if (scalar(@line) != 3) {
+ next;
+ }
+ my ($key,$type,$value) = @line;
+# print "$entry => $key / $type / $value \n";
+ if ($key =~ /filter/i) {
+ $value =~ s/.00000//;
+ $colData{FILTER}[$i] = $value;
+ $known_filters{$colData{FPA_ID}[$i]} = $value;
+ }
+ }
+ }
+ }
+ # Calculate a MJD
+ if (($colData{'MJD-OBS'}[$i] eq "Not_Set")&&($colData{FPA_ID}[$i] ne "Not_Set")) {
+ # HACK!
+ my $mjd = $colData{FPA_ID}[$i];
+ $mjd =~ s/o(....)g.*/$1/;
+ $mjd += 50000;
+ $colData{'MJD-OBS'}[$i] = $mjd;
+ }
+ # Calculate an exp_id
+ if (($colData{FPA_ID}[$i] eq "Not_Set")&&(($colData{'FILTER'}[$i] ne "Not_Set")&&
+ ($colData{'MJD-OBS'}[$i] ne "Not_Set"))) {
+ my $mjd_obs = int $colData{'MJD-OBS'}[$i];
+ my $dateobs_begin = mjd_to_dateobs($mjd_obs);
+ my $dateobs_end = mjd_to_dateobs($mjd_obs + 1);
+ my $ra = $colData{'RA1_DEG'}[$i];
+ my $dec = $colData{'DEC1_DEG'}[$i];
+ my $filter = $colData{'FILTER'}[$i] . ".00000";
+ my $cmd;
+ if ($colData{STAGE}[$i] eq 'SSdiff') {
+ # Assumption: All Stack Stack diffs are for a medium deep field
+ # go find which one the coordinates are in
+ # XXX: If we don't find one we'll just get the first SSdiff
+ # in the filter on the day in question
+ my $field = find_md_field($ra, $dec, 1.6);
+ $cmd = "$difftool -listssrun -dbname $dbname -filter $filter ";
+ $cmd .= " -mjd_obs_begin $mjd_obs -mjd_obs_end " . (1 + $mjd_obs);
+ $cmd .= " -limit 1";
+ $cmd .= " -tess_id $field%" if $field;
+ }
+ elsif ($colData{STAGE}[$i] eq 'WSdiff') {
+ $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
+ $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+ $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1 -diff_mode 2";
+ $cmd .= " -diff_mode 2";
+ }
+ elsif ($colData{STAGE}[$i] eq 'diff') {
+ $cmd = "$difftool -listrun -dbname $dbname -filter $filter ";
+ $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+ $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
+ }
+ else {
+ $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
+ $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+ $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
+ }
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $cmd, verbose => 1);
+ unless ($success) {
+ # This is a problem, because I'm not sure how we handle a failure to read something.
+ # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
+ # we can't send that response back.
+ die("Unable to perform $cmd error code: $error_code");
+ }
+ foreach my $entry (split /\n/, (join "", @$full_buf)) {
+ $entry =~ s/^\s+//;
+ my @line = split /\s+/, $entry;
+ if (scalar(@line) != 3) {
+ next;
+ }
+ my ($key,$type,$value) = @line;
+# print "$entry => $key / $type / $value \n";
+ if ($colData{STAGE}[$i] =~ /diff/) {
+ if ($key =~ /diff_id/i) {
+ $colData{FPA_ID}[$i] = $value;
+ }
+ }
+ else {
+ if ($key =~ /exp_name/i) {
+ $colData{FPA_ID}[$i] = $value;
+ }
+ }
+ }
+ }
+ # Correct diff modes
+ if ($colData{STAGE}[$i] eq 'WSdiff') {
+ $colData{STAGE}[$i] = 'diff';
+ }
+ elsif ($colData{STAGE}[$i] eq 'SSdiff') {
+ $colData{STAGE}[$i] = 'diff';
+ }
+}
+
+# Verify uniqueness of important columns:
+my @unique_fields = ('QUERY_ID','OBSCODE','STAGE');
+foreach my $colName (@unique_fields) {
+ my %counter = ();
+ my $i = 0;
+ foreach my $row (@{ $colData{$colName} }) {
+ $counter{$row} = 1;
+ }
+ if (scalar(keys(%counter)) != 1) {
+ $parse_error = 3;
+ }
+}
+if ($parse_error) {
+ die "Unable to parse detectability query: $parse_error " . &error_message($parse_error) . "\n";;
+}
+
+# # Now produce the output
+
+
+if (!$no_print_label) {
+ print "# ";
+ foreach my $col (@$column_defs) {
+ printf "%-12s ", $col->{name};
+ }
+ print "\n";
+}
+
+for (my $i = 0; $i < $numRows; $i++) {
+ foreach my $col (@$column_defs) {
+ printf "%-12s ", $colData{$col->{name}}->[$i];
+ #foreach my $aref (@col_arrays) {
+ #printf "%-12s ", $aref->[$i];
+ }
+ print "\n";
+}
+
+exit 0;
+
+### Pau.
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ die "CFITSIO error: $status => $msg\n";
+ }
+}
+
+sub error_message {
+ my $error = shift;
+ if ($error == 1) {
+ return("Unknown EXTVER/EXTNAME");
+ }
+ if ($error == 2) {
+ return("Required header field not found");
+ }
+ if ($error == 3) {
+ return("Unique column not uniquely specified");
+ }
+ else {
+ return("Unknown fault.");
+ }
+}
+
+# Stolen from PStamp/Job.pm. Thanks, Bill.
+sub mjd_to_dateobs {
+ my $mjd = shift;
+
+ my $ticks = ($mjd - 40587.0) * 86400;
+
+ my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
+
+ return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+}
+
+
+
+# Given an RA and DEC return the MD field that contains it if any.
+# This is much faster than actuallly examining the tessellation
+# using dvoImagesAtCoords
+
+sub find_md_field {
+ my $ra_deg = shift;
+ my $dec_deg = shift;
+ my $radius_deg = shift;
+
+ my @MD = qw(MD01 MD02 MD03 MD04 MD05 MD06 MD07 MD08 MD09 MD10);
+ my @MD_RA = (35.875, 53.10, 130.592, 150.000, 161.917, 185.000, 213.454, 242.787, 333.98, 352.312);
+ my @MD_DEC = (-4.25, -27.8, 44.317, 2.20, 58.083, 47.117, 53.243, 54.95, 0.283, -0.433);
+ my $PS_DEG_RAD = 0.017453292519943;
+
+ my $ra = $ra_deg * $PS_DEG_RAD;
+ my $dec = $dec_deg * $PS_DEG_RAD;
+ my $radius = $radius_deg * $PS_DEG_RAD;
+
+ for (my $i = 0; $i < scalar @MD; $i++) {
+
+ my $dist = acos((cos($PS_DEG_RAD * $MD_DEC[$i]) * cos($dec) * cos($PS_DEG_RAD * $MD_RA[$i] - $ra)) +(sin($PS_DEG_RAD * $MD_DEC[$i]) * sin($dec)));
+
+ if ($dist < $radius) {
+ return $MD[$i];
+ }
+ }
+ print STDERR "failed to find MD field containing $ra_deg $dec_deg\n";
+
+ return undef;
+}
+
+
+__END__
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_response_create
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_response_create (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detect_response_create (revision 41103)
@@ -0,0 +1,176 @@
+#!/usr/bin/env perl
+
+# program to create a simulated MOPS_DETECTABILITY_RESPONSE file
+# based on text format input file
+
+use warnings;
+use strict;
+
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use Math::Trig;
+use Data::Dumper;
+
+use constant EXTNAME => 'MOPS_DETECTABILITY_RESPONSE'; # Extension name for output table
+use constant EXTVER => 1;
+use constant OBSERVATORY_CODE => 566; # IAU Observatory Code
+
+my ( $input, # Name of input Detectabilty Query table
+ $output, # Name of output table
+ $save_temps, # Save temporary files?
+ );
+
+GetOptions(
+ 'input|i=s' => \$input,
+ 'output|o=s' => \$output,
+ 'save-temps' => \$save_temps,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --input --output",
+ -exitval => 3)
+ unless defined $input
+ and defined $output;
+
+my $status = 0;
+
+# Specification of columns to write
+my $columns = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING },
+
+ # number of pixels used in hypothetical PSF for the query detection
+ { name => 'DETECT_N', type => 'V', writetype => TULONG },
+
+ # detectibility, indicating the fraction of PSF pixels detetable by IPP
+ { name => 'DETECT_F', type => 'D', writetype => TDOUBLE },
+ ];
+
+# Header translation table
+my $headers = {
+ 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
+ comment => 'MOPS Query ID for this batch query' },
+ 'FPA_ID' => { name => 'FPA_ID', type => TSTRING,
+ comment => 'original FPA_ID used at ingest' },
+ };
+
+# Parse the list of columns
+my @colNames; # Names of columns
+my @colTypes; # Types of columns
+my %colData; # Data for each column
+my @colWriteType; # type to use to write
+foreach my $colSpec ( @$columns) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ $colData{$colSpec->{name}} = [];
+}
+
+my $in;
+if ($input eq "-") {
+ $in = *STDIN;
+} else {
+ die "cannot open input file $input" unless open $in, "<$input";
+}
+
+my $numRows;
+my $inHeader = { };
+
+read_input($in);
+
+
+# Write the output
+unlink "$output" if -e "$output";
+# Output file handle
+my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
+check_fitsio( $status );
+
+$outFits->create_img( 16, 0, undef, $status );
+check_fitsio( $status );
+
+# Start the table
+$outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, EXTNAME,
+ $status );
+check_fitsio( $status );
+
+# TODO: get the extension version number from somewhere common
+$outFits->write_key( TINT, 'EXTVER', EXTVER, 'Version of this Extension', $status );
+check_fitsio( $status );
+
+# Write the Extension keywords
+foreach my $keyword ( keys %$headers ) {
+ my $value = $inHeader->{$keyword}->{value}; # Header keyword value
+ unless (defined $value) {
+ print "Can't find header keyword $keyword\n";
+ next;
+ }
+ $value =~ s/\'//g;
+ my $name = $headers->{$keyword}->{name}; # New name
+ my $type = $headers->{$keyword}->{type}; # Type
+ my $comment = $headers->{$keyword}->{comment}; # Comment
+ $outFits->write_key( $type, $name, $value, $comment, $status );
+ check_fitsio( $status );
+}
+
+
+for (my $i = 0; $i < scalar @colNames; $i++) {
+ my $colName = $colNames[$i];# Column name
+ my $writeType = $colWriteType[$i];
+ $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status );
+ check_fitsio( $status );
+}
+
+$outFits->close_file( $status );
+
+### Pau.
+
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ die "CFITSIO error: $msg\n";
+ }
+}
+
+sub read_input
+{
+ my $inh = shift;
+ my $line;
+ # read data for header
+ while ($line = <$inh>) {
+ next if ($line =~ /^#/); # skip comment lines
+ chomp $line;
+ ($inHeader->{QUERY_ID}->{value}, $inHeader->{FPA_ID}->{value}, $inHeader->{MJD_OBS}->{value},
+ $inHeader->{FILTER}->{value}, $inHeader->{OBSCODE}->{value})
+ = split " ", $line;
+ last;
+ }
+ die "failed to read valid header words from $input"
+ unless defined($inHeader->{QUERY_ID}->{value}) &&
+ ($inHeader->{FPA_ID}->{value}) &&
+ ($inHeader->{MJD_OBS}->{value}) && ($inHeader->{FILTER}->{value}) &&
+ ($inHeader->{OBSCODE}->{value}) ;
+
+
+ $numRows = 0;
+ while ($line = <$inh> ) {
+ next if ($line =~ /^#/); # skip comment lines
+ chomp $line;
+
+ my ($rownum, $npix, $flux) = split " ", $line;
+
+ push @{$colData{'ROWNUM'}}, $rownum;
+ push @{$colData{'DETECT_N'}}, $npix;
+ push @{$colData{'DETECT_F'}}, $flux;
+ $numRows++;
+ }
+}
+
+__END__
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detectability_respond.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detectability_respond.pl (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/detectability_respond.pl (revision 41103)
@@ -0,0 +1,891 @@
+#! /usr/bin/env perl
+#
+#
+# Create a response to a MOPS_DETECTABILITY_QUERY
+#
+#
+
+use strict;
+use warnings;
+
+use Sys::Hostname;
+use Carp;
+use File::Basename;
+use File::Copy;
+use IPC::Cmd 0.36 qw( can_run run );
+use Getopt::Long qw( GetOptions );
+use Pod::Usage qw( pod2usage );
+use File::Temp qw( tempfile tempdir);
+
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
+use PS::IPP::Config qw( :standard );
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+
+#
+# Set up
+#
+my $host = hostname();
+
+my $EXTVER = 1.0;
+my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
+my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project);
+my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
+GetOptions(
+ 'input=s' => \$request_file,
+ 'output=s' => \$output,
+ 'workdir=s' => \$workdir,
+ 'job_id=s' => \$job_id,
+ 'dbname=s' => \$dbname,
+ 'dbserver=s' => \$dbserver,
+ 'verbose' => \$verbose,
+ 'save-temps' => \$save_temps,
+ 'ignore-wisdom' => \$ignore_wisdom,
+ ) or pod2usage(2);
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --input --output --dbname",
+ -exitval => 3,
+ ) unless
+ defined $request_file and defined $output and defined $workdir and defined $dbname;
+
+my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1);
+my $psphotForced = can_run('psphotForced') or (warn "Can't find psphotForced" and $missing_tools = 1);
+my $dquery_finish = can_run('dquery_finish.pl') or (warn "Can't find dquery_finish.pl" and $missing_tools = 1);
+my $ppCoord = can_run('ppCoord') or (warn "Can't find ppCoord" and $missing_tools = 1);
+if ($missing_tools) {
+ warn("Can't find required tools.");
+ exit($PS_EXIT_CONFIG_ERROR);
+}
+
+unless (defined $verbose) {
+ $verbose = 0;
+}
+
+my $ipprc = PS::IPP::Config->new();
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+
+if (!$dbserver) {
+ $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+# This is hardcoded in for the moment.
+$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver);
+my $imagedb = $project->{dbname};
+if (!$imagedb) {
+ carp("failed to find imagedb for project: $project");
+}
+
+my %query = ();
+my %image_list_hash = ();
+my $wisdom_file = "${workdir}/wisdom.dat";
+if ((-e $wisdom_file)&&!($ignore_wisdom)) {
+ print "Reading wisdom file $wisdom_file instead of parsing...\n";
+ open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
+ while() {
+ chomp;
+ my ($fpa_id,@key_values) = split /\s+/;
+ my $index = $#{ $query{$fpa_id}{ROWNUM} } + 1;
+ while ($#key_values > -1) {
+ my $key = shift(@key_values);
+ my $val = shift(@key_values);
+ # if we have wisdom, then we should have updated already. If not, we'll bomb out later in the code.
+ if ($key eq 'FAULT') {
+ $val = 0;
+ }
+ $query{$fpa_id}{$key}[$index] = $val;
+ }
+ }
+ close(WISDOM);
+} # End reading wisdom.
+else {
+ #
+ # Parse input request file using detect_query_read (as it's already written).
+ #
+ my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $dqr_command, verbose => $verbose);
+ unless ($success) {
+ # This is a problem, because I'm not sure how we handle a failure to read something.
+ # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
+ # we can't send that response back.
+ die("Unable to perform $dqr_command error code: $error_code");
+ }
+ my $Nrows = 0;
+ {
+ my @column_names = ();
+ # split output into lines skip until the line which lists the column names is found
+ # Parse subsequent lines
+ foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
+ if ($entry =~ /^#/) {
+ @column_names = split /\s+/, $entry;
+ shift(@column_names); # Dump the hash sign.
+ }
+ elsif (scalar @column_names) {
+ # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE
+ my %row_data;
+ @row_data{@column_names} = (split /\s+/, $entry);
+ for (my $i = 0; $i <= $#column_names; $i++) {
+ push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
+ $Nrows = scalar(keys(%query));
+ # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
+ }
+ }
+ }
+ }
+ #
+ # Identify target images. This should properly collate targets on a single imfile.
+ #
+ foreach my $fpa_id (keys %query) {
+ my %temp_hash;
+ my $query_style = 'byexp';
+ my $stage;
+ my $filter;
+ my $mjd;
+ # Confirm that we only have one stage/filter/mjd
+ if ($fpa_id ne 'Not_Set') { # We only need to check things that aren't the known odd case.
+ for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+ $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
+ $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
+ $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
+ }
+ if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
+ $stage = (keys(%{ $temp_hash{STAGE} }))[0];
+ }
+ else {
+ exit_with_failure(21,"Too many STAGEs specified");
+ }
+ if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
+ $filter = (keys(%{ $temp_hash{FILTER} }))[0];
+ }
+ else {
+ exit_with_failure(21,"Too many FILTERs specified");
+ }
+ if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
+ $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
+ }
+ else {
+ exit_with_failure(21,"Too many MJD-OBS specified");
+ }
+ }
+
+ # Set up a rowList with default values
+ my @rowList;
+ for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+ $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i];
+ $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i];
+ $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i];
+ $rowList[$i]->{COORD_MASK} = 0;
+ # Set default values
+ $query{$fpa_id}{BAD_COMPONENT}[$i] = 1;
+ $query{$fpa_id}{IMAGE}[$i] = 'no_image';
+ $query{$fpa_id}{MASK}[$i] = 'no_mask';
+ $query{$fpa_id}{WEIGHT}[$i] = 'no_weight';
+ $query{$fpa_id}{PSF}[$i] = 'no_psf';
+
+ $query{$fpa_id}{STAGE_ID}[$i] = 'no_id';
+ $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb';
+ $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic';
+ $query{$fpa_id}{MAGICKED}[$i] = 'no_magic';
+ $query{$fpa_id}{CATALOG}[$i] = 'no_catalog';
+ $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component';
+ $query{$fpa_id}{CLASS_ID}[$i] = 'no_class';
+
+ $query{$fpa_id}{STATE}[$i] = 'no_state';
+ $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate';
+ $query{$fpa_id}{FAULT}[$i] = 'no_fault';
+ $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate';
+ }
+
+ # Determine the query style for this fpa_id
+ if ($fpa_id =~ /o.*g.*o/) {
+ $query_style = 'byexp';
+ }
+ elsif ($fpa_id =~ /\d+/) {
+ $query_style = 'byid';
+ }
+ elsif ($fpa_id eq 'Not_Set') {
+ # no matching file was found skip
+ next;
+ }
+ else {
+ exit_with_failure(21,"Parse error in request file");
+ }
+ # Set common request components
+ my $option_mask |= 1;
+ $option_mask |= $PSTAMP_SELECT_IMAGE;
+ $option_mask |= $PSTAMP_SELECT_MASK;
+ $option_mask |= $PSTAMP_SELECT_VARIANCE;
+ $option_mask |= $PSTAMP_SELECT_PSF;
+ my $need_magic = 1;
+ if ($stage eq 'stack') {
+ $need_magic = 0;
+ }
+ my $mjd_min = int $mjd;
+ my $mjd_max = $mjd + 1;
+
+ my $req_filter;
+ if ($filter ne 'Not_Set') {
+ $req_filter = $filter . '%';
+ }
+
+ # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
+ my $pstamp_images_ref = locate_images($ipprc,$imagedb,
+ \@rowList,
+ $query_style,$stage,
+ $fpa_id,undef,undef,
+ $option_mask,$need_magic,
+ $mjd_min,$mjd_max,$req_filter,undef,$verbose);
+
+ foreach my $this_image_ref (@{ $pstamp_images_ref }) {
+
+ if (0) {
+ foreach my $key (sort (keys %{ $this_image_ref } )) {
+ my $value = ${ $this_image_ref }{$key};
+ if ($key eq 'row_index') {
+ $value = join ' ', @{ $this_image_ref->{$key} };
+ }
+ print "$this_image_ref $key $value\n";
+ }
+ }
+
+
+ foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
+ $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
+ $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
+ $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
+ $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
+ $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
+ $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
+ $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
+ $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0;
+
+ if (exists($this_image_ref->{astrom})) {
+ $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
+ }
+ else {
+ $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
+ }
+ if (exists($this_image_ref->{class_id})) {
+ $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
+ $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
+
+ }
+ else {
+ $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
+ $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
+ }
+
+ $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
+ if (exists($this_image_ref->{data_state})) {
+ $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
+ }
+ else {
+ $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
+ }
+ $query{$fpa_id}{FAULT}[$valid_index] = 0;
+ $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked};
+ if ($stage eq 'chip') {
+ $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
+ }
+
+ # Determine if the data exists.
+ if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or
+ ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
+ ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or
+ ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or
+ ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or
+ ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
+
+ # image is gone and it's not coming back
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
+ }
+ elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
+ }
+ elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
+
+ # updating stacks isn't implemented
+ if (($stage eq 'stack')) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+ }
+ # updating old burntool data isn't implemented
+ elsif ($stage eq 'chip') {
+ if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and
+ (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+ }
+ }
+ } # End determining error faults.
+ }
+ }
+ }
+} # End calculating wisdom
+my %update_request;
+my %processing_request;
+
+if ($job_id) {
+ $wisdom_file = "$wisdom_file.$job_id";
+}
+open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
+foreach my $fpa_id (keys %query) {
+ for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
+ print WISDOM "$fpa_id\t";
+ foreach my $key (keys %{ $query{$fpa_id} }) {
+ print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
+ }
+ print WISDOM "\n";
+ my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
+ if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
+ if ($data_state ne 'full') {
+
+ @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
+ ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
+ $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
+ }
+ push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
+ }
+ }
+}
+close(WISDOM);
+
+if (! defined $job_id) {
+ # If there is anything that needs to be updated, create the update request list, and then exit the program.
+ my $exit_code = 0;
+ my $update_request_file = "${workdir}/update_request.dat";
+ open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
+ foreach my $images (keys %update_request) {
+ foreach my $fault (keys %{ $update_request{$images} }) {
+ if ($fault == 25) {
+ $exit_code = 25;
+ }
+ elsif ($fault != 0) {
+ $exit_code = 21;
+ }
+ my $update_request = join ' ', @{ $update_request{$images}{$fault} };
+ print UPDATE_REQUEST "$update_request\n";
+ }
+ }
+ close(UPDATE_REQUEST);
+ if ($exit_code != 0) {
+ exit($exit_code);
+ }
+}
+
+# This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read
+# from the wisdom.dat file. This means only one pass on all that potentially slow stuff. If that's the case, then
+# I can recalculate the processing request.
+
+foreach my $fpa_id (keys %processing_request) {
+ foreach my $image (keys %{ $processing_request{$fpa_id} }) {
+ print "$fpa_id $image\t";
+ foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
+ print "$i ";
+ }
+ print "\n";
+ }
+}
+
+# run ppCoord and psphotForced to calculate the required data.
+foreach my $fpa_id (keys %processing_request) {
+ foreach my $image (keys %{ $processing_request{$fpa_id} }) {
+ # Get this image specific data from the first entry. That entry is now the king of this set.
+ my $index = $processing_request{$fpa_id}{$image}[0];
+ my $fault = $query{$fpa_id}{FAULT}[$index];
+ my $catalog = $query{$fpa_id}{CATALOG}[$index];
+ my $psf = $query{$fpa_id}{PSF}[$index];
+ my $mask = $query{$fpa_id}{MASK}[$index];
+ my $weight= $query{$fpa_id}{WEIGHT}[$index];
+ my $stage = $query{$fpa_id}{STAGE}[$index];
+ my $stage_id = $query{$fpa_id}{STAGE_ID}[$index];
+ my $component = $query{$fpa_id}{COMPONENT_ID}[$index];
+ # print "Input is from $stage $stage_id $component\n";
+
+ # if there's a fault or quality problem, then we can't process this image.
+ # if (($fault != 0)||($query{$fpa_id}{BAD_COMPONENT}[$index] == 1)) {
+ if ($fault = check_component($stage, $stage_id, $component, $imagedb)) {
+ foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
+ $query{$fpa_id}{PROC_ERROR}[$i] = $fault;
+
+ $query{$fpa_id}{NPIX}[$i] = 0;
+ $query{$fpa_id}{QFACTOR}[$i] = 0.0;
+ $query{$fpa_id}{FLUX}[$i] = 0.0;
+ $query{$fpa_id}{FLUX_SIG}[$i] = 0.0;
+ }
+
+ next;
+ }
+
+ # Create coordinate file to convert to positions.
+ my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$index.XXXX",
+ UNLINK => !$save_temps);
+ my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$index.XXXX",
+ UNLINK => !$save_temps);
+ foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
+ print $coordfile "$query{$fpa_id}{RA1_DEG}[$i] $query{$fpa_id}{DEC1_DEG}[$i]\n";
+ }
+
+ # Convert the sky coordinates to image coordinates with ppCoord.
+ my $command = "ppCoord -astrom $catalog -radec $coordname";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("Unable to perform $command. Error_code: $error_code",
+ $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
+ $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],
+ $error_code);
+ }
+ my @response = split /\n/, (join "", @$stdout_buf);
+ my $i = 0;
+ foreach my $line (@response) {
+ my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
+ print $targetfile "$r_x $r_y\n";
+ if ((abs($r_ra - $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]]) < 1e-8)&&
+ (abs($r_dec - $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]]) < 1e-8)) {
+ $query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x;
+ $query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y;
+ $query{$fpa_id}{EXTENSION_BASE}[$processing_request{$fpa_id}{$image}[$i]] = $r_chip;
+ }
+ else {
+ $error_code = $PS_EXIT_PROG_ERROR;
+ my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) $error_code",
+ $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
+ $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],
+ $error_code);
+ }
+ $i++;
+ }
+
+ # Run psphotForced on the target list.
+ my $tmpdir = tempdir("detect.$index.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
+ my $outroot = "$tmpdir/detectability.${stage}.${fpa_id}.${index}";
+ $query{$fpa_id}{PROC_ERROR}[$index] = 0;
+ my $psphot_cmd = "$psphotForced -psf $psf ";
+ $psphot_cmd .= "-file $image ";
+ $psphot_cmd .= "-mask $mask ";
+ $psphot_cmd .= "-variance $weight ";
+ $psphot_cmd .= "-srctext $targetname $outroot ";
+ $psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 ";
+
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $psphot_cmd, verbose => $verbose);
+ unless ($success) {
+ $query{$fpa_id}{PROC_ERROR}[$index] = $PSTAMP_SYSTEM_ERROR;
+ }
+
+ # Why not parse out results here?
+# Convert psphot output to response
+ if ($query{$fpa_id}{PROC_ERROR}[$index] == 0) {
+ my $class_id = $query{$fpa_id}{CLASS_ID}[$index];
+ my $cmf = "${outroot}.${class_id}.cmf";
+ my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$query{$fpa_id}{EXTENSION_BASE}[$index]);
+ foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) {
+ my $result_index = $processing_request{$fpa_id}{$image}[$i];
+ $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index];
+
+ $query{$fpa_id}{NPIX}[$result_index] = ${ $tmp_Npix }[$i];
+ $query{$fpa_id}{QFACTOR}[$result_index] = ${ $tmp_Qfact }[$i];
+ $query{$fpa_id}{FLUX}[$result_index] = ${ $tmp_flux }[$i];
+ $query{$fpa_id}{FLUX_SIG}[$result_index] = ${ $tmp_flux_error }[$i];
+
+# print "$fpa_id $image $#{ $processing_request{$fpa_id}{$image} } $result_index $i ${ $tmp_Npix }[$i]\n";
+ }
+ }
+ else {
+ foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) {
+ my $result_index = $processing_request{$fpa_id}{$image}[$i];
+ $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index];
+
+ $query{$fpa_id}{NPIX}[$result_index] = 0;
+ $query{$fpa_id}{QFACTOR}[$result_index] = 0.0;
+ $query{$fpa_id}{FLUX}[$result_index] = 0.0;
+ $query{$fpa_id}{FLUX_SIG}[$result_index] = 0.0;
+ }
+ }
+
+
+ }
+}
+
+write_response_file($output,\%query);
+
+#
+# Utilities
+#
+
+# Taken largely from detect_query_read
+sub read_cmf_file {
+ my $cmf_file = shift;
+ my $class_id = shift;
+ my $extname = $class_id . ".psf";
+
+ my @tmp_Npix = ();
+ my @tmp_Qfact = ();
+ my @tmp_flux = ();
+ my @tmp_flux_err = ();
+
+ my ($detect_F_col,$detect_N_col,$detect_flux_col,$detect_mag_col) = (-1, -1, -1, -1);
+
+
+ my $status = 0;
+ my $inFits = Astro::FITS::CFITSIO::open_file( $cmf_file, READONLY, $status ); # Open CMF
+ check_fitsio($status);
+ $inFits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status);
+
+ my $inHeader = $inFits->read_header();
+
+ my $CMFversion = $inHeader->{EXTTYPE};
+ $CMFversion =~ s/\'//g;
+ $CMFversion =~ s/\s+//g;
+
+ # This is the only data we actually care about
+ my $column_defs;
+ if ($CMFversion eq 'PS1_V2') {
+ $column_defs = [
+ { name => 'PSF_QF', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_NPIX', type => '1J', writetype => TLONG },
+ { name => 'PSF_INST_MAG', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_INST_MAG_SIG', type => '1E', writetype => TDOUBLE },
+ ];
+ }
+ elsif ($CMFversion eq 'PS1_DV1') {
+ $column_defs = [
+ { name => 'PSF_QF', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_NPIX', type => '1J', writetype => TLONG },
+ { name => 'PSF_INST_FLUX', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_INST_FLUX_SIG', type => '1E', writetype => TDOUBLE },
+ ];
+ }
+
+ my @colNames;
+ my @colTypes;
+ my @colWriteType;
+ my %colData;
+ foreach my $colSpec (@$column_defs) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ }
+ my $numRows;
+ $inFits->get_num_rows($numRows, $status) and check_fitsio($status);
+
+ my $correct_error = 0;
+# print STDERR "Ncols:" . $#{ $column_defs } . "\n";
+ foreach my $col (@$column_defs) {
+ my ($col_num,$col_type,$col_data);
+ $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status);
+ if ($status != 0) {
+ $status = 0;
+ next;
+ }
+ $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
+ $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) and check_fitsio($status);
+# print STDERR "$col\t>>" . $col->{name} . "<<\t>>" . @{ $col_data } . "<<\n";
+ if ($col->{name} eq 'PSF_QF') {
+ @tmp_Qfact = @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_NPIX') {
+ @tmp_Npix = @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_INST_MAG') {
+ @tmp_flux = map { $_ = 10**(-0.4 * $_) } @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_INST_MAG_SIG') {
+ @tmp_flux_err = map { $_ = $_ / (2.5 * (log(exp(1)) / log(10))) } @{ $col_data };
+ $correct_error = 1;
+ }
+ elsif ($col->{name} eq 'PSF_INST_FLUX') {
+ @tmp_flux = @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_INST_FLUX_SIG') {
+ @tmp_flux_err = @{ $col_data };
+ }
+ }
+ if ($correct_error) {
+ for (my $c = 0; $c <= $#tmp_flux_err; $c++) {
+ $tmp_flux_err[$c] = $tmp_flux_err[$c] * $tmp_flux[$c];
+ }
+ }
+ $inFits->close_file( $status ) and check_fitsio($status);
+# print STDERR "$CMFversion\n";
+# print STDERR "Q: $#tmp_Qfact\t@tmp_Qfact\nN: $#tmp_Npix\t@tmp_Npix\n";
+ return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux, \@tmp_flux_err);
+}
+
+# A lot of this pulled verbatim from Bill's detect_response_create file
+sub write_response_file {
+ my $outfile = shift;
+ my $query_ref = shift;
+
+ my %query = %{ $query_ref };
+
+ my $columns;
+ my $headers;
+
+ my $EXTVER_IS_1 = (scalar(keys(%query)) == 1);
+# print "EXTVER: $EXTVER_IS_1\n";
+ my ($query_id,$FPA_ID,$MJD_OBS,$filter,$obscode,$status);
+ if ($EXTVER_IS_1 == 1) {
+ # Specification of columns to write
+ $columns = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING },
+ # any errors that occurred during processing
+ { name => 'ERROR_CODE', type => 'V', writetype => TULONG },
+ # number of pixels used in hypothetical PSF for the query detection
+ { name => 'DETECT_N', type => 'V', writetype => TULONG },
+ # detectibility, indicating the fraction of PSF pixels detetable by IPP
+ { name => 'DETECT_F', type => 'D', writetype => TDOUBLE },
+ # flux of the target source
+ { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+ # error in the flux of the target source
+ { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+ ];
+
+ # Header translation table
+ $headers = {
+ 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
+ comment => 'MOPS Query ID for this batch query' },
+ 'FPA_ID' => { name => 'FPA_ID', type => TSTRING,
+ comment => 'original FPA_ID used at ingest' },
+# 'MJD-OBS' => { name => 'FPA_ID', type => TSTRING,
+# comment => 'original FPA_ID used at ingest' },
+# 'FILTER' => { name => 'FPA_ID', type => TSTRING,
+# comment => 'original FPA_ID used at ingest' },
+# 'OBSCODE' => { name => 'FPA_ID', type => TSTRING,
+# comment => 'original FPA_ID used at ingest' },
+ };
+ }
+ else {
+ # Specification of columns to write
+ $columns = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING},
+ # any errors that occurred during processing
+ { name => 'ERROR_CODE', type => 'V', writetype => TULONG },
+ # number of pixels used in hypothetical PSF for the query detection
+ { name => 'DETECT_N', type => 'V', writetype => TULONG },
+ # detectibility, indicating the fraction of PSF pixels detetable by IPP
+ { name => 'DETECT_F', type => 'D', writetype => TDOUBLE },
+ # flux of the target source
+ { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+ # error in the flux of the target source
+ { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+ # The FPA That would be in the header if it were to be there.
+ { name => 'FPA_ID', type => '20A', writetype => TSTRING },
+ ];
+
+ # Header translation table
+ $headers = {
+ 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
+ comment => 'MOPS Query ID for this batch query' },
+ };
+ }
+
+ # Parse the list of columns
+ my @colNames; # Names of columns
+ my @colTypes; # Types of columns
+ my %colData; # Data for each column
+ my @colWriteType; # type to use to write
+ foreach my $colSpec ( @$columns) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ $colData{$colSpec->{name}} = [];
+ }
+
+ my $inHeader = { };
+
+ # Hack to force the data to match the detect_response_create formats
+
+ $inHeader->{QUERY_ID}->{value} = $query_id;
+ if ($EXTVER_IS_1 == 1) {
+ my $fpa_id = (keys(%query))[0];
+ $inHeader->{FPA_ID}->{value} = $fpa_id;
+ }
+
+ # Fill the table columns with the data, making sure the flux is defined
+ foreach my $fpa_id (keys %query) {
+ $inHeader->{QUERY_ID}->{value} = $query{$fpa_id}{QUERY_ID}[0];
+ if ($EXTVER_IS_1 == 1) {
+ my $fpa_id = (keys(%query))[0];
+ $inHeader->{FPA_ID}->{value} = $fpa_id;
+ }
+
+ my @keysa = keys %{$query{$fpa_id}};
+
+ print "$fpa_id: @keysa\n";
+
+ push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} };
+ push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} };
+ push @{$colData{'DETECT_N'}}, @{ $query{$fpa_id}{NPIX} };
+ push @{$colData{'DETECT_F'}}, @{ $query{$fpa_id}{QFACTOR} };
+ push @{$colData{'TARGET_FLUX'}}, @{ $query{$fpa_id}{FLUX} };
+ push @{$colData{'TARGET_FLUX_SIG'}}, @{ $query{$fpa_id}{FLUX_SIG} };
+ if ($EXTVER_IS_1 != 1) {
+ push @{$colData{'FPA_ID'}}, (map {$fpa_id} @{ $query{$fpa_id}{ROWNUM} });
+ }
+ }
+ my $numRows = 0;
+ # Back to detect_response_create:
+ $status = 0;
+# print "$output\n";
+ if (-e $output) {
+ unlink $output or die "failed to unlink existing response file $output\n";
+ }
+ my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
+ check_fitsio( $status );
+ $outFits->create_img( 16, 0, undef, $status );
+ check_fitsio( $status );
+ my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
+
+ $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status);
+ check_fitsio( $status );
+
+ # TODO: get the extension version number from somewhere common
+ $outFits->write_key( TINT, 'EXTVER', $EXTVER, 'Version of this Extension', $status );
+ check_fitsio( $status );
+
+ # Write the Extension keywords
+ foreach my $keyword ( keys %$headers ) {
+ my $value = $inHeader->{$keyword}->{value}; # Header keyword value
+ unless (defined $value) {
+ print "Can't find header keyword $keyword\n";
+ next;
+ }
+ $value =~ s/\'//g;
+ my $name = $headers->{$keyword}->{name}; # New name
+ my $type = $headers->{$keyword}->{type}; # Type
+ my $comment = $headers->{$keyword}->{comment}; # Comment
+ $outFits->write_key( $type, $name, $value, $comment, $status );
+ check_fitsio( $status );
+ }
+
+ for (my $i = 0; $i < scalar @colNames; $i++) {
+ my $colName = $colNames[$i];# Column name
+ my $writeType = $colWriteType[$i];
+ my $numRows = scalar(@{$colData{$colName}});
+ unless(defined($writeType)) {
+ print "write type undefined for $colName\n";
+ }
+ unless(defined($numRows)) {
+ print "num Rows undefined for $colName\n";
+ }
+ unless(defined($status)) {
+ print "status undefined for $colName\n";
+ }
+ unless(defined($colData{$colName})) {
+ print "col data undefined for $colName\n";
+ }
+ unless(defined($colName)) {
+ print "column name undefined for $i\n";
+ }
+ print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n";
+ $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status );
+ print STDERR "$writeType $i $numRows $colName $status\n";
+ check_fitsio( $status );
+ }
+ $outFits->close_file( $status );
+ return($status);
+}
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ carp("CFITSIO error: $status => $msg");
+ die "CFITSIO error: $msg\n";
+ }
+}
+
+sub check_component {
+ my ($stage, $stage_id, $component, $imagedb) = @_;
+
+ print "Checking status of component $stage $stage_id $component\n";
+
+ my $command;
+ if ($stage eq 'diff') {
+ $command = "difftool -diffskyfile -diff_id $stage_id -skycell_id $component";
+ } elsif ($stage eq 'stack') {
+ $command = "stacktool -sumskyfile -stack_id $stage_id";
+ } elsif ($stage eq 'warp') {
+ $command = "warptool -warped -warp_id $stage_id -skycell_id $component";
+ } elsif ($stage eq 'chip') {
+ $command = "chiptool -processedimfile -chip_id $stage_id -class_id $component";
+ } else {
+ die("check_component not implemented for stage $stage yet");
+ }
+
+ $command .= " -dbname $imagedb";
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my $rc = $error_code >> 8;
+ carp "$command failed $error_code $rc";
+ exit $PS_EXIT_UNKNOWN_ERROR;
+ }
+ my $output = join "", @$stdout_buf;
+ my $metadata = $mdcParser->parse($output);
+ my $results = parse_md_list($metadata);
+ if (scalar @$results != 1) {
+ carp "$command returned too many components: " . scalar @$results;
+ exit $PS_EXIT_UNKNOWN_ERROR;
+ }
+ my $it = $results->[0];
+
+ if ($stage eq 'stack') {
+ $it->{data_state} = $it->{state};
+ }
+
+ my $return_status = 0;
+ if ($it->{quality}) {
+ print " Bad quality: $it->{quality}\n";
+ $return_status = $PSTAMP_GONE;
+ } elsif ($it->{fault}) {
+ print " Faulted: $it->{fault}\n";
+ $return_status = $PSTAMP_GONE;
+ } elsif ($it->{data_state} ne 'full') {
+ # XXX does this work for stack?
+ carp " Faulted: $it->{data_state}\n";
+ $return_status = $PSTAMP_GONE;
+ } else {
+ print " Component ok.\n";
+ }
+
+ return $return_status;
+}
+
+sub my_die {
+ my $message = shift;
+ my $QUERY_ID = shift;
+ my $FPA_ID = shift;
+ my $MJD_OBS = shift;
+ my $FILTER = shift;
+ my $OBSCODE = shift;
+ my $STAGE = shift;
+
+ my $exit_code = shift; # Exit code to add
+
+ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+ carp("$message : $QUERY_ID $FPA_ID $MJD_OBS $FILTER $OBSCODE $STAGE");
+ exit($exit_code);
+
+}
+
+sub exit_with_failure {
+ my $status = shift;
+ my $message = shift;
+ carp("$message");
+ exit($status);
+}
+
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dquery_finish.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dquery_finish.pl (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dquery_finish.pl (revision 41103)
@@ -0,0 +1,400 @@
+#!/bin/env perl
+
+# dquery_finish.pl
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+use Sys::Hostname;
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Copy;
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config qw( :standard );
+
+my ( $req_id, $req_name, $req_file, $product, $outdir, $dbname, $dbserver, $verbose, $save_temps );
+
+GetOptions(
+ 'req_id=s' => \$req_id,
+ 'req_name=s' => \$req_name,
+ 'req_file=s' => \$req_file,
+ 'product=s' => \$product,
+ 'outdir=s' => \$outdir,
+ 'dbname=s' => \$dbname,
+ 'dbserver=s' => \$dbserver,
+ 'verbose' => \$verbose,
+ 'save-temps' => \$save_temps,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+
+my $err = "";
+
+$err .= "--req_id is required\n" if !$req_id;
+$err .= "--req_name is required\n" if !$req_name;
+$err .= "--product is required\n" if !$product;
+$err .= "--outdir is required\n" if !$outdir;
+
+die "$err" if $err;
+
+my $missing_tools;
+
+my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
+my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
+
+if ($missing_tools) {
+ warn("Can't find required tools.");
+ exit ($PS_EXIT_CONFIG_ERROR);
+}
+
+my $ipprc = PS::IPP::Config->new(); # IPP Configuration
+
+my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
+exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message
+
+if (!$dbserver) {
+ $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+
+if ($product eq "NULL") {
+ # parsing failed just with fault = 0 (this leaves previously set fault in place
+ update_request($req_id, 0, $verbose);
+ exit 0;
+}
+
+if (! -e $outdir ) {
+ # something must have gone wrong at the parse stage
+ print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
+ if (! mkdir $outdir ) {
+ update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+ die "cannot create output directory $outdir";
+ }
+} elsif (! -d $outdir) {
+ update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+ die "output fileset directory $outdir exists but is not a directory";
+}
+
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $request_fault = 0;
+my @jobs;
+{
+ my $command = "$pstamptool -listjob -req_id $req_id";
+ $command .= " -jobType detect_query"; # temporary
+ $command .= " -dbname $dbname" if $dbname;
+ $command .= " -dbserver $dbserver" if $dbserver;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ die("Unable to perform $command error code: $error_code");
+ }
+ my $output = join "", @$stdout_buf;
+ if (!$output) {
+ if ($verbose) {
+ print STDERR "Request $req_id produced no jobs.\n";
+ }
+ # assume that the parser set the fault
+ } else {
+ my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc for jobs list");
+ my $jobs = parse_md_list($metadata);
+ @jobs = @$jobs;
+ }
+}
+
+my ($REGLIST, $reg_list) = tempfile("$outdir/reglist.XXXX", UNLINK => !$save_temps);
+my $response_file = "response.$req_id.fits";
+my $response_path = "$outdir/$response_file";
+
+print $REGLIST "$response_file|||table|\n";
+
+my @columns = qw(ROWNUM PROC_ERROR NPIX QFACTOR FLUX FLUX_SIG FPA_ID STAGE STAGE_ID COMPONENT CMFFILE);
+my %colData;
+foreach my $col (@columns) {
+ $colData{$col} = [];
+}
+
+my $i = 0;
+foreach my $job (@jobs) {
+ my $job_id = $job->{job_id};
+
+ if ($job->{fault}) {
+ # how do we get this information back to the user?
+ print STDERR "job: $job_id faulted with $job->{fault}\n";
+ next;
+ }
+ my $outputBase = $job->{outputBase};
+ my $results = "${outputBase}results.txt";
+
+ read_results($results, \%colData);
+
+ my $cmf = $colData{CMFFILE}->[$i];
+ print $REGLIST "$cmf|||table|\n";
+
+ $i++;
+}
+close $REGLIST;
+
+write_response_file($response_path, $req_name, \%colData);
+
+if (-s $reg_list) {
+ my $command = "$dsreg --add $req_name --product $product --list $reg_list";
+ $command .= " --copy --datapath $outdir";
+ $command .= " --type MOPS_DETECTABILITY_RESPONSE";
+ $command .= " --ps0 $req_id";
+ $command .= " --dbname $dbname" if $dbname;
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ $request_fault = $error_code >> 8;
+ print STDERR "Unable to perform $command return code: $request_fault";
+ # fall through to call stop request
+ }
+}
+
+update_request($req_id, $request_fault, $verbose);
+
+exit 0;
+
+sub read_results {
+ my $results_file = shift;
+ my $results = shift;
+
+ open IN, "<$results_file" or my_die("failed to open $results_file\n", $PS_EXIT_UNKNOWN_ERROR);
+
+ foreach my $line () {
+ # skip header
+ next if $line =~ /ROWNUM/;
+ chomp $line;
+ my @words = split " ", $line;
+ my ($rownum, $proc_error, $npix, $qfactor, $flux, $flux_sig, $fpa_id, $stage, $stage_id, $component, $cmf) = @words;
+ push @{$results->{ROWNUM}}, $rownum;
+ push @{$results->{PROC_ERROR}}, $proc_error;
+ push @{$results->{NPIX}}, $npix;
+ push @{$results->{QFACTOR}}, $qfactor;
+ push @{$results->{FLUX}}, $flux;
+ push @{$results->{FLUX_SIG}}, $flux_sig;
+ push @{$results->{FPA_ID}}, $fpa_id;
+ push @{$results->{STAGE}}, $stage;
+ push @{$results->{STAGE_ID}}, $stage_id;
+ push @{$results->{COMPONENT}}, $component;
+ push @{$results->{CMFFILE}}, $cmf;
+ }
+ # XXX: make sure that the size of each of these arrays is the same
+}
+
+sub write_response_file {
+ my $output = shift;
+ my $query_id = shift;
+ my $results = shift;
+
+ my $columns;
+ my $headers;
+
+ my $EXTVER_IS_1 = 0 ; # (scalar(keys(%query)) == 1);
+# print "EXTVER: $EXTVER_IS_1\n";
+ my ($FPA_ID,$MJD_OBS,$filter,$obscode,$status);
+ if ($EXTVER_IS_1 == 1) {
+ # Specification of columns to write
+ $columns = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING },
+ # any errors that occurred during processing
+ { name => 'ERROR_CODE', type => 'V', writetype => TULONG },
+ # number of pixels used in hypothetical PSF for the query detection
+ { name => 'DETECT_N', type => 'V', writetype => TULONG },
+ # detectibility, indicating the fraction of PSF pixels detetable by IPP
+ { name => 'DETECT_F', type => 'D', writetype => TDOUBLE },
+ # flux of the target source
+ { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+ # error in the flux of the target source
+ { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+ ];
+
+ # Header translation table
+ $headers = {
+ 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
+ comment => 'MOPS Query ID for this batch query' },
+ 'FPA_ID' => { name => 'FPA_ID', type => TSTRING,
+ comment => 'original FPA_ID used at ingest' },
+# 'MJD-OBS' => { name => 'FPA_ID', type => TSTRING,
+# comment => 'original FPA_ID used at ingest' },
+# 'FILTER' => { name => 'FPA_ID', type => TSTRING,
+# comment => 'original FPA_ID used at ingest' },
+# 'OBSCODE' => { name => 'FPA_ID', type => TSTRING,
+# comment => 'original FPA_ID used at ingest' },
+ };
+ }
+ else {
+ # Specification of columns to write
+ $columns = [
+ # matching rownum from detectability original request
+ { name => 'ROWNUM', type => '20A', writetype => TSTRING},
+ # any errors that occurred during processing
+ { name => 'ERROR_CODE', type => 'V', writetype => TULONG },
+ # number of pixels used in hypothetical PSF for the query detection
+ { name => 'DETECT_N', type => 'V', writetype => TULONG },
+ # detectibility, indicating the fraction of PSF pixels detetable by IPP
+ { name => 'DETECT_F', type => 'D', writetype => TDOUBLE },
+ # flux of the target source
+ { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+ # error in the flux of the target source
+ { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+ # The FPA That would be in the header if it were to be there.
+ { name => 'FPA_ID', type => '20A', writetype => TSTRING },
+ ];
+
+ # Header translation table
+ $headers = {
+ 'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
+ comment => 'MOPS Query ID for this batch query' },
+ };
+ }
+
+ # Parse the list of columns
+ my @colNames; # Names of columns
+ my @colTypes; # Types of columns
+ my %colData; # Data for each column
+ my @colWriteType; # type to use to write
+ foreach my $colSpec ( @$columns) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ $colData{$colSpec->{name}} = [];
+ }
+
+ my $inHeader = { };
+
+ # Hack to force the data to match the detect_response_create formats
+
+ $inHeader->{QUERY_ID}->{value} = $query_id;
+ if ($EXTVER_IS_1 == 1) {
+ my $fpa_id = $results->{FPA_ID}->[0];
+ $inHeader->{FPA_ID}->{value} = $fpa_id;
+ }
+
+ # Fill the table columns with the data, making sure the flux is defined
+
+ @{$colData{'ROWNUM'}} = @{ $results->{ROWNUM} };
+ @{$colData{'ERROR_CODE'}} = @{ $results->{PROC_ERROR} };
+ @{$colData{'DETECT_N'}} = @{ $results->{NPIX} };
+ @{$colData{'DETECT_F'}} = @{ $results->{QFACTOR} };
+ @{$colData{'TARGET_FLUX'}} = @{ $results->{FLUX} };
+ @{$colData{'TARGET_FLUX_SIG'}} = @{ $results->{FLUX_SIG} };
+ @{$colData{'FPA_ID'}} = @{ $results->{FPA_ID} };
+
+ my $numRows = 0;
+ # Back to detect_response_create:
+ $status = 0;
+# print "$output\n";
+ if (-e $output) {
+ unlink $output or die "failed to unlink existing response file $output\n";
+ }
+ my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
+ check_fitsio( $status );
+ $outFits->create_img( 16, 0, undef, $status );
+ check_fitsio( $status );
+ my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
+
+ $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status);
+ check_fitsio( $status );
+
+ # TODO: get the extension version number from somewhere common
+ $outFits->write_key( TINT, 'EXTVER', 2, 'Version of this Extension', $status );
+ check_fitsio( $status );
+
+ # Write the Extension keywords
+ foreach my $keyword ( keys %$headers ) {
+ my $value = $inHeader->{$keyword}->{value}; # Header keyword value
+ unless (defined $value) {
+ print "Can't find header keyword $keyword\n";
+ next;
+ }
+ $value =~ s/\'//g;
+ my $name = $headers->{$keyword}->{name}; # New name
+ my $type = $headers->{$keyword}->{type}; # Type
+ my $comment = $headers->{$keyword}->{comment}; # Comment
+ $outFits->write_key( $type, $name, $value, $comment, $status );
+ check_fitsio( $status );
+ }
+
+ for (my $i = 0; $i < scalar @colNames; $i++) {
+ my $colName = $colNames[$i];# Column name
+ my $writeType = $colWriteType[$i];
+ my $numRows = scalar(@{$colData{$colName}});
+ unless(defined($writeType)) {
+ print "write type undefined for $colName\n";
+ }
+ unless(defined($numRows)) {
+ print "num Rows undefined for $colName\n";
+ }
+ unless(defined($status)) {
+ print "status undefined for $colName\n";
+ }
+ unless(defined($colData{$colName})) {
+ print "col data undefined for $colName\n";
+ }
+ unless(defined($colName)) {
+ print "column name undefined for $i\n";
+ }
+ print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n";
+ $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status );
+ print STDERR "$writeType $i $numRows $colName $status\n";
+ check_fitsio( $status );
+ }
+ $outFits->close_file( $status );
+ return($status);
+}
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ carp("CFITSIO error: $status => $msg");
+ die "CFITSIO error: $msg\n";
+ }
+}
+sub update_request {
+ my $req_id = shift;
+ my $fault = shift;
+ my $verbose = shift;
+
+ my $command = "$pstamptool -updatereq -req_id $req_id";
+ if ($fault) {
+ $command .= " -set_fault $fault";
+ } else {
+ $command .= " -set_state stop";
+ }
+ $command .= " -dbname $dbname" if $dbname;
+ $command .= " -dbserver $dbserver" if $dbserver;
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ die("Unable to perform $command error code: $error_code");
+ }
+}
+
+sub my_die {
+ my $msg = shift;
+ my $fault = shift;
+
+ print STDERR $msg;
+
+ update_request($req_id, $fault);
+
+ exit $fault;
+}
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dquery_job_run.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dquery_job_run.pl (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dquery_job_run.pl (revision 41103)
@@ -0,0 +1,400 @@
+#! /usr/bin/env perl
+#
+#
+# Run a detectability query job
+#
+#
+
+use strict;
+use warnings;
+
+use Sys::Hostname;
+use Carp;
+use File::Basename;
+use File::Copy;
+use IPC::Cmd 0.36 qw( can_run run );
+use Getopt::Long qw( GetOptions );
+use Pod::Usage qw( pod2usage );
+use File::Temp qw( tempfile tempdir);
+
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
+use PS::IPP::Config qw( :standard );
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use Astro::FITS::CFITSIO qw( :constants );
+Astro::FITS::CFITSIO::PerlyUnpacking(1);
+
+#
+# Set up
+#
+my $host = hostname();
+
+my $EXTVER = 1.0;
+my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
+my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project);
+my ($output_base,$dbname,$dbserver,$verbose,$save_temps);
+GetOptions(
+ 'output_base=s' => \$output_base,
+ 'job_id=s' => \$job_id,
+ 'dbname=s' => \$dbname,
+ 'dbserver=s' => \$dbserver,
+ 'verbose' => \$verbose,
+ 'save-temps' => \$save_temps,
+ ) or pod2usage(2);
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --job_id --output_base",
+ -exitval => 3,
+ ) unless
+ defined $job_id and defined $output_base;
+
+my $psphotForced = can_run('psphotForced') or (warn "Can't find psphotForced" and $missing_tools = 1);
+my $ppCoord = can_run('ppCoord') or (warn "Can't find ppCoord" and $missing_tools = 1);
+if ($missing_tools) {
+ warn("Can't find required tools.");
+ exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $workdir = dirname($output_base);
+my $params_file = "${output_base}params.mdc";
+
+unless (defined $verbose) {
+ $verbose = 0;
+}
+
+my $ipprc = PS::IPP::Config->new();
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+if (!$dbserver) {
+ $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+
+my $imagedb = 'gpc1';
+my %args;
+
+open IN, "<$params_file"
+ or my_die("failed to open params file: $params_file", "", "", "", "", "", "", $PS_EXIT_UNKNOWN_ERROR);
+
+my $data = $mdcParser->parse(join "", ())
+ or my_die("failed to parse paramsfile: $params_file", "", "", "", "", "", "", $PS_EXIT_UNKNOWN_ERROR);
+my $refs = parse_md_list($data);
+if (scalar @$refs < 2) {
+ my_die("invalid paramsfile: $params_file", "", "", "", "", "", "", $PS_EXIT_UNKNOWN_ERROR);
+}
+
+my $params = shift @$refs;
+my @rows = @$refs;
+
+my $nRows = scalar @rows;
+print "Read $nRows rows from $params_file\n";
+
+#exit 0;
+
+# run ppCoord and psphotForced to calculate the required data.
+my $fpa_id = $params->{FPA_ID};
+my $fault = $params->{FAULT};
+my $catalog = $params->{CATALOG};
+my $psf = $params->{PSF};
+my $image = $params->{IMAGE};
+my $mask = $params->{MASK};
+my $weight= $params->{WEIGHT};
+my $stage = $params->{STAGE};
+my $stage_id = $params->{STAGE_ID};
+my $component = $params->{COMPONENT_ID};
+# print "Input is from $stage $stage_id $component\n";
+
+# XXX: TODO: if there's a fault or quality problem, then we can't process this image.
+# Perhaps we shouldn't even get here
+
+# Create coordinate file to convert to positions.
+my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$job_id.XXXX",
+ UNLINK => !$save_temps);
+my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$job_id.XXXX",
+ UNLINK => !$save_temps);
+foreach my $row (@rows) {
+ print $coordfile "$row->{RA1_DEG} $row->{DEC1_DEG}\n";
+}
+
+# Convert the sky coordinates to image coordinates with ppCoord.
+my $command = "ppCoord -astrom $catalog -radec $coordname";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+unless ($success) {
+ my_die("Unable to perform $command. Error_code: $error_code",
+ $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'},
+ $params->{FILTER},$params->{OBSCODE},$params->{STAGE},
+ $error_code);
+}
+my @response = split /\n/, (join "", @$stdout_buf);
+my $i = 0;
+foreach my $line (@response) {
+ my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
+ print $targetfile "$r_x $r_y\n";
+ my $row = $rows[$i];
+ if ((abs($r_ra - $row->{RA1_DEG}) < 1e-8) && (abs($r_dec - $row->{DEC1_DEG}) < 1e-8)) {
+ $row->{X_PXL} = $r_x;
+ $row->{Y_PXL} = $r_y;
+ $row->{EXTENSION_BASE} = $r_chip;
+ } else {
+ $error_code = $PS_EXIT_PROG_ERROR;
+ my_die("Unable to match input RA/DEC with output RA/DEC: ($row->{RA1_DEG},$row->{DEC1_DEG}) -> ($r_ra,$r_dec) $error_code",
+ $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'},
+ $params->{FILTER},$params->{OBSCODE},$params->{STAGE},
+ $error_code);
+ }
+ $i++;
+}
+
+# Run psphotForced on the target list.
+my $outroot = "${output_base}detectability.${stage}.${stage_id}";
+$outroot .= ".$component" unless $stage eq 'chip';
+
+$params->{PROC_ERROR} = 0;
+my $psphot_cmd = "$psphotForced -psf $psf ";
+$psphot_cmd .= "-file $image ";
+$psphot_cmd .= "-mask $mask ";
+$psphot_cmd .= "-variance $weight ";
+$psphot_cmd .= "-srctext $targetname $outroot ";
+$psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 ";
+
+( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $psphot_cmd, verbose => $verbose);
+unless ($success) {
+ # $params->{PROC_ERROR} = $PSTAMP_SYSTEM_ERROR;
+ my $rc = $error_code >> 8;
+ my_die("$command failed: $error_code $rc",
+ $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'},
+ $params->{FILTER},$params->{OBSCODE},$params->{STAGE},
+ $rc);
+}
+
+my $results = "${output_base}results.txt";
+open OUT, ">$results" or
+ my_die("Failed to open results file $results",
+ $params->{QUERY_ID},$fpa_id,$params->{'MJD-OBS'},
+ $params->{FILTER},$params->{OBSCODE},$params->{STAGE},
+ $PS_EXIT_UNKNOWN_ERROR);
+
+# Read the output cmf and produce the data we need for the response file
+my $class_id = $params->{CLASS_ID};
+my $cmf = "${outroot}.${class_id}.cmf";
+my $cmfbase = basename($cmf);
+my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$rows[0]->{EXTENSION_BASE});
+$i = 0;
+print OUT "ROWNUM PROC_ERROR NPIX QFACTOR FLUX FLUX_SIG FPA_ID STAGE STAGE_ID COMPONENT CMFFILE\n";
+foreach my $row (@rows) {
+ $row->{PROC_ERROR} = $params->{PROC_ERROR};
+
+ $row->{NPIX} = ${ $tmp_Npix }[$i];
+ $row->{QFACTOR} = ${ $tmp_Qfact }[$i];
+ $row->{FLUX} = ${ $tmp_flux }[$i];
+ $row->{FLUX_SIG} = ${ $tmp_flux_error }[$i];
+
+ print OUT "$row->{ROWNUM} $row->{PROC_ERROR} $row->{NPIX} $row->{QFACTOR} $row->{FLUX} $row->{FLUX_SIG} $fpa_id $stage $stage_id $component $cmfbase\n";
+ $i++;
+}
+close(OUT);
+
+exit 0;
+
+#
+# Utilities
+#
+
+# Taken largely from detect_query_read
+sub read_cmf_file {
+ my $cmf_file = shift;
+ my $class_id = shift;
+ my $extname = $class_id . ".psf";
+
+ my @tmp_Npix = ();
+ my @tmp_Qfact = ();
+ my @tmp_flux = ();
+ my @tmp_flux_err = ();
+
+ my ($detect_F_col,$detect_N_col,$detect_flux_col,$detect_mag_col) = (-1, -1, -1, -1);
+
+
+ my $status = 0;
+ my $inFits = Astro::FITS::CFITSIO::open_file( $cmf_file, READONLY, $status ); # Open CMF
+ check_fitsio($status);
+ $inFits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status);
+
+ my $inHeader = $inFits->read_header();
+
+ my $CMFversion = $inHeader->{EXTTYPE};
+ $CMFversion =~ s/\'//g;
+ $CMFversion =~ s/\s+//g;
+
+ # This is the only data we actually care about
+ my $column_defs;
+ if ($CMFversion eq 'PS1_V2') {
+ $column_defs = [
+ { name => 'PSF_QF', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_NPIX', type => '1J', writetype => TLONG },
+ { name => 'PSF_INST_MAG', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_INST_MAG_SIG', type => '1E', writetype => TDOUBLE },
+ ];
+ }
+ elsif ($CMFversion eq 'PS1_DV1') {
+ $column_defs = [
+ { name => 'PSF_QF', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_NPIX', type => '1J', writetype => TLONG },
+ { name => 'PSF_INST_FLUX', type => '1E', writetype => TDOUBLE },
+ { name => 'PSF_INST_FLUX_SIG', type => '1E', writetype => TDOUBLE },
+ ];
+ }
+
+ my @colNames;
+ my @colTypes;
+ my @colWriteType;
+ my %colData;
+ foreach my $colSpec (@$column_defs) {
+ push @colNames, $colSpec->{name};
+ push @colTypes, $colSpec->{type};
+ push @colWriteType, $colSpec->{writetype};
+ }
+ my $numRows;
+ $inFits->get_num_rows($numRows, $status) and check_fitsio($status);
+
+ my $correct_error = 0;
+# print STDERR "Ncols:" . $#{ $column_defs } . "\n";
+ foreach my $col (@$column_defs) {
+ my ($col_num,$col_type,$col_data);
+ $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status);
+ if ($status != 0) {
+ $status = 0;
+ next;
+ }
+ $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
+ $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) and check_fitsio($status);
+# print STDERR "$col\t>>" . $col->{name} . "<<\t>>" . @{ $col_data } . "<<\n";
+ if ($col->{name} eq 'PSF_QF') {
+ @tmp_Qfact = @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_NPIX') {
+ @tmp_Npix = @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_INST_MAG') {
+ @tmp_flux = map { $_ = 10**(-0.4 * $_) } @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_INST_MAG_SIG') {
+ @tmp_flux_err = map { $_ = $_ / (2.5 * (log(exp(1)) / log(10))) } @{ $col_data };
+ $correct_error = 1;
+ }
+ elsif ($col->{name} eq 'PSF_INST_FLUX') {
+ @tmp_flux = @{ $col_data };
+ }
+ elsif ($col->{name} eq 'PSF_INST_FLUX_SIG') {
+ @tmp_flux_err = @{ $col_data };
+ }
+ }
+ if ($correct_error) {
+ for (my $c = 0; $c <= $#tmp_flux_err; $c++) {
+ $tmp_flux_err[$c] = $tmp_flux_err[$c] * $tmp_flux[$c];
+ }
+ }
+ $inFits->close_file( $status ) and check_fitsio($status);
+# print STDERR "$CMFversion\n";
+# print STDERR "Q: $#tmp_Qfact\t@tmp_Qfact\nN: $#tmp_Npix\t@tmp_Npix\n";
+ return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux, \@tmp_flux_err);
+}
+
+
+# From Astro::FITS::CFITSIO demo
+sub check_fitsio
+{
+ my $status = shift; # Status of FITSIO calls
+
+ if ($status != 0) {
+ my $msg; # Message to output
+ Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
+ carp("CFITSIO error: $status => $msg");
+ die "CFITSIO error: $msg\n";
+ }
+}
+
+sub check_component {
+ my ($stage, $stage_id, $component, $imagedb) = @_;
+
+ print "Checking status of component $stage $stage_id $component\n";
+
+ my $command;
+ if ($stage eq 'diff') {
+ $command = "difftool -diffskyfile -diff_id $stage_id -skycell_id $component";
+ } elsif ($stage eq 'stack') {
+ $command = "stacktool -sumskyfile -stack_id $stage_id";
+ } elsif ($stage eq 'warp') {
+ $command = "warptool -warped -warp_id $stage_id -skycell_id $component";
+ } elsif ($stage eq 'chip') {
+ $command = "chiptool -processedimfile -chip_id $stage_id -class_id $component";
+ } else {
+ die("check_component not implemented for stage $stage yet");
+ }
+
+ $command .= " -dbname $imagedb";
+
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my $rc = $error_code >> 8;
+ carp "$command failed $error_code $rc";
+ exit $PS_EXIT_UNKNOWN_ERROR;
+ }
+ my $output = join "", @$stdout_buf;
+ my $metadata = $mdcParser->parse($output);
+ my $results = parse_md_list($metadata);
+ if (scalar @$results != 1) {
+ carp "$command returned too many components: " . scalar @$results;
+ exit $PS_EXIT_UNKNOWN_ERROR;
+ }
+ my $it = $results->[0];
+
+ if ($stage eq 'stack') {
+ $it->{data_state} = $it->{state};
+ }
+
+ my $return_status = 0;
+ if ($it->{quality}) {
+ print " Bad quality: $it->{quality}\n";
+ $return_status = $PSTAMP_GONE;
+ } elsif ($it->{fault}) {
+ print " Faulted: $it->{fault}\n";
+ $return_status = $PSTAMP_GONE;
+ } elsif ($it->{data_state} ne 'full') {
+ # XXX does this work for stack?
+ carp " Faulted: $it->{data_state}\n";
+ $return_status = $PSTAMP_GONE;
+ } else {
+ print " Component ok.\n";
+ }
+
+ return $return_status;
+}
+
+sub my_die {
+ my $message = shift;
+ my $QUERY_ID = shift;
+ my $FPA_ID = shift;
+ my $MJD_OBS = shift;
+ my $FILTER = shift;
+ my $OBSCODE = shift;
+ my $STAGE = shift;
+
+ my $exit_code = shift; # Exit code to add
+
+ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+ carp("$message : $QUERY_ID $FPA_ID $MJD_OBS $FILTER $OBSCODE $STAGE");
+ exit($exit_code);
+
+}
+
+sub exit_with_failure {
+ my $status = shift;
+ my $message = shift;
+ carp("$message");
+ exit($status);
+}
+
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dqueryparse.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dqueryparse.pl (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/dqueryparse.pl (revision 41103)
@@ -0,0 +1,553 @@
+#! /usr/bin/env perl
+#
+#
+# Create a response to a MOPS_DETECTABILITY_QUERY
+#
+#
+
+use strict;
+use warnings;
+
+use Sys::Hostname;
+use Carp;
+use File::Basename;
+use File::Copy;
+use IPC::Cmd 0.36 qw( can_run run );
+use Getopt::Long qw( GetOptions );
+use Pod::Usage qw( pod2usage );
+use File::Temp qw( tempfile tempdir);
+use POSIX; # for strftime
+
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
+use PS::IPP::Config qw( :standard );
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+# use Astro::FITS::CFITSIO qw( :constants );
+# Astro::FITS::CFITSIO::PerlyUnpacking(1);
+
+my $host = hostname();
+
+my $mode = 'queue_job';
+
+my ($req_id, $product, $need_magic, $missing_tools, $project, $label);
+my ($request_file, $outdir, $dbname, $dbserver, $verbose, $save_temps, $no_update);
+GetOptions(
+ 'file=s' => \$request_file,
+ 'req_id=s' => \$req_id,
+ 'outdir=s' => \$outdir,
+ 'label=s' => \$label,
+ 'product=s' => \$product,
+ 'mode=s' => \$mode,
+ 'dbname=s' => \$dbname,
+ 'dbserver=s' => \$dbserver,
+ 'verbose' => \$verbose,
+ 'save-temps' => \$save_temps,
+ 'no-update' => \$no_update,
+ ) or pod2usage(2);
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --file --req_id --outdir --dbname",
+ -exitval => 3,
+ ) unless
+ defined $request_file and defined $outdir and defined $dbname and defined $req_id;
+
+my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1);
+my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
+my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
+if ($missing_tools) {
+ warn("Can't find required tools.");
+ exit($PS_EXIT_CONFIG_ERROR);
+}
+
+unless (defined $verbose) {
+ $verbose = 0;
+}
+
+my $ipprc = PS::IPP::Config->new();
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+
+if (!$dbserver) {
+ $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+
+$pstamptool .= " -dbname $dbname -dbserver $dbserver";
+
+# Project name is hardcoded to gpc1 for the moment.
+$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver);
+my $imagedb = $project->{dbname};
+if (!$imagedb) {
+ my_die("failed to find imagedb for project: $project", $PS_EXIT_CONFIG_ERROR);
+}
+
+# get the query id and check the extname and version from the header
+my $fields_output;
+{
+ my $command = "echo $request_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+
+ $fields_output = join "", @$stdout_buf;
+}
+my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+
+my_die("$request_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR)
+ if !(defined($extname) and defined($extver) and defined($req_name));
+my_die("$request_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR)
+ if $extname ne "MOPS_DETECTABILITY_QUERY";
+my_die("$request_file is version $extver expecting 1 or 2", $PS_EXIT_PROG_ERROR)
+ if ($extver ne 1) and ($extver ne 2);
+
+if ($req_id) {
+ my $command = "$pstamptool -listreq -name $req_name -not_req_id $req_id";
+ # set verbose to false so that error message about request not found doesn't appear in parse_error.txt
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => 0);
+ if ($success) {
+ # -listreq succeeded duplicate request name
+ print STDERR "QUERY_ID $req_name has already been used\n";
+ $command = "$pstamptool -addjob -job_type none -rownum 0 -state stop -fault $PSTAMP_DUP_REQUEST -req_id $req_id";
+ unless ($no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ if (!$success) {
+ my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+ }
+ } else {
+ print STDERR "skipping $command\n";
+ }
+
+ my $datestr = strftime "%Y%m%d%H%M%S.$req_id", gmtime;
+ $req_name = "ERROR.$datestr";
+ }
+ $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
+ unless ($no_update) {
+ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+ }
+ } else {
+ print STDERR "skipping $command\n";
+ }
+}
+
+
+
+#
+# query is a hash which uses $fpa_id as the key
+# The values are hash references which have the various parameter name as the key
+# The values of the parameter hashes are arrays which contain the values for individual rows
+# in the detectabilty query request
+my %query = ();
+
+{
+ #
+ # Parse input request file using detect_query_read
+ # result is a simple text file
+ #
+ my $dqr_command = "$detect_query_read --dbname $imagedb --input $request_file";
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $dqr_command, verbose => $verbose);
+ unless ($success) {
+ # This is a problem, because I'm not sure how we handle a failure to read something.
+ # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
+ # we can't send that response back.
+ die("Unable to perform $dqr_command error code: $error_code");
+ }
+ my $Nrows = 0;
+ {
+ my @column_names = ();
+ # split output into lines skip until the line which lists the column names is found
+ # Parse subsequent lines
+ foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
+ if ($entry =~ /^#/) {
+ @column_names = split /\s+/, $entry;
+ shift(@column_names); # Dump the hash sign.
+ }
+ elsif (scalar @column_names) {
+ # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE
+ my %row_data;
+ @row_data{@column_names} = (split /\s+/, $entry);
+ for (my $i = 0; $i <= $#column_names; $i++) {
+ push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
+ $Nrows = scalar(keys(%query));
+ # print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
+ }
+ }
+ }
+ }
+ #
+ # Identify target components. This should properly collate targets on a single imfile.
+ #
+ my $query_id;
+ foreach my $fpa_id (keys %query) {
+ my %temp_hash;
+ my $query_style = 'byexp';
+ my $stage;
+ my $filter;
+ my $mjd;
+ # Confirm that we only have one stage/filter/mjd
+ if ($fpa_id ne 'Not_Set') { # We only need to check things that aren't the known odd case.
+ for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+ $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
+ $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
+ $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
+ }
+ if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
+ $stage = (keys(%{ $temp_hash{STAGE} }))[0];
+ }
+ else {
+ exit_with_failure(21,"Too many STAGEs specified");
+ }
+ if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
+ $filter = (keys(%{ $temp_hash{FILTER} }))[0];
+ }
+ else {
+ exit_with_failure(21,"Too many FILTERs specified");
+ }
+ if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
+ $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
+ }
+ else {
+ exit_with_failure(21,"Too many MJD-OBS specified");
+ }
+ }
+
+ ## 20190317 MEH hack to add to row
+ # Set common request components
+ my $option_mask |= 1;
+ $option_mask |= $PSTAMP_SELECT_IMAGE;
+ $option_mask |= $PSTAMP_SELECT_MASK;
+ $option_mask |= $PSTAMP_SELECT_VARIANCE;
+ $option_mask |= $PSTAMP_SELECT_PSF;
+
+ # Set up a rowList with default values
+ my @rowList;
+ for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+ $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i];
+ $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i];
+ $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i];
+ $rowList[$i]->{COORD_MASK} = 0;
+ # 20190317 MEH hack add to row
+ $rowList[$i]->{OPTION_MASK} = $option_mask;
+ # Set default values
+ $query{$fpa_id}{BAD_COMPONENT}[$i] = 1;
+ $query{$fpa_id}{IMAGE}[$i] = 'no_image';
+ $query{$fpa_id}{MASK}[$i] = 'no_mask';
+ $query{$fpa_id}{WEIGHT}[$i] = 'no_weight';
+ $query{$fpa_id}{PSF}[$i] = 'no_psf';
+
+ $query{$fpa_id}{STAGE_ID}[$i] = 'no_id';
+ $query{$fpa_id}{IMAGE_DB}[$i] = 'no_imdb';
+ $query{$fpa_id}{NEED_MAGIC}[$i] = 'no_magic';
+ $query{$fpa_id}{MAGICKED}[$i] = 'no_magic';
+ $query{$fpa_id}{CATALOG}[$i] = 'no_catalog';
+ $query{$fpa_id}{COMPONENT_ID}[$i] = 'no_component';
+ $query{$fpa_id}{CLASS_ID}[$i] = 'no_class';
+
+ $query{$fpa_id}{STATE}[$i] = 'no_state';
+ $query{$fpa_id}{DATA_STATE}[$i] = 'no_dstate';
+ $query{$fpa_id}{FAULT}[$i] = 'no_fault';
+ $query{$fpa_id}{BURNTOOL_STATE}[$i] = 'no_btstate';
+ }
+
+ # Determine the query style for this fpa_id
+ if ($fpa_id =~ /o.*g.*o/) {
+ $query_style = 'byexp';
+ }
+ elsif ($fpa_id =~ /\d+/) {
+ $query_style = 'byid';
+ }
+ elsif ($fpa_id eq 'Not_Set') {
+ # no matching file was found skip
+ next;
+ }
+ else {
+ exit_with_failure(21,"Parse error in request file");
+ }
+ # Set common request components
+# my $option_mask |= 1;
+# $option_mask |= $PSTAMP_SELECT_IMAGE;
+# $option_mask |= $PSTAMP_SELECT_MASK;
+# $option_mask |= $PSTAMP_SELECT_VARIANCE;
+# $option_mask |= $PSTAMP_SELECT_PSF;
+
+ # magic is dead
+ my $need_magic = 0;
+ if ($stage eq 'stack') {
+ $need_magic = 0;
+ }
+ my $mjd_min = int $mjd;
+ my $mjd_max = $mjd + 1;
+
+ my $req_filter;
+ if ($filter ne 'Not_Set') {
+ $req_filter = $filter . '%';
+ }
+
+ # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
+ my $pstamp_images_ref = locate_images($ipprc,$imagedb,
+ \@rowList,
+ $query_style,$stage,
+ $fpa_id,undef,undef,
+ $option_mask,$need_magic,
+ $mjd_min,$mjd_max,$req_filter,undef,$verbose);
+
+ foreach my $this_image_ref (@{ $pstamp_images_ref }) {
+
+ # loop over the rows that this component matched
+ foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
+ $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
+ $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
+ $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
+ $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
+ $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
+ $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
+ $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
+ $query{$fpa_id}{BAD_COMPONENT}[$valid_index] = 0;
+
+ if (exists($this_image_ref->{astrom})) {
+ $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
+ }
+ else {
+ $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
+ }
+ if (exists($this_image_ref->{class_id})) {
+ $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
+ $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
+
+ }
+ else {
+ $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
+ $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
+ }
+
+ $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
+ if (exists($this_image_ref->{data_state})) {
+ $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
+ }
+ else {
+ $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
+ }
+ $query{$fpa_id}{FAULT}[$valid_index] = 0;
+ $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked};
+ if ($stage eq 'chip') {
+ $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
+ }
+
+ # Determine if the data exists.
+ if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or
+ ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
+ ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or
+ ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or
+ ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or
+ ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
+
+ # image is gone and it's not coming back
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
+ }
+ elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
+ }
+ elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
+
+ # updating stacks isn't implemented
+ if (($stage eq 'stack')) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+ }
+ # updating old burntool data isn't implemented
+ elsif ($stage eq 'chip') {
+ if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and
+ (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
+ $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+ }
+ }
+ } # End determining error faults.
+ }
+ }
+ }
+}
+
+my %update_request;
+my %rows_for_component;
+my %faulted_rows;
+
+# now build a hash using image name as key which contains the list of rows for that image
+foreach my $fpa_id (keys %query) {
+ for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
+ if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
+ push @{ $rows_for_component{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
+ } else {
+ $faulted_rows{$query{$fpa_id}{ROWNUM}[$i]} = $query{$fpa_id}{FAULT}[$i];
+ }
+ }
+}
+
+#
+# Finally build the parameter lists and queue jobs for each component
+#
+
+my $job_num = 1;
+foreach my $fpa_id (keys %rows_for_component) {
+ foreach my $component (keys % {$rows_for_component{$fpa_id} } ) {
+ my $i = $rows_for_component{$fpa_id}{$component}[0];
+ my $dep_id = 0;
+ my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
+ if ($data_state ne 'full') {
+ # Queue this data to be updated
+ my $stage = $query{$fpa_id}{STAGE}[$i];
+ my $stage_id = $query{$fpa_id}{STAGE_ID}[$i];
+ my $component = $query{$fpa_id}{COMPONENT_ID}[$i];
+ print "Need to UPDATE $stage $stage_id $component from $data_state\n";
+ $dep_id = queue_update_run($req_id, $outdir, $label, $data_state, $stage, $stage_id, $component, 0, $imagedb);
+ }
+
+ my $outputBase = "$outdir/$job_num" . "_";
+
+ # params_file is an mdc description containing the parameters for the job
+ my $params_file = $outputBase . "params.mdc";
+
+ open PARAMS, ">$params_file" or my_die("failed to create $params_file", $PS_EXIT_UNKNOWN_ERROR);
+
+ # first the job params
+ print PARAMS "dqueryJobParams METADATA\n";
+ foreach my $key (keys %{ $query{$fpa_id} } ) {
+ my $value = $query{$fpa_id}{$key}[0];
+ print PARAMS "\t$key\t\tSTR\t$value\n";
+ }
+ print PARAMS "END\n\n";
+
+ # then one structure with the coordinates for each row
+ print PARAMS "dqueryCoord MULTI\n";
+
+ my @coordkeys = qw(ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG);
+ foreach my $row ( @{ $rows_for_component{$fpa_id}{$component} }) {
+ print PARAMS "\ndqueryCoord METADATA\n";
+ foreach my $key (@coordkeys) {
+ my $value = $query{$fpa_id}{$key}[$row];
+ print PARAMS "\t$key\t\tSTR\t$value\n";
+ }
+ print PARAMS "END\n";
+ }
+ close(PARAMS);
+
+ # use first rownum for this component as the rownum for the job
+ my $rownum = $query{$fpa_id}{ROWNUM}[$rows_for_component{$fpa_id}{$component}[0]];
+
+ my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outputBase -job_type detect_query -state run -rownum $rownum";
+ $command .= " -dep_id $dep_id" if $dep_id;
+ unless ($no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ if ($success) {
+ my $job_id = join "", @$stdout_buf;
+ print " Queued job: $job_id\n";
+ } else {
+ my $exit_status = $error_code >> 8;
+ $exit_status = $PS_EXIT_UNKNOWN_ERROR unless $exit_status;
+ my_die("Unable to perform $command error code: $error_code", $exit_status);
+ }
+ } else {
+ print STDERR "skipping $command\n";
+ }
+ $job_num++;
+ }
+}
+
+foreach my $rownum (keys %faulted_rows ) {
+ my $outputBase = "$outdir/$job_num" . "_";
+ my $fault = $faulted_rows{$rownum};
+ $fault = $PSTAMP_NO_IMAGE_MATCH if ($fault eq 'no_fault');
+ print STDERR "insert faulted job for $rownum: $fault\n";
+ my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outputBase -job_type detect_query -state stop -rownum $rownum -fault $fault";
+ unless ($no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ if ($success) {
+ my $job_id = join "", @$stdout_buf;
+ print " Queued job: $job_id\n";
+ } else {
+ my $exit_status = $error_code >> 8;
+ $exit_status = $PS_EXIT_UNKNOWN_ERROR unless $exit_status;
+ my_die("Unable to perform $command error code: $error_code", $exit_status);
+ }
+ } else {
+ print STDERR "skipping $command\n";
+ }
+ $job_num++;
+}
+
+{
+ my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
+ # $command .= " -set_fault $result" if $result;
+
+ unless ($no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
+ }
+ } else {
+ print STDERR "Skipping $command\n";
+ }
+}
+
+exit 0;
+
+sub queue_update_run {
+ my ($req_id, $outdir, $label, $state, $stage, $stage_id, $component, $need_magic, $imagedb) = @_;
+
+# my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
+
+ if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
+ # We should have received one of these states, so if not, signal that we have a problem.
+ my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
+ }
+ my $dep_id;
+ my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component ";
+ $command .= " -outdir $outdir";
+ $command .= " -need_magic" if $need_magic;
+
+ my $rlabel = "ps_ud_" . $label if $label;
+ $command .= " -rlabel $rlabel" if $rlabel;
+
+ if (!$no_update) {
+ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+ run(command => $command, verbose => $verbose);
+ unless ($success) {
+ my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+ }
+ my $output = join "", @$stdout_buf;
+ chomp $output;
+ $dep_id = $output;
+
+ my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
+ }
+ else {
+ print STDERR "skipping $command\n";
+ $dep_id = 42;
+ }
+
+ return($dep_id);
+}
+
+
+sub my_die {
+ my $message = shift;
+ my $exit_code = shift;
+
+ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+ carp("$message");
+ exit($exit_code);
+}
+
+sub exit_with_failure {
+ my $status = shift;
+ my $message = shift;
+ my_die($message, $status);
+}
+
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/fakedresponse.pl
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/fakedresponse.pl (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/fakedresponse.pl (revision 41103)
@@ -0,0 +1,103 @@
+#!/usr/bin/env perl
+#
+# parse a MOPS_DETCTABILITY_QUERY table and create a results file
+#
+# Note: this file is currently only a placeholder which creates a fake response file
+#
+
+use strict;
+use warnings;
+
+use Getopt::Long qw( GetOptions );
+use Pod::Usage qw( pod2usage );
+use DBI;
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use File::Copy;
+use File::Basename;
+
+use PS::IPP::Config qw($PS_EXIT_SUCCESS
+ $PS_EXIT_UNKNOWN_ERROR
+ $PS_EXIT_SYS_ERROR
+ $PS_EXIT_CONFIG_ERROR
+ $PS_EXIT_PROG_ERROR
+ $PS_EXIT_DATA_ERROR
+ $PS_EXIT_TIMEOUT_ERROR
+ metadataLookupStr
+ metadataLookupBool
+ caturi
+ );
+my ($input, $output, $verbose, $save_temps, $workdir);
+#
+# parse args
+#
+
+
+GetOptions(
+ 'input=s' => \$input,
+ 'output=s' => \$output,
+ 'workdir=s' => \$workdir,
+ 'verbose' => \$verbose,
+ 'save-temps' => \$save_temps,
+) or pod2usage(2);
+
+my $err = "";
+
+if (!$input) {
+ $err .= "--input is required\n";
+}
+if (!$output) {
+ $err .="--output is required\n";
+}
+
+die $err if ($err);
+
+# spit the contents of the file in text format
+# XXX Do we lose precision on the floats when we do this?
+
+my $query_text = `detect_query_read --nolabel --input $input`;
+
+my @lines = split "^", $query_text;
+
+# first line is the header keywords
+my $header = shift @lines;
+chomp $header;
+die "failed to parse $input" unless $header;
+
+my ($query_id, $fpa_id, $dateobs, $filter, $obscode) = split " ", $header;
+die "failed to parse $input" unless $query_id && $fpa_id && $dateobs && $filter && $obscode;
+
+if (!$workdir) {
+ $workdir="/tmp";
+}
+my $txt_file = "$workdir/response$$.txt";
+open OUT, ">$txt_file";
+
+print OUT "$query_id $fpa_id $dateobs $filter $obscode\n";
+
+while (my $line = shift @lines) {
+ chomp $line;
+ my ($rownum, $detect_n, $detect_f) = fake_dquery_response($line);
+
+ printf OUT "%-12s %-8d %-8.4f\n", $rownum, $detect_n, $detect_f;
+}
+close OUT;
+
+my $result = system "detect_response_create --input $txt_file --output $output";
+
+unlink $txt_file unless $save_temps;
+
+exit $result;
+
+
+sub fake_dquery_response {
+ my $line = shift;
+ my ($rownum, $ra1, $dec1, $ra2, $dec2, $mag) = split " ", $line;
+
+ # todo perhaps think more about these values
+ my $n = rand(16);
+ my $f = rand(1);
+
+ return ($rownum, $n, $f);
+}
Index: /branches/ccl_branches/ipponly-20191108/pstamp/scripts/ftpsrequest
===================================================================
--- /branches/ccl_branches/ipponly-20191108/pstamp/scripts/ftpsrequest (revision 41103)
+++ /branches/ccl_branches/ipponly-20191108/pstamp/scripts/ftpsrequest (revision 41103)
@@ -0,0 +1,203 @@
+#!/usr/bin/perl
+
+# ftpsrequest
+
+# Creates a PS1 postage stamp request table using the FTOOLS program ftcreate.
+# The input file format is one line per row with columns described in the array columnsAndTypes given
+# below.
+#
+# Note: The input format is the same as that used for the rows by the IPP program pstamp_request_file with
+# the exception that the comment is contained in the last column as a quoted string rather than by the |
+# character.
+
+# the option --print-sample may be used to print out an example of a line in an input file to stdout.
+#
+
+# The program ftcreate is part of the FTOOLS package which is described at http://heasarc.gsfc.nasa.gov/ftools
+# and in
+# Blackburn, J. K. 1995, in ASP Conf. Ser., Vol. 77, Astronomical Data Analysis Software and Systems IV,
+# ed. R. A. Shaw, H. E. Payne, and J. J. E. Hayes (San Francisco: ASP), 367.
+
+use strict;
+use warnings;
+
+# using a minimal number of perl modules for ease of portablity
+
+use File::Temp qw(tempfile);
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Array describing the columns in the request table and the order of the columns in the input file
+# third column is value for sample request. A stamp for an i band stack of the tadpole galaxy.
+my @columnsAndTypes = qw(
+ ROWNUM J 1
+ CENTER_X D 241.516417
+ CENTER_Y D 55.425369
+ WIDTH D 1000
+ HEIGHT D 1000
+ COORD_MASK J 2
+ JOB_TYPE 16A stamp
+ OPTION_MASK J 65
+ PROJECT 16A gpc1
+ SURVEY_NAME 16A 3PI
+ IPP_RELEASE 64A null
+ REQ_TYPE 16A bycoord
+ IMG_TYPE 16A stack
+ ID 16A null
+ TESS_ID 64A null
+ COMPONENT 64A null
+ DATA_GROUP 64A null
+ REQFILT 16A i
+ MJD_MIN D 0
+ MJD_MAX D 0
+ RUN_TYPE 16A null
+ FWHM_MIN D 0
+ FWHM_MAX D 0
+ COMMENT 64A tadpole_galaxy
+);
+
+
+my ( $input, # Name of input text file
+ $output, # Name of output table
+ $req_name,
+ $print_sample,
+ $email,
+ $clobber,
+ $help,
+ $verbose,
+ $save_temps,
+ );
+
+
+GetOptions(
+ 'input|i=s' => \$input,
+ 'output|o=s' => \$output,
+ 'req_name|r=s' => \$req_name,
+ 'email=s' => \$email,
+ 'help' => \$help,
+ 'print-sample' => \$print_sample,
+ 'clobber' => \$clobber,
+ 'save-temps' => \$save_temps,
+ 'verbose' => \$verbose,
+) or pod2usage( 2 );
+
+my $usageMessage = "$0: --input --output