Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/Makefile.am =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/Makefile.am (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/Makefile.am (revision 41112) @@ -0,0 +1,5 @@ +SUBDIRS = src + +CLEANFILES = *.pyc *~ core core.* + +ACLOCAL_AMFLAGS = -I m4 Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/autogen.sh =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/autogen.sh (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/autogen.sh (revision 41112) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppTranslate +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal -I `pwd`/m4 $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOLIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/configure.ac =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/configure.ac (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/configure.ac (revision 41112) @@ -0,0 +1,44 @@ +dnl Process this file with autoconf to produce a configure script. +AC_PREREQ(2.61) + +AC_INIT([ppMops], [0.1.1], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) +AC_CONFIG_MACRO_DIR([m4]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDLDFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.0.0]) +CHECK_ZLIB + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +dnl Set CFLAGS for build +IPP_STDOPTS +IPP_STDCFLAGS + +IPP_VERSION + +AC_SUBST([PPMOPS_CFLAGS]) +AC_SUBST([PPMOPS_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile +]) + +AC_OUTPUT Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/doc/MOPS_ICD_lite.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/doc/MOPS_ICD_lite.txt (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/doc/MOPS_ICD_lite.txt (revision 41112) @@ -0,0 +1,192 @@ +== General notes == + +Though efforts have been made to avoid this from occurring, where +header keyword names are longer than the usual 8 character maximum, +the HIERARCH convention will be used. + +Celestial coordinates are in ICRS. + +Magnitudes are AB magnitudes (flux zero point is 3631 Jy). + +Unless otherwise noted, error values are statistical only, and include +no contribution by systematic errors. + +== Primary header == + +The primary header will not contain any information beyond that +written automatically by cfitsio, and is present only for compliance +with the FITS standard. + +== FITS table == + +The FITS table will include all detections within an exposure. + +Duplicates from overlapping skycells will have been filtered out by +IPP, with the measurements from the source closest to the centre of a +skycell included (and all others discarded). + +=== Header === + +The header will contain information relevant to the exposure as a whole. + + * Version information + * SWSOURCE (string): source of software (e.g., "60eb6cdc-a59c-4636-a4e0-dba66a9721fd") + * SWVERSN (string): version of software (e.g., "trunk/ppMops@24658") + + * Provenance information + * EXP_NAME (string): Exposure name (e.g., "o1234g5678") + * EXP_ID (S64): Exposure identifier + * CHIP_ID (S64): Chip stage identifier + * CAM_ID (S64): Camera stage identifier + * FAKE_ID (S64): Fake stage identifier + * WARP_ID (S64): Warp stage identifier + * DIFF_ID (S64): Diff stage identifier + * DIFF_POS (boolean): Sense of subtraction; T for forward, F for backward + + * Exposure details + * MJD-OBS (F64): TAI MJD of exposure mid-point + * RA (string): Reported Right Ascension of telescope boresight, sexagesimal hours (e.g., "12:34:56.789") + * DEC (string): Reported Declination of telescope boresight, sexagesimal degrees (e.g., "-12:34:56.78") + * TEL_ALT (F64): Reported telescope altitude, degrees + * TEL_AZ (F64): Reported telescope azimuth, degrees + * EXPTIME (F32): Exposure time, seconds + * ROTANGLE (F64): Rotator angle, degrees + * FILTER (string): Filter name (e.g., "r.00000") + * AIRMASS (F32): Airmass for exposure + * OBSCODE (string): IAU observatory code (i.e., "F51" for PS1) + * SEEING (F32): Measured seeing at diff stage, arcsec + * MAGZP (F32): Magnitude zero point + * MAGZPERR (F32): Error in magnitude zero point + * ASTRORMS (F32): RMS of astrometric fit, arcsec + + * Detection efficiency + * DE_MAGnn (F32): Magnitude (calibrated) for detection efficiency + * DE_EFFnn (F32): Detection efficiency (0..1) + +==== Current limitations ==== + +The IPP is not yet calculating detection efficiencies (it is still +being developed). Further, it is not yet clear how to merge the +detection efficiency measurements for different skycells. Until these +issues are resolved, the detection efficiency values will be fake. + +==== Differences from original ICD ==== + + * SWSOURCE, SWVERSN has replaced TABLEVER which was never defined + * All of the "Provenance information" has replaced FPA_ID, for better tracking of the source of each detection + * SEEING and MAGZP added as indicators of the quality of the exposure + * MAGZPERR added to make absolute (i.e., across exposures) magnitude errors more accurate + * ASTRORMS added to make absolute (i.e., across exposures) astrometry errors more accurate + * DE_MAGnn and DE_EFFnn replace DE1 through DE10, which were never well defined + * Removed LIMITMAG, which was never well defined, and is unnecessary given the DE_MAGnn and DE_EFFnn + +=== Table === + +The table will contain information relevant to the individual +detections within the exposure. + + * RA (F64): Right Ascension of detection centre, degrees + * RA_ERR (F64): Error in RA, degrees + * DEC (F64): Declination of detection centre, degrees + * DEC_ERR (F64): Error in DEC, degrees + * MAG (F32): Calibrated magnitude of detection + * MAG_ERR (F32): Error in MAG + * STARPSF (F32): A PSF/extended source separator + * ANGLE (F64): Angle of trail fit to source, degrees E of N + * ANGLE_ERR (F64): Error in ANGLE, degrees + * LENGTH (F32): Length of trail fit to source, degrees + * LENGTH_ERR (F32): Error in LENGTH, degrees + * FLAGS (S32): IPP detection flags, bit mask + * DIFF_SKYFILE_ID (S64): IPP diff_skyfile_id for source + +==== Current limitations ==== + +The IPP is not yet fitting trails to sources in the difference images. +Until this is being done, the ANGLE, ANGLE_ERR, LENGTH and LENGTH_ERR +values will be zero. + +The value being written as STARPSF is EXT_NSIGMA from the IPP CMF +files; it is not clear that this is what MOPS wants, but this is +probably irrelevant until the trail fitting has been implemented. + +==== Differences from original ICD ==== + + * RA_DEG, DEC_DEG renamed RA, DEC to match apparent naming policy + * *_SIG renamed *_ERR to be more clear + * ANG, ANG_SIG, LEN, LEN_SIG spelled out as ANGLE, ANGLE_ERR, LENGTH, LENGTH_ERR + * FLUX, FLUX_SIG renamed MAG, MAG_ERR since IPP writes magnitudes + * FLAGS added to allow additional weeding out of bad detections + * DIFF_SKYFILE_ID added to allow trace back to IPP diff skyfile, for postage stamps + + +=== Example === + +{{{ +XTENSION= 'BINTABLE' / binary table extension +BITPIX = 8 / 8-bit bytes +NAXIS = 2 / 2-dimensional binary table +NAXIS1 = 72 / width of table in bytes +NAXIS2 = 42032 / number of rows in table +PCOUNT = 0 / size of special data area +GCOUNT = 1 / one data group (required keyword) +TFIELDS = 13 / number of fields in each row +TTYPE1 = 'RA ' / label for field 1 +TFORM1 = '1D ' / data format of field: 8-byte DOUBLE +TTYPE2 = 'RA_ERR ' / label for field 2 +TFORM2 = '1D ' / data format of field: 8-byte DOUBLE +TTYPE3 = 'DEC ' / label for field 3 +TFORM3 = '1D ' / data format of field: 8-byte DOUBLE +TTYPE4 = 'DEC_ERR ' / label for field 4 +TFORM4 = '1D ' / data format of field: 8-byte DOUBLE +TTYPE5 = 'MAG ' / label for field 5 +TFORM5 = '1E ' / data format of field: 4-byte REAL +TTYPE6 = 'MAG_ERR ' / label for field 6 +TFORM6 = '1E ' / data format of field: 4-byte REAL +TTYPE7 = 'STARPSF ' / label for field 7 +TFORM7 = '1E ' / data format of field: 4-byte REAL +TTYPE8 = 'ANGLE ' / label for field 8 +TFORM8 = '1E ' / data format of field: 4-byte REAL +TTYPE9 = 'ANGLE_ERR' / label for field 9 +TFORM9 = '1E ' / data format of field: 4-byte REAL +TTYPE10 = 'LENGTH ' / label for field 10 +TFORM10 = '1E ' / data format of field: 4-byte REAL +TTYPE11 = 'LENGTH_ERR' / label for field 11 +TFORM11 = '1E ' / data format of field: 4-byte REAL +TTYPE12 = 'FLAGS ' / label for field 12 +TFORM12 = '1J ' / data format of field: 4-byte INTEGER +TZERO12 = 2147483648 / offset for unsigned integers +TSCAL12 = 1 / data are not scaled +TTYPE13 = 'DIFF_SKYFILE_ID' / label for field 13 +TFORM13 = '1K ' / data format of field: 8-byte INTEGER +SWSOURCE= '60eb6cdc-a59c-4636-a4e0-dba66a9721fd' / Software source +SWVERSN = 'branches/pap_mops/ppMops@25227' / Software version +HISTORY ppMops at 2009-09-02T03:48:46.695783 +HISTORY psLib version: branches/pap_mops/psLib@25227 +HISTORY psLib source: 60eb6cdc-a59c-4636-a4e0-dba66a9721fd +HISTORY ppMops version: branches/pap_mops/ppMops@25227 +HISTORY ppMops source: 60eb6cdc-a59c-4636-a4e0-dba66a9721fd +EXP_NAME= 'o4995g0129o' / Exposure name +EXP_ID = 77164 / Exposure identifier +CHIP_ID = 24019 / Chip stage identifier +CAM_ID = 17726 / Cam stage identifier +FAKE_ID = 10227 / Fake stage identifier +WARP_ID = 8842 / Warp stage identifier +DIFF_ID = 0 / Diff stage identifier +DIFF_POS= F / Positive subtraction? +MJD-OBS = 54995.4740598313 / MJD of exposure midpoint +RA = '18:25:01.988' / Right Ascension of boresight +DEC = '-17:20:40.069' / Declination of boresight +TEL_ALT = 51.951873 / Telescope altitude +TEL_AZ = 179.483883 / Telescope azimuth +EXPTIME = 38. / Exposure time (sec) +ROTANGLE= 333.1039 / Rotator position angle +FILTER = 'r.00000 ' / Filter name +AIRMASS = 1.269 / Airmass of exposure +OBSCODE = 'F51 ' / IAU Observatory code +SEEING = 1.678401 / Mean seeing +MAGZP = 28.65226 / Magnitude zero point +MAGZPERR= 0.353063 / Error in magnitude zero point +ASTRORMS= 0.3111496 / RMS of astrometric fit +EXTNAME = 'MOPS_TRANSIENT_DETECTIONS' +END +}}} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/Makefile =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/Makefile (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/Makefile (revision 41112) @@ -0,0 +1,27 @@ +SOURCE = PSDC-940-005 + +FIGURES = figures/exposure_ota.eps \ + figures/ext_theta.eps \ + figures/ipp_workflow.eps + +RM=/bin/rm -f + +figures/%.eps: figures/%.fig + fig2dev -L eps $< $@ + +all: figures + latex $(SOURCE).tex + bibtex $(SOURCE) + latex $(SOURCE).tex + latex $(SOURCE).tex + dvipdf PSDC-940-005.dvi + @echo + @echo + @echo "IPP-MOPS documentation in pdf format has been succesfully built" + @echo + +figures: $(FIGURES) + +clean: + $(RM) *.aux *.dvi *.out *.pdf *.toc *.log *~ *.blg *.bbl + $(RM) $(FIGURES) Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/PSDC-940-005.tex =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/PSDC-940-005.tex (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/PSDC-940-005.tex (revision 41112) @@ -0,0 +1,1607 @@ +%%% $Id: PSDC-940-005.tex,v 1.9 2011/04/18 23:17:36 schastel Exp $ +\documentclass[panstarrs,spec]{panstarrs} +\usepackage{nonfloat} +\usepackage{caption} + +\usepackage{etoolbox} +\patchcmd{\thebibliography}{\chapter*}{\subsection*}{}{} + +\newcommand{\outformatversion}{% + \texttt{PS1\_DV3}} + +\newcommand{\commentFrom}[4]{% + \mbox{ }\\\null\hfill\begin{tabular}{|p{4in}} + \hline + \emph{Comment from {#1} (#2)}: {#3}\\ + \hline + Answer: {#4}\\ + \hline + \end{tabular}} + +\newcommand{\commentFromSC}[3]{% + \commentFrom{SC}{#1}{#2}{#3}} + +% basic document variables +\title{IPP-MOPS ICD} +\subtitle{Pan-STARRS IPP-MOPS ICD for PS-1} +\shorttitle{IPP-MOPS ICD} +\author{\parbox{3in}{Serge Chastel; Larry Denneau, Jr.; Denver Green; Robert Jedicke; Eugene Magnier}} +\audience{Pan-STARRS PMO} +\group{Pan-STARRS IPP \& MOPS} +\project{Pan-STARRS PS-1} +\organization{Institute for Astronomy} +\version{03} +\docnumber{PSDC-940-005} + +\begin{document} +\maketitle +\sloppy + +\RevisionsStart +% version Date Description +04 & 2012 Nov 20 & Modifications integrating DG comments/questions \\ +\hline +03 & 2012 Oct 04 & \texttt{PS1\_DV3}: Fitted trails, new parameters\\ +\hline +02 & 2011 Apr 19 & Synchronization with "ICD Lite" wiki page \\ +\hline +01 & 2010 Oct 27 & New CMF Parameters \\ +\hline +00 & 2006 Mar 01 & Initial version \\ +\hline +\RevisionsEnd + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\bibliographystyle{amsplain} +\bibliography{icd} + +\DocumentsInternal +PSDC-230-001 & PS-1 Design Reference Mission \\ \hline +PSDC-430-011 & Pan-STARRS PS-1 IPP System/Subsystem Design Description \\ \hline +PSDC-510-001 & Pan-STARRS PS-1 MOPS System Concept Definition \\ \hline +PSDC-530-003 & Pan-STARRS PS-1 MOPS Software Design Description \\ \hline +\DocumentsExternal +\DocumentsEnd + +\tableofcontents +\pagebreak +\pagenumbering{arabic} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\section{Scope} + +\subsection{Identification} + +This document defines the interface between the Image Processing +Pipeline (IPP) and Moving Object Processing System (MOPS) for the +Panoramic Survey Telescope and Rapid Response System (Pan-STARRS) +for the prototype telescope PS-1, and is a System-level controlled +specification/design description document in the official Pan-STARRS +engineering specification tree. + +\subsection{System Overview} + +The Institute for Astronomy at the University of Hawai`i is developing +a large optical synoptic survey telescope system, the Panoramic Survey +Telescope and Rapid Response System (Pan-STARRS). The science goals, +priorities, top-level concept of operations with associated +operational requirements, and system performance drivers with +associated system performance requirements are described in the +Pan-STARRS Science Goals Statement (SGS). As described in this +document, The system conceptual design for Pan-STARRS utilizes an +array of four 1.8m telescopes each with a 7 degree$^2$ field of view, +giving the system an \'etendue larger than all existing survey +instruments combined (defined as the product of the collecting area +$A$ multiplied by the field-of-view solid angle $\Omega$). Each +telescope will be equipped with a 1 billion pixel CCD camera with low +noise and rapid read-out, and the data will be reduced in near real +time to produce both cumulative static sky and difference images from +which transient, moving, and variable objects can be +detected. Pan-STARRS will be able to survey up to $\approx 6,000$ +degree$^{2}$ per night to a detection limit of approximately 24$^{th}$ +magnitude. This unique combination of sensitivity and sky coverage +will open up many new possibilities in time domain astronomy including +a major goal of surveying the Potentially Hazardous Object (PHO) +population down to a diameter of $\approx 300$ meters. In addition, +the Pan-STARRS data will be used to investigate a broad range of +astronomical problems of extreme current interest concerning the Solar +System, the Galaxy, and the Cosmos at large. A prototype single +telescope system, PS-1, is being developed as a preliminary step +before construction of the complete four telescope system. + +\begin{tabular}{ll} + Project sponsor:& AFRL, United States Air Force \\ + Acquirer: & University of Hawaii Institute for Astronomy \\ + User: & Astronomical community \\ + Developer: & University of Hawaii Institute for Astronomy, participating \\ + & institutions, and associated subcontractors +\end{tabular} + +\subsection{Document Overview} + +This document identifies the interface between the IPP and MOPS +subsystems of the Pan-STARRS PS-1 Telescope. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\DocumentsInternalSection +PSDC-230-001 & PS-1 Design Reference Mission \\ \hline +PSDC-230-002 & PS-1 System Concept Definition \\ \hline +\DocumentsExternalSection +POSIX Standard & Open Group Based Specifications Issue 6, IEEE Std 1003.1, 2003 \\ +FITS Standard & Definition of the Flexible Image Transport System (FITS), +March 29, 1999, NOST 100-2.0 \\ +\DocumentsEnd + +\subsection{Acronyms} + +\begin{table} + \small + \caption{Acronyms used in this document\label{table-acronyms}} + \begin{tabular}{ll} + \hline + {\bf Acronym} & {\bf Definition} \\ + \hline + ADU & Analog to Digital Units (counts in an image) \\ + ICRS & International Celestial Reference System \\ + IPP & Pan-STARRS Image Processing Pipeline \\ + FPA & Focal Plane Array \\ + MOPS & Pan-STARRS Moving Object Processing System \\ + MPC & Center for Astrophysics Minor Planet Center \\ + TTI & Transient-Time Interval (about 60 minutes) \\ + \hline + \end{tabular} +\end{table} + +\section{Interface Overview} + +\subsection{Pan-STARRS Software Interfaces} + +The Pan-STARRS PS-1 Telescope system has multiple +subsystem-to-subsystem interfaces, illustrated in +Figure~\ref{fig:overview}. The subsystem-to-subsystem interfaces are +each documented in separate documents. The interface numbers refer to +the Pan-STARRS Document Control numbering scheme for the interface +documents. All Interface Control Documents are numbered +PSDC-940-XXX-VV, while all Interface Requirements Specifications are +numbered PSDC-930-XXX-VV, where XXX is the number listed above and VV +is a document version number. Table~\ref{tab:overview} lists the +organizational structure of all Pan-STARRS interfaces, giving the +corresponding interface number listed in Figure~\ref{fig:overview}, +the group of data being transferred (the assembly), the method in +which the data is formatted (packing), and the data transfer method +(protocol). + +\begin{figure} + \begin{center} + \epsfig{file=figures/PSSWIF.eps} + \caption{Overview of Pan-STARRS Software Interfaces.} + \label{fig:overview} + \end{center} +\end{figure} + +This document discusses the software interface between IPP and MOPS. +IPP must send transient detection descriptions to the MOPS; MOPS must +be able to query the IPP regarding whether a particular (RA, Dec) pair +could have been detected in a particular focal plane array (FPA). +This document defines the data to be transferred between IPP and MOPS +and the mechanisms to be used for that interchange. These interfaces +are identified as \#005 in Figure~\ref{fig:overview}. The Interfaces +addressed by this document are marked in the table. + +\begin{table} + \begin{center} + \caption{Pan-STARRS Interface Data Organization} + \label{tab:overview} + \begin{tabular}{|l|l|l|l|} + \hline + {\bf Interface \#} & {\bf Assembly} & {\bf Packaging} & {\bf Protocol} \\ \hline + 001.2 & Telescope Commands & EOST Command Language & Command Socket \\ + 002.2 & Camera Commands & Conductor Command Language & Command Socket \\ + 002.3 & Camera Status & ??? & Shared Name Space \\ + 002.4 & IQ Images & FITS Images & Data Store \\ + 003.2 & GPC Images \& Metadata & FITS Images \& Headers & Data Store \\ + \hline + 004.2 & OTIS Metadata & FITS Tables & Data Store \\ + 004.3 & ISP Images \& Metadata & FITS Images \& Headers & Data Store \\ + 004.4 & Reference Star Catalog & FITS Tables & Data Store \\ + 004.5 & Focal Plane Layout & FITS Tables & Data Store \\ + \hline + 005.2 & Detections & FITS Tables & Data Store \\ + 005.3 & Test Positions & FITS Tables & Data Store \\ + 006.2 & Detections & FITS Tables & Data Store \\ + 006.3 & Metadata & FITS Tables & Data Store \\ + 006.4 & Images & FITS Tables & Data Store \\ + 007.2 & Orbits \& Detections & FITS Tables & Data Store \\ + 008.2 & Detections & FITS Tables & Data Store \\ + 008.3 & Metadata & FITS Tables & Data Store \\ + 008.4 & Images & FITS Tables & Data Store \\ + \hline + \end{tabular} + \end{center} +\end{table} + +Layers of network communications are often classified using the +\htmladdnormallink{OSI Reference + Model}{http://en.wikipedia.org/wiki/OSI_reference_model}. When addressing +network connectivity between subsystem interfaces, this +Interface Control Document is primarily concerned with two highest +layers of the interface communication: the presentation layer, or how +the data is packaged, and the application layer, or the high-level details of the +data contents. The lower layers of the network communication in the OSI model +are addressed briefly below. + +The leased line between the IPP and MOPS is currently expected to have a practical +maximum sustainable throughput of 50Mbits/s, since the systems will be connected +through the University of Hawaii ITS network. Higher and lower bit rates are available, +and the cost diffferences are quite significant. + +\subsubsection{Physical Layer} + +MOPS and IPP will each be connected to local inbound and outbound hardware +interfaces at rates of no less than 1Gbit/sec using fiber or twisted pair +and onboard NICs where possible. + +\subsubsection{Data Link Layer} + +The endpoints of the route between IPP and MOPS will be Gigabit +Ethernet with normal (1500 byte) max. transmission units (MTU). + +\subsubsection{Network Layer} + +Internet Protocol v4 will be used. Data Store's HTTP +server will have an IP address visible (routable) to IPP. + +\subsubsection{Transport Layer} + +TCP will be used on top of IPv4. + +\subsubsection{Session Layer} + +IPP-MOPS communication will take place entirely via the Data Store +mechanism. + +\subsubsection{Presentation Layer} + +All data exchanged between IPP and MOPS will be in the form of FITS +Tables and FITS images. + +The primary header will not contain any information beyond that +written automatically by cfitsio, and is present only for compliance +with the FITS standard. + +Though efforts have been made to avoid this from occurring, where +header keyword names are longer than the usual 8 character maximum, +the HIERARCH convention will be used (see the NASA webpages: +\htmladdnormallink{Registered FITS + Convention}{http://fits.gsfc.nasa.gov/registry/hierarch_keyword.html} +and \htmladdnormallink{4.12.4 HIERARCH Convention for Extended Keyword + Names}{http://heasarc.nasa.gov/docs/software/fitsio/c/f_user/node28.html}). + +\subsection{IPP-MOPS Exchange} + +MOPS requires from the IPP a stream of transient detections and their +FPA metadata obtained from processing of difference images acquired by +the PS-1 camera. + +IPP is additionally required by MOPS to be able to indicate whether a +set of detections (with RA, Dec, flux, length, orientation, etc.) on a +previously acquired FPA was detectable by the PS-1 telescope. This +functionality is provided through the detectability server. + +\subsubsection{Conventions} + +\begin{enumerate} +\item Celestial coordinates are in ICRS; +\item Magnitudes are AB magnitudes (flux zero point is 3631 Jy); +\item Unless otherwise noted, error values are statistical only, and + include no contribution by systematic errors. +\end{enumerate} + +\subsubsection{Transient Detection and Metadata Exchange} + +Transient detections and their metadata are assembled nightly into +FITS files, one file per FPA. IPP will submit outgoing data into the +IPP-MOPS outbound datastore. Metadata for the detections belonging to +a FPA will be described in the file's FITS headers, and detection data +will be listed in a subsequent FITS BINTABLE table in the same file. + +Each column in the FITS table shall correspond to a detection attribute. +The data type of a FITS column shall be chosen to represent the attribute +without loss of information. + +The FITS table will include all detections within an exposure. + +Duplicates from overlapping skycells will have been filtered out by +IPP, with the measurements from the source closest to the centre of a +skycell included (and all others discarded). + +\subsubsection{Detectability} + +MOPS will need to query the IPP periodically to inquire whether a +particular elongated trailed detection specified by a ($RA_0$, +$Dec_0$, $Time_0$, $RA_1$, $Dec_1$, $Time_1$) tuple was detectable by +the PS-1 telescope. This query is important in evaluating +``non-detections,'' or failures of MOPS to locate predicted detections +of a known object, and for accurate simulation of detections of +synthetic objects that are injected into the MOPS pipeline to measure +efficiency. + +MOPS will assemble a list of (RA, Dec, Time) tuples in a single FITS +file and submit them to the MOPS-IPP outgoing datastore. Later MOPS +will poll the IPP datastore for results of this query. If results of +a query do not appear after a pre-configured timeout, an operator is +notified. + +\subsubsection{Table Description Files} + +FITS tables generated by IPP or MOPS shall be described by table +description files. These files shall be controlled by CVS, with the +authoritative versions of these files being the ones contained in +Appendix A of this document. FITS header data units (HDUs) which do +not conform to the corresponding table description files shall be +rejected by the receiver, and the ingest software shall notify an +operator about the error. + +In addition to the required keywords specified in the FITS standard +for a BINTABLE extension, each FITS HDU shall have the following +keywords: + +\begin{itemize} +\item{EXTNAME} - A \emph{string} giving a unique name for the metadata + table. Such a name shall contain only the characters a-z, A-Z, and + ``\_''. +\item{CMFVERSION} - A \emph{string} specifying the version of the + table description file for this table. +\end{itemize} + +The format of the metadata tables is described in the following sections. + +\subsubsection{Fields in Table Description Files} + +The IPP software generating the FITS tables may change. Parameters, +i.e. columns in FITS tables, may also be added requiring changes both +in ICD and in software. To keep track of such changes, a Version will +be assigned to each keyword in the tables descriptions. + +To improve the readability, keywords appearing only in the +\outformatversion{} software version are shown. For previous releases +of this documentation and learn about the former output format, +checkout \htmladdnormallink{the older SVN revision + 34506}{https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/ppTranslate/documentation/ICD}. + +The following conventions for the Version numbering is used: $X+$ +(where $X$ is the output format version in \texttt{PS1\_DV$X$}) means +that the keyword appeared in software version X until the current +version. + +\subsubsection{ICD/Software Versions} + +\begin{tabular}{|c|c|c|c|p{3in}|} + \hline + SW Version & ICD Version & Output Version & Software Tag & Comment \\ + \hline + 0 & 00 & 0 & n/a & Preliminary ICD. No sw tag \\ + 1 & 01 & \texttt{PS1\_DV1} & 20100823 & First software release \\ + 2 & 01 & \texttt{PS1\_DV2} & 20101029 & "V2" software version \\ + 2 & 02 & \texttt{PS1\_DV2} & 20110406 & Deletion of "ICD lite" wiki page (Documentation update only)\\ + 4 & 03 & \texttt{PS1\_DV3} & \emph{No tag yet} & Extended parameters (fitted trails), warp-stack\\ + \hline +\end{tabular} + +\subsubsection{Transfer via Data Store} + +The FITS files containing metadata shall be tranferred using the protocol +described in [Data Store doc]. There will be one instance of a datastore +within IPP that will be used to submit transient detections and results +of detectability queries to MOPS, and +another instance residing within MOPS to submit detectability requests to the IPP. + +During a datastore transaction, the ``sender'' places a file into the datastore with a unique +transfer ID. The ``receiver'' must regularly poll the datastore to +detect new FITS files. The receiver then downloads each file from a URL +provided by the Data Store interface. The receiver is responsible for +managing which files it has already downloaded. Files remain in +the datastore for a preset period of time and then are deleted without +warning. If a file transfer fails, the receiver may retry the transfer, +If a retry fails, the failure must be resolved by operator intervention. + +The data transfers will have the following parameters: + +\begin{table}[htbp] + \begin{center} + \caption{Data transfer parameters} + \begin{tabular}{|l|l|l|} + \hline + \hline + {\bf Parameter} & {\bf IPP to MOPS} & {\bf MOPS to IPP} \\ + \hline + Dump interval & Per FPA & Per FPA \\ + Max polling Interval & 60s & 60s \\ + File Lifetime & 30 days & 10 days \\ + Datastore location & IPP & MOPS \\ \hline + \end{tabular} + \end{center} +\end{table} + +\subsection{Transient Detection List} + +IPP will submit all transient detections per FPA and their associated +metadata in a single FITS file. The metadata will be placed in FITS +header data unit (HDU) key-value pairs, and the detection attributes +will be placed in the associated FITS table following the header. All +time-dependent table values shall be provided at the midpoint of the +exposure. + +{\centering% + \tabcaption{IPP-MOPS Data Types Definition\label{table-datatypes}} + \begin{longtable}{|cp{5in}|} + \hline + \textbf{Alias} & \textbf{Description} \\ + \texttt{bool} & (Not a FITS type) An enumerated type defining a Boolean + value, either \texttt{true} or \texttt{false}\\ + \texttt{char($n$)} & (Not a FITS type) A sequence of exactly $n$ bytes that + can be interpreted + as a sequence of ASCII characters\\ + \texttt{HH:MM:SS.sss} & (Not a FITS type) a \texttt{char(12)} encoding + a time or an angle: (1) the hours \texttt{HH} are encoded with two + decimal figures; (2) a separator \texttt{:} between the hours and + the minutes; (3) the minutes \texttt{MM} are encoded with two + decimal figures; (4) a separator \texttt{:} between the minutes + and the seconds; (5) the seconds \texttt{SS} are encoded with two + decimal figures; (6) a separator \texttt{.} between the seconds + and the milli(arc)seconds; (7) + the milli(arc)seconds \texttt{ss} are encoded with three decimal figures; \\ + \texttt{sDD:MM:SS} & (Not a FITS type) a \texttt{char(9)} encoding an + angle. (1) \texttt{s} is either the \texttt{+} or \texttt{-} sign; + (2) \texttt{DD} is the number of degrees encoded with two decimal + figures; (3) a separator \texttt{:} between the degrees and the + minutes; (4) the minutes \texttt{MM} are encoded with two decimal + figures; (5) a separator \texttt{:} between the minutes and the + seconds; (6) the + seconds \texttt{SS} are encoded with two decimal figures;\\ + \texttt{enum} & (Not a FITS type) a \texttt{string} having a fixed number of + different values. The explicit set of values shall be expressed in + this ICD.\\ + \texttt{F32} & (FITS type) IEEE 32-bit single precision floating point real numbers \\ + \texttt{F64} & (FITS type) IEEE 64-bit double precision floating point real numbers \\ + \texttt{S16} & (FITS type) 16-bit signed integer \\ + \texttt{S32} & (FITS type) 32-bit signed integer \\ + \texttt{S64} & (FITS type) 64-bit signed integer \\ + \texttt{string} & (Not a FITS type) Sequence of bytes of arbitrary length + usually interpreted as a sequence of ASCII characters \\ + \texttt{U16} & (FITS type) 16-bit unsigned integer \\ + \texttt{U32} & (FITS type) 32-bit unsigned integer \\ + \hline + \end{longtable} +} + +In addition to the required BINTABLE extension header keywords +described above (TODO: Where? What?), the FITS header shall include the +keywords shown in tables relative to versioning +\ref{table-keywords-versioning}, data provenance +\ref{table-keywords-provenance}, and exposure +\ref{table-keywords-exposure}. The subsequent FITS table, with EXTNAME +of \code{MOPS_TRANSIENT_DETECTIONS}, shall include columns whose +definitions are listed in table \ref{table-detections} and, when trail +fitting has been succesfully performed, \ref{table-detections-ext}. + +\begin{center} + \begin{longtable}{|lcp{0.5in}p{3in}|} + \caption{IPP-MOPS Transient Detection Keywords:\\ Version Information}\label{table-keywords-versioning}\\ + \hline\hline + \multicolumn{4}{|l|}{\null\hspace*{1in}\textbf{Version Information}}\\ + \hline\hline + {\bf FITS Keyword} + & \parbox{0.5in}{\textbf{Version}} + & {\bf Precision} + & {\bf Description} \\ + \hline + SWSOURCE & 1+ & \texttt{string} & source of software (e.g., "60eb6cdc-a59c-4636-a4e0-dba66a9721fd") \\ + \hline + SWVERSN & 1+ & \texttt{string} & version of software (e.g., "trunk/ppMops@24658") \\ + \hline + HIERARCH CMFVERSION & 2+ & \texttt{enum} & \texttt{PS1\_DV3} \\ + \hline\hline + \end{longtable} +\end{center} + +\begin{center} + \begin{longtable}{|lcp{0.5in}p{3in}|} + \caption{IPP-MOPS Transient Detection Keywords:\\ Provenance Information}\label{table-keywords-provenance}\\ + \hline\hline + \multicolumn{4}{|l|}{\null\hspace{10mm}\textbf{Provenance Information}}\\ + \hline \hline + {\bf FITS Keyword} + & \parbox{0.5in}{\textbf{Version}} + & {\bf Precision} + & {\bf Description} \\ + \hline + EXP\_NAME & 1+ & \texttt{string} & Exposure name (e.g., "o1234g5678")\\ + \hline + EXP\_ID & 1+ & \texttt{S64} & Exposure identifier \\ + \hline + CHIP\_ID & 1+ & \texttt{S64} & Chip stage identifier \\ + \hline + CAM\_ID & 1+ & \texttt{S64} & Camera stage identifier \\ + \hline + FAKE\_ID & 1+ & \texttt{S64} & Fake stage identifier \\ + \hline + WARP\_ID & 1+ & \texttt{S64} & Warp stage identifier \\ + \hline + DIFF\_ID & 1+ & \texttt{S64} & Diff stage \\ + \hline + DIFF\_POS & 1+ & \texttt{bool} & Sense of subtraction; \texttt{true} for forward, + \texttt{false} for backward\\ + \hline + \end{longtable} +\end{center} + +\begin{center} + \begin{longtable}{|lcp{0.5in}p{3in}c|} + \caption{IPP-MOPS Transient Detection Keywords:\\ Exposure Details}\label{table-keywords-exposure}\\ + \hline \hline + \multicolumn{5}{|l|}{\null\hspace{10mm}\textbf{Exposure Details}}\\ + \hline \hline {\bf FITS Keyword} + & \parbox{0.5in}{\textbf{Version}} & {\bf Precision} & {\bf + Description} + & \textbf{Units}\\ + \hline MJD-OBS & 0+ & \texttt{F64} & midpoint time of the exposure + as a MJD and day fraction + & N/A\\ + \hline RA & 0+ & {\tiny \texttt{HH:MM:SS.SSS}} + & Field center RA at exposure midpoint & N/A \\ + \hline DEC & 0+ & {\tiny \texttt{sDD:MM:SS}} & + (s is + or -) Field center declination at exposure midpoint & N/A \\ + \hline EXPTIME & 0+ & \texttt{F64} + & Exposure time & seconds \\ + \hline ROTANGLE & 0+ & \texttt{F64} + & Image rotation of the y-axis from local N toward E & degrees \\ + \hline FILTER & 0+ & \texttt{char(7)} & Effective filter used for + the exposure, one of \texttt{g}, \texttt{r}, \texttt{i}, + \texttt{z}, \texttt{y}, \texttt{B}, + \texttt{V}, \texttt{w} appended with '.00000' & N/A \\ + \hline AIRMASS & 0+ & \texttt{F64} + & Airmass at exposure midpoint & N/A\\ + \hline OBSCODE & 0+ & char(5) & + MPC 3-character observatory code (F51 for PS1) & N/A \\ + \hline TEL\_ALT & 0+ & \texttt{F64} & + Telescope altitude at exposure midpoint & degrees \\ + \hline TEL\_AZ & 0+ & \texttt{F64} + & Telescope azimuth at exposure midpoint & degrees \\ + \hline SEEING & 1+ & \texttt{F32} & + Measured seeing at diff stage & arcsec \\ + \hline MAGZP & 1+ & \texttt{F32} & + Magnitude zero point & N/A \\ + \hline MAGZPERR & 1+ & \texttt{F32} & + Error in magnitude zero point & N/A \\ + \hline ASTRORMS & 1+ & \texttt{F32} & + RMS of astrometric fit & arcsec \\ + \hline CMMTOBS & 3+ & \texttt{string} & + Comment associated to the exposure & N/A\\ + \hline OBS\_MODE & 3+ & \texttt{enum} & Observation modes list: + 3PI, CNP, ESS, M31, MD, MSS, OSS, PI, STD (Note: The exposures + having the following PS1 observation modes are not used by MOPS: + CAL, PR201108, SAS2, SS, STS, STS2A, STS2B) & N/A\\ + \hline DIFFTYPE & 3+ & \texttt{enum} & + Difference type: \texttt{WW} (Warp-Warp Difference), \texttt{WS} (Warp-Stack Difference) & N/A\\ + \hline SKY & 3+ & \texttt{F64} & + Exposure average sky background & counts/pixel \\ + \hline SHUTOUTC & 3+ & \texttt{string} & + Camera exposure shutter open (UTC) & N/A \\ + \hline \hline + \end{longtable} +\end{center} + +\begin{center} + \begin{longtable}{|lclp{3.5in}c|} + \caption{IPP-MOPS Transient Detection Table Columns:\\ Detection Details}\label{table-detections}\\ + \hline \hline + \multicolumn{5}{|l|}{\null\hspace{10mm}\textbf{Detection Details}}\\ + \hline \hline + {\bf Column} & \textbf{Version} & {\bf Precision} & {\bf Description} & \textbf{Units}\\ + \hline + RA & 1+ & F64 & Detection center coordinates & degrees\\ + \hline + RA\_ERR & 1+ & F64 & Error in the detection center & degrees \\ + \hline + DEC & 1+ & F64 & Detection center coordinates & degrees \\ + \hline + DEC\_ERR & 1+ & F64 & Error in the detection center & degrees \\ + \hline + MAG & 1+ & F32 & Filter magnitude & N/A \\ + \hline + MAG\_ERR & 1+ & F32 & Filter magnitude error & N/A \\ + \hline + PSF\_CHI2 & 1+ & F32 & $\chi^2$ of PSF fit & N/A \\ + \hline + PSF\_DOF & 1+ & S32 & Degrees of freedom of PSF fit & N/A \\ + \hline + CR\_SIGNIFICANCE & 1+ & F32 & Significance of cosmic ray (source: \emph{src/psphot/psphotSourceSize.c}) & N/A \\ + \hline + EXT\_SIGNIFICANCE & 1+ & F32 & Significance of extendedness (source: \emph{src/psphot/psphotSourceSize.c}) & N/A \\ + \hline + PSF\_MAJOR & 1+ & F32 & PSF major axis & pixels \\ + \hline + PSF\_MINOR & 1+ & F32 & PSF minor axis & pixels \\ + \hline + PSF\_THETA & 1+ & F32 & PSF position angle on chip & degrees \\ + \hline + PSF\_QUALITY & 1+ & F32 & PSF quality factor & N/A \\ + \hline + PSF\_NPIX & 1+ & S32 & Number of pixels in PSF & N/A \\ + \hline + MOMENTS\_XX & 1+ & F32 & xx moment & N/A \\ + \hline + MOMENTS\_XY & 1+ & F32 & xy moment & N/A \\ + \hline + MOMENTS\_YY & 1+ & F32 & yy moment & N/A \\ + \hline + N\_POS & 1+ & S32 & Number of positive pixels & N/A \\ + \hline + F\_POS & 1+ & F32 & Fraction of positive pixels & N/A \\ + \hline + RATIO\_BAD & 1+ & F32 & Ratio of positive pixels to negative & N/A \\ + \hline + RATIO\_MASK & 1+ & F32 & Ratio of positive pixels to masked & N/A \\ + \hline + RATIO\_ALL & 1+ & F32 & Ratio of positive pixels to all & N/A \\ + \hline + FLAGS & 1+ & U32 & Detection bit flags\footnote{See \htmladdnormallink{http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PSPSFlags\#StackDetectionFull.infoFlagsFLAGS232FLAGS}{http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PSPSFlags\#StackDetectionFull.infoFlagsFLAGS232FLAGS} for the meaning of each bit}. The PM\_SOURCE\_MODE\_EXTENDED\_FIT flag (at position 0x00800000) is set when the Trail Fitting is attempted. It does not mean that the Trail Fitting is successful. \\ + \hline DIFF\_SKYFILE\_ID & 1+ & S64 & Identifier for diff + skyfile. Either a diff\_id (for Warp-Warp difference) or a + stack\_id + (for Warp-Stack difference) & N/A \\ + \hline + IPP\_IDET & 2+ & U32 & IPP detection identifier index & N/A \\ + \hline + PSF\_INST\_FLUX & 2+ & F32 & PSF fit instrumental magnitude & N/A \\ + \hline + PSF\_INST\_FLUX\_SIG & 2+ & F32 & Sigma of PSF instrumental magnitude & N/A \\ + \hline + AP\_MAG & 2+ & F32 & magnitude in standard aperture (i.e. 25 pixels radius) & N/A \\ + \hline + AP\_MAG\_RAW & 2+ & F32 & magnitude in real aperture & N/A \\ + \hline + AP\_MAG\_RADIUS & 2+ & F32 & radius used for aperture mags & pixels \\ + \hline + AP\_FLUX & 2+ & F32 & instrumental flux in standard aperture & ADU \\ + \hline + AP\_FLUX\_SIG & 2+ & F32 & aperture flux error & N/A \\ + \hline + PEAK\_FLUX\_AS\_MAG & 2+ & F32 & Peak flux expressed as magnitude & N/A \\ + \hline + CAL\_PSF\_MAG & 2+ & F32 & PSF Magnitude using supplied calibration & N/A \\ + \hline + CAL\_PSF\_MAG\_SIG & 2+ & F32 & Measured standard-deviation of zero point calibration & N/A \\ + \hline + SKY & 2+ & F32 & Sky level & counts/pixel \\ + \hline + SKY\_SIGMA & 2+ & F32 & Sigma of sky level & counts/pixel\\ + \hline PSF\_QF\_PERFECT & 2+ & F32 & PSF coverage/quality factor + (poor): PSF value weighted by unmasked pixels + (source: psmodules/src/objects/pmSourcePhotometry.c) & N/A \\ + \hline + MOMENTS\_R1 & 2+ & F32 & first radial moment & N/A \\ + \hline + MOMENTS\_RH & 2+ & F32 & half radial moment & N/A \\ + \hline + KRON\_FLUX & 2+ & F32 & Kron Flux (that is, flux in 2.5 $\times$ first radial moment) & N/A \\ + \hline + KRON\_FLUX\_ERR & 2+ & F32 & Kron Flux Error & N/A \\ + \hline + KRON\_FLUX\_INNER & 2+ & F32 & Inner Kron Flux (that is, flux in 1.0 $\times$ first radial moment) & N/A \\ + \hline + KRON\_FLUX\_OUTER & 2+ & F32 & Outer Kron Flux (that is, flux in 4.0 $\times$ first radial moment) & N/A \\ + \hline + DIFF\_R\_P & 2+ & F32 & distance from detection to positive matching source & pixels \\ + \hline + DIFF\_SN\_P & 2+ & F32 & signal-to-noise of pos match src & N/A \\ + \hline + DIFF\_R\_M & 2+ & F32 & distance from detectionto negative match source & pixels \\ + \hline + DIFF\_SN\_M & 2+ & F32 & signal-to-noise of neg match src & N/A \\ + \hline + FLAGS2 & 2+ & U32 & psphot analysis flags (group 2)\footnote{See \htmladdnormallink{http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PSPSFlags\#StackDetectionFull.infoFlagsFLAGS232FLAGS}{http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/PSPSFlags\#StackDetectionFull.infoFlagsFLAGS232FLAGS} for the meaning of each bit}\\ + \hline + N\_FRAMES & 2+ & U16 & Number of exposures overlapping source center \\ + \hline + PADDING & 2+ & S16 & padding \\ + \hline \hline + \end{longtable} +\end{center} + +\begin{center} + \begin{longtable}{|lclp{3.5in}c|} + \caption{IPP-MOPS Transient Detection Table Columns:\\ Extended (Trail Fitting) Details}\label{table-detections-ext}\\ + \hline \hline + \multicolumn{5}{|c|}{\textbf{Extended (trail fitting) parameters}}\\ + \multicolumn{5}{|c|}{Values like \texttt{NaN} mean that the extended parameters are undefined for the detection}\\ + \hline + X\_EXT & 2+ & F32 & + Extended model X coordinate & pixels\\ + \hline + Y\_EXT & 2+ & F32 & + Extended model Y coordinate & pixels \\ + \hline + X\_EXT\_SIG & 2+ & F32 & + Sigma in extended model X coordinate & pixels \\ + \hline + Y\_EXT\_SIG & 2+ & F32 & + Sigma in extended model Y coordinate & pixels \\ + \hline + EXT\_INST\_MAG & 2+ & F32 & + Extended fit instrumental magnitude & N/A \\ + \hline + EXT\_INST\_SIG\_MAG & 2+ & F32 & + Sigma of extended fit instrumental magnitude & N/A \\ + \hline + NPARAMS & 2+ & S32 & + Number of model parameters (see \cite{VerJedDen2012IAAaPwTF}) & N/A \\ + \hline + EXT\_WIDTH\_MAJ & 2+ & F32 & % The name is weird but it's because of normalization + Length of trail & pixels \\ % in the IPP: Check if MOPS want to change it + \hline + EXT\_WIDTH\_MIN & 2+ & F32 & % The name is weird but it's because of normalization + Sigma of trail & pixels \\ % in the IPP: Check if MOPS want to change it + \hline + EXT\_THETA & 2+ & F32 & EXT orientation angle & radians \\ + \hline + EXT\_WIDTH\_MAJ\_ERR & 2+ & F32 & EXT width error (major axis) & pixels \\ + \hline + EXT\_WIDTH\_MIN\_ERR & 2+ & F32 & EXT width error (minor axis) & pixels \\ + \hline + EXT\_THETA\_ERR & 2+ & F32 & EXT orientation angle (error) & radians \\ + \hline + RA\_EXT & 3+ & F32 & Fitted centroid RA & degrees \\ + \hline + RA\_EXT\_SIGMA & 3+ & F32 & Fitted RA sigma & degrees? \\ + \hline + DEC\_EXT & 3+ & F32 & Fitted centroid DEC & degrees \\ + \hline + DEC\_EXT\_SIGMA & 3+ & F32 & Fitted DEC sigma & degrees? \\ + \hline + POSANG\_EXT & 3+ & F32 & Fitted position angle & degrees \\ + \hline + PLTSCALE\_EXT & 3+ & F32 & Plate scale at centroid & arcsec/pixel \\ + \hline + EXT\_FLUX & 3+ & F32 & Fitted flux & ADU \\ + \hline + EXT\_CAL\_MAG & 3+ & F32 & Calibrated magnitude & N/A \\ + \hline + EXT\_MAG\_SIG & 3+ & F32 & Sigma of calibrated magnitude & N/A \\ + \hline + EXT\_CHISQ & 3+ & F32 & $\chi^2$ of fit & N/A \\ + \hline + EXT\_NDOF & 3+ & S32 & Fit degrees of freedom & N/A \\ + \hline + \end{longtable} +\end{center} + +\subsection{MOPS Detectability Query} + +MOPS will query the IPP regarding the detectability of hypothetical +elongated detections specified using ($RA_0$, $Dec_0$, $Time_0$, +$RA_1$, $Dec_1$, $Time_1$) tuples by submitting all information +required to construct a FPA PSF at the time of the detections. It is +the responsibility of MOPS to generate and manage unique query IDs. + +All MOPS detectability queries will occur with respect to an already-ingested +FPA transient detection list. The query being asked by MOPS is essentially +whether a particular trail in an acquired FPA image was masked for any +reason and to what degree. Thus the originating IPP FPA\_ID is required. + +MOPS will simulate elongated (``trailed'') PSFs, relevant for +fast-moving objects. Since some or all of a trail may be masked, MOPS +will for all detections provide starting and ending RA and Dec. For +most detections these two positions will be nearly coincident, +appearing as a circular point-source PSF, but for fast-moving objects +these positions may differ by many pixels in the FPA. + +In addition to the required BINTABLE extension header keywords +described above, the FITS header for the detectability query shall +include the following required keywords shown in table +\ref{detquery-keywords}. The subsequent FITS table, with EXTNAME of +\code{MOPS_DETECTABILITY_QUERY}, shall include columns whose +definitions are listed in table \ref{detquery-table}. + +% \begin{table}[htbp] +% \begin{center} +{\centering% +\small + \tabcaption{MOPS-IPP Detectability Query Keywords\label{detquery-keywords}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column} & {\bf Precision} & {\bf Description} \\ +\hline +QUERY\_ID & char(20) & MOPS query ID for this batch query \\ +FPA\_ID & char(20) & original FPA\_ID used at ingest \\ +MJD-OBS & F64 & starting time of the exposure, MJD \\ +FILTER & char(3) & effective filter used for the exposure as a string, allowed values of g, r, i, z, y, B, V, w \\ +OBSCODE & char(3) & site identifier (MPC observatory code) \\ +\end{tabular} +} +% \end{center} +% \end{table} + + +% \begin{table}[htbp] +% \begin{center} +{\centering% +\small +\tabcaption{MOPS-IPP Detectability Query Table Columns\label{detquery-table}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column} & {\bf Precision} & {\bf Description} \\ +\hline +ROWNUM & char(20) & identifier for this row in the table \\ +RA1\_DEG & F64 & coordinate at the start of exposure, in degrees \\ +DEC1\_DEG & F64 & coordinate at the start of exposure, in degrees \\ +RA2\_DEG & F64 & coordinate at the end of exposure, in degrees \\ +DEC2\_DEG & F64 & coordinate at the end of exposure, in degrees \\ +MAG & F64 & apparent magnitude \\ +\end{tabular} +} +% \end{center} +% \end{table} + +\subsection{IPP Detectability Response} + +IPP will return a table listing the submitted query detections and a +pair of values $(N, f)$ indicating the number of pixels relevant to the detection's PSF +and a fraction of pixels detectable by IPP. + +For infrequent cases where a trail may completely traverse a masked area (due to chip gap, streak removal, or similar), the response may be erroneous or incomplete. This error and its effects will be studied during PS-1 operations. + +In addition to the required BINTABLE extension header keywords +described above, the FITS header for the detectability response shall include +the following required keywords shown in table \ref{detresponse-keywords}. +The subsequent FITS table, with EXTNAME of \code{MOPS_DETECTABILITY_RESPONSE}, shall include +columns whose definitions are listed in table \ref{detresponse-table}. + +{\centering% +% \begin{table}[htbp] +% \begin{center} +\small +\tabcaption{MOPS-IPP Detectability Response Keywords\label{detresponse-keywords}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column} & {\bf Precision} & {\bf Description} \\ +\hline +QUERY\_ID & char(20) & originating MOPS query ID \\ +FPA\_ID & char(20) & IPP FPA ID used in detectability request \\ +\end{tabular} +} +% \end{center} +% \end{table} + +% \begin{table}[htbp] +% \begin{center} +{\centering% +\small +\tabcaption{MOPS-IPP Detectability Response Table Columns\label{detresponse-table}} +\begin{tabular}{lll} +\hline +\hline +{\bf Column} & {\bf Precision} & {\bf Description} \\ +\hline +ROWNUM & char(20) & matching rownum from original request \\ +DETECT\_N & U32 & number of pixels used in the hypothetical PSF for the query detection \\ +DETECT\_F & F64 & detectability, indicating the fraction of PSF pixels detectable by IPP \\ +\end{tabular} +} +% \end{center} +% \end{table} + +\pagebreak + +\section{Changes} + +\subsection{Changes in PS1\_DV3 from PS1\_DV2} + +\subsubsection{New Keywords} + +\paragraph{Per Exposure (in FITS header)} +\begin{itemize} +\item CMMTOBS +\item OBS\_MODE +\item DIFFTYPE +\item SKY +\item SHUTOUTC +\end{itemize} + +\paragraph{Per Detection (in FITS extension)} +\begin{itemize} +\item RA\_EXT +\item RA\_EXT\_SIGMA +\item DEC\_EXT +\item DEC\_EXT\_SIGMA +\item POSANG\_EXT +\item PLTSCALE\_EXT +\item EXT\_FLUX +\item EXT\_CAL\_MAG +\item EXT\_MAG\_SIG +\item EXT\_CHISQ +\item EXT\_NDOF +\end{itemize} + +\subsubsection{Removed Keywords} + +None + +\subsubsection{Datatypes} +\begin{itemize} +\item NPARAMS is now a S32 and no more a F32. +\end{itemize} + +\subsubsection{Known Bugs and Associated Fix} + +\begin{itemize} +\item None +\end{itemize} + +\newpage + +\appendix + +\section{The IPP Data Workflow} + +The objective of this section is to provide some details about the IPP +Workflow. It is intentionnally MOPS oriented in the sense that some +details have been voluntarily omitted. + +\begin{minipage}{0.65\linewidth} + The IPP receives "exposures" from the telescope. An "exposure" is an + image of 60 OTAs (see figure~\ref{fig-exposure-otas}) taken by the + GPC1 camera and its associated metadata. All science exposures + (which are interesting MOPS) are processed in the IPP according to + the following stages. + + The first stage is the CHIP stage. Detrending and objects detections + are performed on each OTA at this stage. + + The second stage is the CAMERA stage. Calibration and astrometry are + performed on the exposure at this stage. + + The third stage is the WARP stage\footnote{For simplicity and since + nothing is performed in this stage, the FAKE stage has been + ommitted.}. The image pixels are warped to fit a predefined sky + tessellation (it is actually a covering). Cells of the tessellation + are called \emph{skycells} and each pixel in a skycell has a size of + $0.25" \times 0.25"$. The skycell is oriented so that increasing X + (wrt Y) pixel coordinate corresponds to a move toward the West (wrt + North). + + The fourth stage is the DIFF stage. The difference between two + skycells having the same spatial coordinates but being distant in + time (by less than the TTI) is computed. To avoid deconvolution, + the image difference can be performed in bothways When the newer + image is diffed with the older image, the difference image is called + the \emph{positive} image. When the older image is diffed with the + newer image, the difference image is called the \emph{negative} + image. + + Another stage is executed in parallel of the fourth stage: the STACK + stage. Its products (the stacks, that is, "average" observations in + a skycell) can be used in the DIFF stage to compute WARP-STACK + differences (e.g for MD observations). STACK-WARP are not computed + (so far). + + The fifth and last stage is the PPMOPS stage. The exposure is + rebuilt from the differences obtained in each skycell of the DIFF + stage. For detections falling in areas covered by two or more + skycells, the data of the detection being the nearest to a skycell + center are taken. +\end{minipage}\hfill\begin{minipage}{0.3\linewidth} + \begin{center} + \includegraphics[width=\linewidth]{figures/ipp_workflow.eps} + \captionof{figure}{Caption for figure} + \label{fig-ipp-workflow} + \end{center} +\end{minipage} + +\begin{center} + \includegraphics[width=4in]{figures/exposure_ota.eps} + \captionof{figure}{Exposure / OTA TODO + Move somewhere else} + \label{fig-exposure-otas} +\end{center} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\newpage +\section{\texttt{ppMops} Implementation Details} +\label{sec-ppmops} + +\subsection{Source Code} +\label{sec-ppmops-sourcecode} + +The \texttt{ppMops} source code can be found at +\url{https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/ppTranslate}. The +SVN revision of the analyzed code as well as the corresponding +software version depends on the version of this document and can be +found in the History subsection (see subsection +\ref{sec-ppmops-sourcecodehistory}). + +\subsection{Source Code Analysis History} +\label{sec-ppmops-sourcecodehistory} +\begin{tabular}{|c|c|c|p{2in}|c|c|} + \hline + \parbox{.5in}{Doc Version} & Date & Author & Description & SVN Rev. & \parbox{.5in}{Software Version}\\ + \hline + \hline + --- & 2012-09-14 & SC & Creation & 34439 & 3\footnotemark{}\\ + \hline +\end{tabular}\footnotetext{Modified version 3} + +\subsection{Main} +Source code: \texttt{src/ppMops.c}. + +\texttt{ppMops} can be run on the command line independently from the +IPP framework. It is functionnally made of four parts: +\begin{enumerate} +\item Analyzing the command-line arguments (see subsection + \ref{sec-implementation-args}); +\item Reading the detections from the input diff files and the exposure + parameters (see subsection \ref{sec-implementation-read}); +\item Removing the duplicated detections (see subsection + \ref{sec-implementation-merge}); +\item Writing the products (see subsection + \ref{sec-implementation-write}); +\end{enumerate} + +\subsection{Command-Line Arguments Analysis} +\label{sec-implementation-args} + +Source code: \texttt{src/ppMopsArguments.c}. + +\subsubsection{Parameters} + +There are only two mandatory arguments: +\begin{enumerate} +\item The first argument is a filename. The associated file contains + the list of the names of all the diff files that will be used as + input. For simplicity, we will identify that list by: + \texttt{list\_of\_diffs\_inputs}. +\item The second argument is the name of the output file. We will call + it \texttt{output\_file\_name}. +\end{enumerate} + +The following parameters can be taken from the command line arguments +options (if they are provided, otherwise they take the default value): +\begin{itemize} +\item \texttt{exp\_name}: exposure name (default: NULL); +\item \texttt{exp\_id}: Exposure identifier (IPP internal; default: 0); +\item \texttt{chip\_id}: Chip stage identifier (IPP internal; default: 0); +\item \texttt{cam\_id}: Camera stage identifier (IPP internal; default: 0); +\item \texttt{fake\_id}: Fake stage identifier (IPP internal; default: 0); +\item \texttt{warp\_id}: Warp stage identifier (IPP internal; default: 0); +\item \texttt{diff\_id}: Diff stage identifier (IPP internal; default: 0); +\item \texttt{inverse}: Inverse subtraction (boolean; default: false); +\item \texttt{zp}: Magnitude zero point (default: NAN); +\item \texttt{zp\_error}: Error in magnitude zero point (default: NAN); +\item \texttt{astrom\_rms}: Astrometric solution RMS (default: NAN); +\item \texttt{version}: CMF file version (default: 0); +\end{itemize} + +\subsubsection{Parameters Values Validation} +\begin{itemize} +\item If the list of input file names is empty, an error is displayed + and \texttt{ppMops} exits. +\item There is no check on the validity of the different parameters + (values, files existence...). +\end{itemize} + +\subsection{Reading Detections} +\label{sec-implementation-read} + +Source code: \texttt{src/ppMopsRead.c}. + +Each file from the \texttt{list\_of\_diffs\_inputs} is opened. A +\texttt{ppMopsDetections} object is populated from the data that are +read from that file. The set (namely a \texttt{psArray}) of all the +valid \texttt{ppMopsDetections} objects is the product of this part. + +The data which are extracted from each file are the following: +\begin{itemize} +\item From the header are extracted: + \begin{itemize} + \item \texttt{diffSkyfileId}: Its value is + associated to the keyword \texttt{IMAGEID}. If not present or 0, the + program exits with an error. + \item The current diff version is derived from the value associated + to EXTTYPE. If the \texttt{version} has not already been set, it + is set to the value which has just been read. If it was set at + some point (either through the command line, or from a previous + input), the current diff version value is checked against the + \texttt{version}. Different values will produce warning messages. + \item \texttt{raBoresight} is given the value associated to + \texttt{FPA.RA}; + \item \texttt{decBoresight} is given the value associated to + \texttt{FPA.DEC}; + \item \texttt{filter} is given the value associated to + \texttt{FPA.FILTER}; + \item \texttt{airmass} is given the value associated to + \texttt{AIRMASS}; + \item \texttt{exptime} is given the value associated to + \texttt{EXPTIME}; + \item \texttt{posangle} is given the value associated to + \texttt{FPA.POSANGLE}; + \item \texttt{alt} is given the value associated to + \texttt{FPA.ALT}; + \item \texttt{az} is given the value associated to + \texttt{FPA.AZ}; + \item \texttt{mjd} is computed from the \texttt{mjdobs} value + associated to \texttt{MJD-OBS} by: \[\mathtt{mjd} = \mathtt{mjdobs} + + \frac{\mathtt{exptime}}{2*3600*24}\] + \item \texttt{seeing} is initialized (note that it is modified in + the loop over the valid detections) using + $\mathtt{fwhm}_\mathtt{maj}$ (wrt $\mathtt{fwhm}_\mathtt{min}$) + associated to the keyword \texttt{FWHM\_MAJ} (wrt + \texttt{FWHM\_MIN}); + \item \texttt{naxis1} is set to the value associated to + \texttt{IMNAXIS1}, that is the number of columns of the original + image; + \item \texttt{naxis2} is set to the value associated to + \texttt{IMNAXIS2}, that is the number of rows of the original + image; + \end{itemize} +\item From the \texttt{SkyChip.psf} extension: + \begin{itemize} + \item This extension contains all the detections. One row is + associated to each detection. + \item A detection is rejected if at least one of the following + parameters has an non-finite value: \texttt{x}, \texttt{y}, + \texttt{ra}, \texttt{dec}, \texttt{mag}, \texttt{magErr}, + \texttt{xErr}, \texttt{yErr}, \texttt{scale}, \texttt{angle}. + \item A detection is rejected if its associated \texttt{flags} + matches: + {\small + \texttt{PM\_SOURCE\_MODE\_FAIL | PM\_SOURCE\_MODE\_BADPSF | + PM\_SOURCE\_MODE\_SATURATED | PM\_SOURCE\_MODE\_CR\_LIMIT | + PM\_SOURCE\_MODE\_SKY\_FAILURE}} + \item \texttt{x} is associated to the \texttt{X\_PSF} keyword; + \item \texttt{y} is associated to the \texttt{Y\_PSF} keyword; + \item \texttt{ra} is associated to the \texttt{RA\_PSF} keyword. It + is converted to radians; + \item \texttt{dec} is associated to the \texttt{DEC\_PSF} keyword. It + is converted to radians; + \item \texttt{mag} is computed from the value associated the + \texttt{PSF\_INST\_MAG} keyword, namely: + \[ + \mathtt{mag} = \mathtt{PSF\_INST\_MAG} + \mathtt{zp} + \] + where \texttt{zp} is defined in the arguments list (see subsection + \ref{sec-implementation-args}); + \item \texttt{magErr} is associated to the + \texttt{PSF\_INST\_MAG\_SIG} keyword; + \item \texttt{xErr} is associated to the \texttt{X\_PSF\_SIG} + keyword; + \item \texttt{yErr} is associated to the \texttt{Y\_PSF\_SIG} + keyword; + \item \texttt{scale} is associated to the \texttt{PLTSCALE} + keyword; + \item \texttt{angle} is associated to the \texttt{POSANGLE} + keyword; + \item \texttt{flags} is associated to the \texttt{FLAGS} keyword; + \item \texttt{raErr} is defined by: + \[ + \mathtt{raErr} = \frac{\mathtt{scale}}{3600} + \sqrt{\cos^2(\mathtt{angle}) \mathtt{xErr}^2 + + \sin^2(\mathtt{angle}) \mathtt{yErr}^2} + \] + \item \texttt{decErr} is defined by: + \[ + \mathtt{decErr} = \frac{\mathtt{scale}}{3600} + \sqrt{\sin^2(\mathtt{angle}) \mathtt{xErr}^2 + + \cos^2(\mathtt{angle}) \mathtt{yErr}^2} + \] + \end{itemize} +\item \texttt{seeing} is defined as the product of the mean of + $\mathtt{fwhm}_\mathtt{maj}$ and $\mathtt{fwhm}_\mathtt{min}$ by the + mean of the \texttt{scale} values for valid detections, that is: + \[ + \mathtt{seeing} = \frac{\mathtt{fwhm}_\mathtt{maj} + + \mathtt{fwhm}_\mathtt{min}}{2} \times \frac{\sum_{\mathtt{valid\ + detections}}(scale)}{\#(\mathtt{valid\ detections})} + \] +\item The number of valid detections is also kept (but not used). +\item For each row of the \texttt{SkyChip.xfit} extension: + \begin{itemize} + \item This extension contains only a subset of the detections. Each + row contains the index of the current detection which is the value + associated to the keyword \texttt{IPP\_IDET}; + \item Note that detections parameters whose index could not be found + in the \texttt{SkyChip.xfit} extension are set to NaN. + \item The following columns are extracted \texttt{X\_EXT}, + \texttt{Y\_EXT}, \texttt{X\_EXT\_SIG}, \texttt{Y\_EXT\_SIG}, + \texttt{EXT\_INST\_MAG}, \texttt{EXT\_INST\_MAG\_SIG}, + \texttt{NPARAMS}, \texttt{EXT\_WIDTH\_MAJ}, \texttt{EXT\_WIDTH\_MIN}, + \texttt{EXT\_THETA}, \texttt{EXT\_WIDTH\_MAJ\_ERR}, + \texttt{EXT\_WIDTH\_MIN\_ERR}, and \texttt{EXT\_THETA\_ERR}. No check is + performed on them at this stage. + \end{itemize} +\end{itemize} + +At the end of this stage, all input files have been used to populate a +collection of sets of detections. Let us call that collection $({\cal + D}_{i})_{i \in \{1..n\}}$. Each of those sets contains data read +from one file and it is important to note that only a few checks have +been performed on the data (more precisely, only non-masked detections +which have defined values have been inserted). The following cases can +therefore happen: (1) Any ${\cal D}_i$ can be "empty", that is, does +not contain any valdiated detection; (2) Some parameters like +\texttt{seeing} can have an undefined value (e.g. \texttt{NaN}). + +\subsection{Merging Detections} +\label{sec-implementation-merge} + +Source code: \texttt{src/ppMopsMerge.c}. + +That part of the code essentially uses a kd-tree implementation to +compare distances between detections. The collection of sets of +detections will be called $({\cal D}_{i})_{i \in \{1..n\}}$ (see +previous section). + +\subsubsection{Checks} +\label{sec-implementation-merge-checks} + +Checks are performed on data that have been extracted from FITS files +headers. Namely, they are: +\begin{itemize} +\item \texttt{raBoresight} constancy across all the detections; +\item \texttt{decBoresight} constancy across all the detections; +\item \texttt{filter} constancy across all the detections; +\item \texttt{airmass} constancy across all the detections; +\item \texttt{exptime} constancy across all the detections; +\item \texttt{posangle} constancy across all the detections; +\item \texttt{alt} constancy across all the detections; +\item \texttt{az} constancy across all the detections; +\item \texttt{mjd} constancy across all the detections; +\end{itemize} + +Any inconsistency displays an error and the program exits. + +\subsubsection{kd-Tree Merging} +\label{sec-implementation-merge-kdtree} + +All valid detections are inserted into a kd-tree (using the +\texttt{psTree} implementation). \texttt{ra} and \texttt{dec} are used +as keys for the nearest neighbor search. An iteration over all valid +detections is then performed. + +The detection in a 1 arcsec neighborhood around the current valid +detection which is the closest to the image center is chosen as the +best candidate for the source. Then, the best of the candidates is +chosen as the source. All other detections are considered duplicates, +and marked as invalid, then removed from the detections set to which +they belong. + +Note that \texttt{numGood} which is supposed to give the number of +valid detections is not updated. + +\subsection{Writing Final Product} +\label{sec-implementation-write} + +Source code: \texttt{src/ppMopsWrite.c}. + +To resolve any ambiguity, we will note the parameter named +\texttt{param} which was built for the set of detections ${\cal + D}_{i}$ will be denoted by ${\cal D}_{i}(\mathtt{param})$. + +Note that, except for the parameters that have been checked at the +merge stage (see paragraph~\ref{sec-implementation-merge-checks}), +there is no guarantee that ${\cal D}_{i}(\mathtt{param}) = {\cal + D}_{j}(\mathtt{param})$ when $i \neq j$. There is no guarantee to +that ${\cal D}_{i}(\mathtt{param})$ is defined (in the mathematical +sense), e.g. it could be \texttt{NaN}... + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsubsection{Exposure Parameters: Header Details} +\label{sec-implementation-write-header} + +The first set of detections among the $({\cal D}_{i})_i$ that contains +at least one valid detection is chosen. To fix ideas, let us denote +it by ${\cal D}_{i_0}$. + +The following parameters are written to the product FITS header: + +\begin{itemize} +\item \texttt{SWSOURCE} (Software source): Constructed from the SVN + repository value. Generated when compiled. +\item \texttt{SWVERSN} (Software version): Constructed from the SVN + version value. Generated when compiled. +\item \texttt{EXP\_NAME} (Exposure name): Set by \texttt{exp\_name} + argument value; +\item \texttt{EXP\_ID} (Exposure identifier): Set by \texttt{exp\_id} + argument value; +\item \texttt{CHIP\_ID} (Chip stage identifier): Set by + \texttt{chip\_id} argument value; +\item \texttt{CAM\_ID} (Cam stage identifier): Set by \texttt{cam\_id} + argument value; +\item \texttt{FAKE\_ID} (Fake stage identifier): Set by + \texttt{fake\_id} argument value; +\item \texttt{WARP\_ID} (Warp stage identifier): Set by + \texttt{warp\_id} argument value; +\item \texttt{DIFF\_ID} (Diff stage identifier): Set by + \texttt{diff\_id} argument value; +\item \texttt{DIFF\_POS} (Positive subtraction): Set by + \texttt{inverse} argument value; +\item \texttt{MJD-OBS} (MJD of exposure midpoint): Set by ${\cal + D}_{i_0}(\mathtt{mjd})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{RA} (Right Ascension of boresight): Set by ${\cal + D}_{i_0}(\mathtt{raBoresight})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{DEC} (Declination of boresight): Set by ${\cal + D}_{i_0}(\mathtt{decBoresight})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{TEL\_ALT} (Telescope altitude): Set by ${\cal + D}_{i_0}(\mathtt{alt})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{TEL\_AZ} (Telescope azimuth): Set by ${\cal + D}_{i_0}(\mathtt{az})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{EXPTIME} (Exposure time (sec)): Set by ${\cal + D}_{i_0}(\mathtt{exptime})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{ROTANGLE} (Rotator position angle): Set by ${\cal + D}_{i_0}(\mathtt{rotangle})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{FILTER} (Filter name): Set by ${\cal + D}_{i_0}(\mathtt{filter})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{AIRMASS} (Airmass of exposure): Set by ${\cal + D}_{i_0}(\mathtt{airmass})$. Uniqueness guaranteed by + \ref{sec-implementation-merge-checks}; +\item \texttt{SEEING} (Mean seeing): Set by the median of defined (not + \texttt{NaN}, not \texttt{Infinity}) of $({\cal + D}_{i}(\mathtt{seeing}))_i$; + \commentFromSC{2012-09-18}{Check if it is what we + really expect. Take the example of 3 set of detections for which + \texttt{seeing} is defined, assuming the first one contains 1000 + detections with a \texttt{seeing} of 1.0, the second one contains + 10 detections with a \texttt{seeing} of 2.0, the third one 10 + detections with a \texttt{seeing} of 3.0. Do we really want the + global \texttt{seeing} value to be 2.0?}{} +\item \texttt{OBSCODE} (IAU Observatory code): Hard-coded value + defined in \texttt{src/ppMops.h} (value \texttt{F51}); + \commentFromSC{2012-09-18}{I'm surprised that it's defined locally + and not in some high-level project header file}{} +\item \texttt{MAGZP} (Magnitude zero point): Set by \texttt{zp} + argument value; +\item \texttt{MAGZPERR} (Error in magnitude zero point): Set by + \texttt{zp\_error} argument value; +\item \texttt{ASTRORMS} (Error in magnitude zero point): Set by + \texttt{astrom\_rms} argument value; +\item \texttt{CMFVERSION} (CMF version): Set by \texttt{version} + argument value but possibly modified by first \texttt{EXTTYPE} + value; + \commentFromSC{2012-09-18}{Depends on the order of the input FITS + file and their \texttt{EXTTYPE} value (e.g. if \texttt{-version 0} + is used as argument and two files are used, one with + \texttt{EXTTYPE} of 1, the other with \texttt{EXTTYPE} of + 2). However we do not expect to have different values for + \texttt{EXTTYPE}, do we?}{} +\end{itemize} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsubsection{Detection Parameters: Extension Details} +\label{sec-implementation-write-detection} + +If no detection is present in any detections set, an extension is +still created and contains exactly one row of undefined values +(\texttt{NaN} for floating-point data, 0 for integral types). + +To resolve any ambiguity, we will note the parameter named +\texttt{param} which was built for the detection $j$ in the set of +detections ${\cal D}_{i}$ will be denoted by ${\cal + D}_{ij}(\mathtt{param})$ where \texttt{param} was defined in +paragraph~\ref{sec-implementation-read}. When the \texttt{param} +parameter is the exact copy of the value associated to the keyword +\texttt{INPUT\_KEY}, we will write it as ${\cal + D}_{ij}(\mathtt{INPUT\_KEY})$ (note the use of the uppercase +letters). + +For each detection $j$ in each detections set ${\cal D}_{i}$, the +following products are written to the output FITS file: +\begin{itemize} +\item \texttt{RA} associated to ${\cal D}_{ij}(\mathtt{ra})$ (which is + identical to ${\cal D}_{ij}(\mathtt{RA\_PSF})$); +\item \texttt{RA\_ERR} associated to ${\cal D}_{ij}(\mathtt{raErr})$; +\item \texttt{DEC} associated to ${\cal D}_{ij}(\mathtt{dec})$ (which + is identical to ${\cal D}_{ij}(\mathtt{DEC\_PSF})$); +\item \texttt{DEC\_ERR} associated to ${\cal + D}_{ij}(\mathtt{decErr})$; +\item \texttt{MAG} associated to ${\cal D}_{ij}(\mathtt{mag})$; +\item \texttt{MAG\_ERR} associated to ${\cal D}_{ij}(\mathtt{magErr})$ + (which is identical to ${\cal + D}_{ij}(\mathtt{PSF\_INST\_MAG\_SIG})$); +\item \texttt{PSF\_CHI2} associated to ${\cal + D}_{ij}(\mathtt{PSF\_CHISQ})$; +\item \texttt{PSF\_DOF} associated to ${\cal + D}_{ij}(\mathtt{PSF\_NDOF})$; +\item \texttt{CR\_SIGNIFICANCE} associated to ${\cal + D}_{ij}(\mathtt{CR\_NSIGMA})$; +\item \texttt{EXT\_SIGNIFICANCE} associated to ${\cal + D}_{ij}(\mathtt{EXT\_NSIGMA})$; +\item \texttt{PSF\_MAJOR} associated to ${\cal + D}_{ij}(\mathtt{PSF\_MAJOR})$; +\item \texttt{PSF\_MINOR} associated to ${\cal + D}_{ij}(\mathtt{PSF\_MINOR})$; +\item \texttt{PSF\_THETA} associated to ${\cal + D}_{ij}(\mathtt{PSF\_THETA})$; +\item \texttt{PSF\_QUALITY} associated to ${\cal + D}_{ij}(\mathtt{PSF\_QF})$; +\item \texttt{PSF\_NPIX} associated to ${\cal + D}_{ij}(\mathtt{PSF\_NPIX})$; +\item \texttt{MOMENTS\_XX} associated to ${\cal + D}_{ij}(\mathtt{MOMENTS\_XX})$; +\item \texttt{MOMENTS\_XY} associated to ${\cal + D}_{ij}(\mathtt{MOMENTS\_XY})$; +\item \texttt{MOMENTS\_YY} associated to ${\cal + D}_{ij}(\mathtt{MOMENTS\_YY})$; +\item \texttt{N\_POS} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_NPOS})$; +\item \texttt{F\_POS} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_FRATIO})$; +\item \texttt{RATIO\_BAD} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_NRATIO\_BAD})$; +\item \texttt{RATIO\_MASK} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_NRATIO\_MASK})$; +\item \texttt{RATIO\_ALL} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_NRATIO\_ALL})$; +\item \texttt{FLAGS} associated to ${\cal D}_{ij}(\mathtt{FLAGS})$; +\item \texttt{DIFF\_SKYFILE\_ID} associated to ${\cal + D}_{i}(\mathtt{diffSkyfileId})$ (that is, the \texttt{diff\_id} of + the file which was used to compute the difference); +\item \texttt{IPP\_IDET} associated to ${\cal + D}_{ij}(\mathtt{IPP\_DET})$ (since it is the position of the + detection if the $i$th diff file, it is not very useful. However + gaps in the numbering show that detections have been removed. +\item \texttt{PSF\_INST\_FLUX} associated to ${\cal + D}_{ij}(\mathtt{PSF\_INST\_FLUX})$; +\item \texttt{PSF\_INST\_FLUX\_SIG} associated to ${\cal + D}_{ij}(\mathtt{PSF\_INST\_FLUX\_SIG})$; +\item \texttt{AP\_MAG} associated to ${\cal D}_{ij}(\mathtt{AP\_MAG})$; +\item \texttt{AP\_MAG\_RAW} associated to ${\cal + D}_{ij}(\mathtt{AP\_MAG\_RAW})$; +\item \texttt{AP\_MAG\_RADIUS} associated to ${\cal + D}_{ij}(\mathtt{AP\_MAG\_RADIUS})$; +\item \texttt{AP\_FLUX} associated to ${\cal D}_{ij}(\mathtt{AP\_FLUX})$; +\item \texttt{AP\_FLUX\_SIG} associated to ${\cal + D}_{ij}(\mathtt{AP\_FLUX\_SIG})$; +\item \texttt{PEAK\_FLUX\_AS\_MAG} associated to ${\cal + D}_{ij}(\mathtt{PEAK\_FLUX\_AS\_MAG})$; +\item \texttt{CAL\_PSF\_MAG} associated to ${\cal + D}_{ij}(\mathtt{CAL\_PSF\_MAG})$; +\item \texttt{CAL\_PSF\_MAG\_SIG} associated to ${\cal + D}_{ij}(\mathtt{CAL\_PSF\_MAG\_SIG})$; +\item \texttt{SKY} associated to ${\cal D}_{ij}(\mathtt{SKY})$; +\item \texttt{SKY\_SIGMA} associated to ${\cal D}_{ij}(\mathtt{SKY\_SIGMA})$; +\item \texttt{PSF\_QF\_PERFECT} associated to ${\cal + D}_{ij}(\mathtt{PSF\_QF\_PERFECT})$; +\item \texttt{MOMENTS\_R1} associated to ${\cal D}_{ij}(\mathtt{MOMENTS\_R1})$; +\item \texttt{MOMENTS\_RH} associated to ${\cal D}_{ij}(\mathtt{MOMENTS\_RH})$; +\item \texttt{KRON\_FLUX} associated to ${\cal D}_{ij}(\mathtt{KRON\_FLUX})$; +\item \texttt{KRON\_FLUX\_ERR} associated to ${\cal + D}_{ij}(\mathtt{KRON\_FLUX\_ERR})$; +\item \texttt{KRON\_FLUX\_INNER} associated to ${\cal + D}_{ij}(\mathtt{KRON\_FLUX\_INNER})$; +\item \texttt{KRON\_FLUX\_OUTER} associated to ${\cal + D}_{ij}(\mathtt{KRON\_FLUX\_OUTER})$; +\item \texttt{DIFF\_R\_P} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_R\_P})$; +\item \texttt{DIFF\_SN\_P} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_SN\_P})$; +\item \texttt{DIFF\_R\_M} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_R\_M})$; +\item \texttt{DIFF\_SN\_M} associated to ${\cal + D}_{ij}(\mathtt{DIFF\_SN\_M})$; +\item \texttt{FLAGS2} associated to ${\cal D}_{ij}(\mathtt{FLAGS2})$; +\item \texttt{N\_FRAMES} associated to ${\cal + D}_{ij}(\mathtt{N\_FRAMES})$; +\item \texttt{PADDING} associated to ${\cal + D}_{ij}(\mathtt{PADDING})$; +\item \texttt{X\_EXT} associated to ${\cal D}_{ij}(\mathtt{X\_EXT})$; +\item \texttt{Y\_EXT} associated to ${\cal D}_{ij}(\mathtt{Y\_EXT})$; +\item \texttt{X\_EXT\_SIG} associated to ${\cal + D}_{ij}(\mathtt{X\_EXT\_SIG})$; +\item \texttt{Y\_EXT\_SIG} associated to ${\cal + D}_{ij}(\mathtt{Y\_EXT\_SIG})$; +\item \texttt{EXT\_INST\_MAG} associated to ${\cal + D}_{ij}(\mathtt{EXT\_INST\_MAG})$; +\item \texttt{EXT\_INST\_MAG\_SIG} associated to ${\cal + D}_{ij}(\mathtt{EXT\_INST\_MAG\_SIG})$; +\item \texttt{NPARAMS} associated to ${\cal + D}_{ij}(\mathtt{NPARAMS})$; +\item \texttt{EXT\_WIDTH\_MAJ} associated to ${\cal + D}_{ij}(\mathtt{EXT\_WIDTH\_MAJ})$; +\item \texttt{EXT\_WIDTH\_MIN} associated to ${\cal + D}_{ij}(\mathtt{EXT\_WIDTH\_MIN})$; +\item \texttt{EXT\_THETA} associated to ${\cal + D}_{ij}(\mathtt{EXT\_THETA})$; +\item \texttt{EXT\_WIDTH\_MAJ\_ERR} associated to ${\cal + D}_{ij}(\mathtt{EXT\_WIDTH\_MAJ\_ERR})$; +\item \texttt{EXT\_WIDTH\_MIN\_ERR} associated to ${\cal + D}_{ij}(\mathtt{EXT\_WIDTH\_MIN\_ERR})$; +\item \texttt{EXT\_THETA\_ERR} associated to ${\cal + D}_{ij}(\mathtt{EXT\_THETA\_ERR})$; +\end{itemize} + +\subsection{Testing} +\label{sec-implementation-test} + +No test data have been provided. + +\commentFromSC{2012-09-18}{No data exists to test the software. It + would be good to make sure that the products are consistent at least + (1) From one software release to another; (2) From one software + execution to another; (3) To check if the product depends one the + order in which the input data are ingested and processed. Stress + tests could also be imagined, e.g. involving large data sets}{ } + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\newpage +\section{Obsolete Stuff?} + +TODO (SC): The following stuff seems obsolete. Tell me if I can remove it. + +\subsection{IPP to MOPS} + +\begin{table}[htbp] +\begin{center} +\small +\caption{IPP-MOPS FITS Table Description Files\label{table-types-ipp-mops}} +\begin{tabular}{lll} +\hline +\hline +{\bf Table Name} & {\bf EXTNAME} & {\bf Description} \\ +\hline +Transient Detections & \code{MOPS_TRANSIENT_DETECTIONS} & list of transient detections for an FPA \\ +Detectability Query Result & \code{MOPS_DETECTABILITY_RESPONSE} & table of detectability values in response to query\\ +\end{tabular} +\end{center} +\end{table} + +\subsection{MOPS to IPP} + +\begin{table}[htbp] +\begin{center} +\small +\caption{MOPS-IPP FITS Table Description Files\label{table-types-mops-ipp}} +\begin{tabular}{lll} +\hline +\hline +{\bf Table Name} & {\bf EXTNAME} & {\bf Description} \\ +\hline +Detectability Query & \code{MOPS_DETECTABILITY_QUERY} & table of detectability query tuples (RA, Dec, FPA) \\ +\end{tabular} +\end{center} +\end{table} + +\subsection{Limitations} +\label{sec-limitations} + +\begin{itemize} +\item The IPP is not yet calculating detection efficiencies (it is + still being developed). Further, it is not yet clear how to merge + the detection efficiency measurements for different skycells. Until + these issues are resolved, the detection efficiency values will be + fake (Expected fields: DE\_MAGnn (F32): Magnitude (calibrated) for + detection efficiency; DE\_EFFnn (F32): Detection efficiency (0..1)). +\item ANGLE, ANGLE\_SIG, LENGTH, LENGTH\_SIG removed; these can be + calculated from the moment; +\item FLUX, FLUX\_SIG renamed MAG, MAG\_ERR since IPP writes + magnitudes; +\item FLAGS added to allow additional weeding out of bad detections; +\item DIFF\_SKYFILE\_ID added to allow trace back to IPP diff skyfile, + for postage stamps; +\item STARPSF renamed EXT\_SIGNIFICANCE to be more clear +\end{itemize} + +\end{document} + Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/basics.xfig =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/basics.xfig (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/basics.xfig (revision 41112) @@ -0,0 +1,290 @@ +#FIG 3.2 Produced by xfig version 3.2.5b +Landscape +Center +Metric +A4 +100.00 +Single +-2 +1200 2 +0 32 #dc9d92 +0 33 #f0ebdf +0 34 #c2c2c2 +0 35 #e1c7a7 +0 36 #e0e0e0 +0 37 #d1d1d1 +0 38 #ececec +0 39 #d97a1a +0 40 #f0e31a +0 41 #877dc1 +0 42 #afa092 +0 43 #827cdc +0 44 #d5d5d5 +0 45 #8b8ba4 +0 46 #4a4a4a +0 47 #8b6b6b +0 48 #636363 +0 49 #8d8d8d +0 50 #b69a73 +0 51 #4192fe +0 52 #be703b +0 53 #da7700 +0 54 #d9b700 +0 55 #006400 +0 56 #5a6b3b +0 57 #d2d2d2 +0 58 #a9a9a9 +0 59 #8d8da3 +0 60 #f2b85d +0 61 #88986b +0 62 #646464 +0 63 #d5d5d5 +0 64 #8b8ba4 +0 65 #5a5a5a +0 66 #b6e5fe +0 67 #85bfeb +0 68 #bcbcbc +0 69 #d29552 +0 70 #8d8d8d +0 71 #97d1fd +0 72 #8d8d8d +0 73 #616161 +0 74 #adb1ad +0 75 #717171 +0 76 #fe9900 +0 77 #d5d5d5 +0 78 #8b8ba4 +0 79 #8b6b6b +0 80 #8b9b6b +0 81 #f66b00 +0 82 #5a6b39 +0 83 #8b9b6b +0 84 #d5d5d5 +0 85 #8b8ba4 +0 86 #8b6b6b +0 87 #8b9b6b +0 88 #f66b00 +0 89 #8b9b7b +0 90 #184a18 +0 91 #d5d5d5 +0 92 #acacac +0 93 #8b8ba4 +0 94 #f6bc5a +0 95 #8b9b6b +0 96 #636b9b +0 97 #8b6b6b +0 98 #f6f6f6 +0 99 #dd0000 +0 100 #8b9b6b +0 101 #d5d5d5 +0 102 #acacac +0 103 #8b8ba4 +0 104 #f6bc5a +0 105 #8b9b6b +0 106 #d5d5d5 +0 107 #acacac +0 108 #8b8ba4 +0 109 #f6bc5a +0 110 #8b9b6b +0 111 #636b9b +0 112 #526b29 +0 113 #939393 +0 114 #006300 +0 115 #8b8ba4 +0 116 #8b8ba4 +0 117 #8b8ba4 +0 118 #00634a +0 119 #7b834a +0 120 #e6bc7b +0 121 #8b9b7b +0 122 #a4b4c5 +0 123 #6b6b93 +0 124 #836b6b +0 125 #529b4a +0 126 #d5e6e6 +0 127 #526363 +0 128 #186b4a +0 129 #9ba4b4 +0 130 #fe9300 +0 131 #fe9300 +0 132 #8b6b6b +0 133 #d5d5d5 +0 134 #acacac +0 135 #8b8ba4 +0 136 #8b6b6b +0 137 #acacac +0 138 #8b9b6b +0 139 #acacac +0 140 #d2d2d2 +0 141 #a9a9a9 +0 142 #8d8da3 +0 143 #f2b85d +0 144 #88986b +0 145 #d5d5d5 +0 146 #8b8ba4 +0 147 #d5d5d5 +0 148 #8b8ba4 +0 149 #8b6b6b +0 150 #8b9b6b +0 151 #d5d5d5 +0 152 #8b8ba4 +0 153 #8b6b6b +0 154 #8b9b7b +0 155 #000000 +0 156 #f63829 +0 157 #000000 +0 158 #fefe52 +0 159 #52794a +0 160 #63995a +0 161 #c56142 +0 162 #e66942 +0 163 #fe7952 +0 164 #dddddd +0 165 #acacac +0 166 #8b8ba4 +0 167 #f6bc5a +0 168 #8b9b6b +0 169 #636b9b +0 170 #f6f6f6 +0 171 #d5d5d5 +0 172 #8b6b6b +0 173 #d2d2d2 +0 174 #a9a9a9 +0 175 #8d8da3 +0 176 #f2b85d +0 177 #88986b +0 178 #d2d2d2 +0 179 #a9a9a9 +0 180 #8d8da3 +0 181 #f2b85d +0 182 #88986b +0 183 #d2d2d2 +0 184 #a9a9a9 +0 185 #8d8da3 +0 186 #f2b85d +0 187 #88986b +0 188 #d2d2d2 +0 189 #a9a9a9 +0 190 #8d8da3 +0 191 #f2b85d +0 192 #88986b +0 193 #d2d2d2 +0 194 #a9a9a9 +0 195 #8d8da3 +0 196 #f2b85d +0 197 #88986b +0 198 #d5d5d5 +0 199 #8b8ba4 +0 200 #d2d2d2 +0 201 #a9a9a9 +0 202 #8d8da3 +0 203 #f2b85d +0 204 #88986b +0 205 #d2d2d2 +0 206 #a9a9a9 +0 207 #8d8da3 +0 208 #f2b85d +0 209 #88986b +0 210 #d2d2d2 +0 211 #a9a9a9 +0 212 #8d8da3 +0 213 #f2b85d +0 214 #88986b +0 215 #d2d2d2 +0 216 #a9a9a9 +0 217 #8d8da3 +0 218 #f2b85d +0 219 #88986b +0 220 #d5d5d5 +0 221 #8b8ba4 +0 222 #f2edd2 +0 223 #f4ad5d +0 224 #95cd98 +0 225 #a9a9a9 +0 226 #b4157d +0 227 #ededed +0 228 #838383 +0 229 #d5d5d5 +0 230 #acacac +0 231 #8b8ba4 +0 232 #f6bc5a +0 233 #8b9b6b +0 234 #636b9b +0 235 #7b7b7b +0 236 #005a00 +0 237 #e67373 +0 238 #f6f6f6 +0 239 #dd0000 +0 240 #feca31 +0 241 #29794a +0 242 #dd2821 +0 243 #2159c5 +0 244 #f7f7f7 +0 245 #ededed +0 246 #e5e5e5 +0 247 #7b834a +0 248 #d5d5d5 +0 249 #acacac +0 250 #63737b +0 251 #e6bc7b +0 252 #8b9b7b +0 253 #a4b4c5 +0 254 #6b6b93 +0 255 #836b6b +0 256 #529b4a +0 257 #d5e6e6 +0 258 #9ba4b4 +0 259 #21835a +0 260 #8b8ba4 +0 261 #f6bc5a +0 262 #8b9b6b +0 263 #636b9b +0 264 #d5d5d5 +0 265 #acacac +0 266 #8b8ba4 +0 267 #f6bc5a +0 268 #8b9b6b +0 269 #8b6b6b +0 270 #9b0000 +0 271 #9b0000 +0 272 #8b8b8b +0 273 #424242 +0 274 #8b8b8b +0 275 #424242 +0 276 #8b8b8b +0 277 #424242 +0 278 #8b8b8b +0 279 #424242 +0 280 #8b8b8b +0 281 #424242 +0 282 #8b8b8b +0 283 #424242 +0 284 #8b8b8b +0 285 #8b8b8b +0 286 #8b8b8b +0 287 #8b8b8b +0 288 #8b8b8b +0 289 #8b8b8b +0 290 #8b8b8b +0 291 #8b8b8b +0 292 #8b8b8b +0 293 #8b8b8b +0 294 #8b8b8b +0 295 #8b8b8b +0 296 #6e6e6e +0 297 #444444 +0 298 #8d8e8d +0 299 #8d8e8d +0 300 #adadad +0 301 #333333 +0 302 #939295 +0 303 #747075 +0 304 #555555 +0 305 #b2b2b2 +0 306 #c2c2c2 +0 307 #6d6d6d +0 308 #454545 +0 309 #8d8d8d +0 310 #8d8e8d +1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 6975 4950 5180 5180 6975 4950 7200 10125 +1 3 0 1 0 7 50 -1 -1 0.000 1 0.0000 6975 4950 450 450 6975 4950 6975 5400 Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/exposure_ota.fig =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/exposure_ota.fig (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/exposure_ota.fig (revision 41112) @@ -0,0 +1,12 @@ +#FIG 3.2 Produced by xfig version 3.2.5b +Landscape +Center +Metric +A4 +100.00 +Single +-2 +1200 2 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 2295 1035 6705 1035 6705 3150 2295 3150 2295 1035 +4 0 0 50 -1 0 12 0.0000 4 165 1170 3780 1980 exposure OTAs\001 Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/ext_theta.fig =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/ext_theta.fig (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/ext_theta.fig (revision 41112) @@ -0,0 +1,39 @@ +#FIG 3.2 Produced by xfig version 3.2.5b +Landscape +Center +Metric +A4 +100.00 +Single +-2 +1200 2 +5 1 0 2 0 7 50 -1 -1 0.000 0 1 1 0 5538.104 4894.526 5805 4320 5130 4410 4950 5130 + 3 1 1.00 60.00 120.00 +6 3870 2610 9900 5580 +1 1 0 2 0 7 50 -1 -1 0.000 1 0.4363 8135 3547 1206 424 8135 3547 9398 3395 +2 1 1 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 + 3 1 1.00 55.77 111.55 + 4126 5520 9592 2831 +2 1 1 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2 + 3 1 1.00 55.77 111.55 + 8474 4317 7794 2779 +-6 +2 1 0 2 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 3 1 1.00 60.00 120.00 + 3600 7200 10575 7200 +2 1 0 2 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 3 1 1.00 60.00 120.00 + 3 1 1.00 60.00 120.00 + 7740 5400 9855 4447 +2 1 0 2 0 7 50 -1 -1 0.000 0 0 -1 1 1 2 + 3 1 1.00 60.00 120.00 + 3 1 1.00 60.00 120.00 + 9850 3332 9470 2516 +2 1 0 2 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 3 1 1.00 60.00 120.00 + 5850 7650 5850 1350 +4 0 0 50 -1 0 12 0.0000 4 150 810 4725 4185 EXT_THETA\001 +4 0 0 50 -1 0 16 0.0000 4 135 90 10260 7650 W\001 +4 0 0 50 -1 0 12 0.0000 4 150 1170 8550 5490 EXT_WIDTH_MAJ\001 +4 0 0 50 -1 0 12 0.0000 4 150 1170 9675 2700 EXT_WIDTH_MIN\001 +4 0 0 50 -1 -1 16 0.0000 4 135 90 6030 1530 N\001 Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/ipp_workflow.fig =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/ipp_workflow.fig (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/figures/ipp_workflow.fig (revision 41112) @@ -0,0 +1,84 @@ +#FIG 3.2 Produced by xfig version 3.2.5b +Landscape +Center +Metric +A4 +100.00 +Single +-2 +1200 2 +6 2700 2700 4500 3600 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 2700 2700 4500 2700 4500 3600 2700 3600 2700 2700 +4 0 0 50 -1 0 12 0.0000 4 135 360 3420 3217 CHIP\001 +-6 +6 2700 450 4500 1350 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 900 900 450 3600 900 4500 1350 +4 0 0 50 -1 0 12 0.0000 4 165 900 3150 952 "exposure"\001 +-6 +6 2700 4050 4500 4950 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 2700 4050 4500 4050 4500 4950 2700 4950 2700 4050 +4 0 0 50 -1 0 12 0.0000 4 135 540 3330 4567 CAMERA\001 +-6 +6 2700 5400 4500 6300 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 2700 5400 4500 5400 4500 6300 2700 6300 2700 5400 +4 0 0 50 -1 0 12 0.0000 4 135 360 3420 5917 WARP\001 +-6 +6 2700 6750 4500 7650 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 2700 6750 4500 6750 4500 7650 2700 7650 2700 6750 +4 0 0 50 -1 0 12 0.0000 4 135 360 3420 7267 DIFF\001 +-6 +6 2700 8100 4500 9000 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 2700 8100 4500 8100 4500 9000 2700 9000 2700 8100 +4 0 0 50 -1 0 12 0.0000 4 135 540 3330 8617 PPMOPS\001 +-6 +6 5400 6525 7200 7425 +2 2 0 1 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 5400 6525 7200 6525 7200 7425 5400 7425 5400 6525 +4 0 0 50 -1 0 12 0.0000 4 135 450 6075 7042 STACK\001 +-6 +6 2700 10350 4500 11250 +1 1 0 1 0 7 50 -1 -1 0.000 1 0.0000 3600 10800 900 450 3600 10800 4500 11250 +4 0 0 50 -1 0 12 0.0000 4 165 1170 3015 10852 MOPS products\001 +-6 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 1350 3600 2700 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 3600 3600 4050 +2 1 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 4 + 1800 0 1800 1800 7650 1800 7650 0 +2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 7650 2250 6750 2250 6750 2700 7650 2700 7650 2250 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 4950 3600 5400 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 6300 3600 6750 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 7650 3600 8100 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 6300 6300 6525 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 6300 7425 3600 8100 +2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5 + 1800 2250 7650 2250 7650 9450 1800 9450 1800 2250 +2 1 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 4 + 1800 11700 1800 9900 7650 9900 7650 11700 +2 1 0 1 0 7 50 -1 -1 0.000 0 0 -1 1 0 2 + 1 1 1.00 60.00 120.00 + 3600 9000 3600 10350 +4 0 0 50 -1 0 12 0.0000 4 135 1620 5400 675 Interface with the\001 +4 0 0 50 -1 0 12 0.0000 4 165 810 5400 870 telescope\001 +4 0 0 50 -1 0 12 0.0000 4 135 270 7065 2542 IPP\001 +4 0 0 50 -1 0 12 0.0000 4 135 1620 5400 10575 Interface with the\001 +4 0 0 50 -1 0 12 0.0000 4 135 360 5400 10770 MOPS\001 Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/icd.bib =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/icd.bib (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/documentation/ICD/icd.bib (revision 41112) @@ -0,0 +1,19 @@ +@ARTICLE{VerJedDen2012IAAaPwTF, + author = {{Vere{\v s}}, P. and {Jedicke}, R. and {Denneau}, L. and + {Wainscoat}, R. and {Holman}, M.~J. and {Lin}, + H.-W.}, + title = "{Improved Asteroid Astrometry and Photometry with Trail + Fitting}", + journal = {PASP}, +archivePrefix = "arXiv", + eprint = {1209.6106}, + primaryClass = "astro-ph.IM", + keywords = {Data Analysis and Techniques}, + year = 2012, + month = nov, + volume = 124, + pages = {1197-1207}, + doi = {10.1086/668616}, + adsurl = {http://adsabs.harvard.edu/abs/2012PASP..124.1197V}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/m4/check_zlib.m4 =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/m4/check_zlib.m4 (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/m4/check_zlib.m4 (revision 41112) @@ -0,0 +1,87 @@ +dnl @synopsis CHECK_ZLIB() +dnl +dnl This macro searches for an installed zlib library. If nothing was +dnl specified when calling configure, it searches first in /usr/local +dnl and then in /usr. If the --with-zlib=DIR is specified, it will try +dnl to find it in DIR/include/zlib.h and DIR/lib/libz.a. If +dnl --without-zlib is specified, the library is not searched at all. +dnl +dnl If either the header file (zlib.h) or the library (libz) is not +dnl found, the configuration exits on error, asking for a valid zlib +dnl installation directory or --without-zlib. +dnl +dnl The macro defines the symbol HAVE_LIBZ if the library is found. You +dnl should use autoheader to include a definition for this symbol in a +dnl config.h file. Sample usage in a C/C++ source is as follows: +dnl +dnl #ifdef HAVE_LIBZ +dnl #include +dnl #endif /* HAVE_LIBZ */ +dnl +dnl @category InstalledPackages +dnl @author Loic Dachary +dnl @version 2004-09-20 +dnl @license GPLWithACException + +AC_DEFUN([CHECK_ZLIB], +# +# Handle user hints +# +[AC_MSG_CHECKING(if zlib is wanted) +AC_ARG_WITH(zlib, +[ --with-zlib=DIR root directory path of zlib installation [defaults to + /usr/local or /usr if not found in /usr/local] + --without-zlib to disable zlib usage completely], +[if test "$withval" != no ; then + AC_MSG_RESULT(yes) + if test -d "$withval" + then + ZLIB_HOME="$withval" + else + AC_MSG_WARN([Sorry, $withval does not exist, checking usual places]) + fi +else + AC_MSG_RESULT(no) +fi]) + +ZLIB_HOME=/usr/local +if test ! -f "${ZLIB_HOME}/include/zlib.h" +then + ZLIB_HOME=/usr +fi + +# +# Locate zlib, if wanted +# +if test -n "${ZLIB_HOME}" +then + ZLIB_OLD_LDFLAGS=$LDFLAGS + ZLIB_OLD_CPPFLAGS=$LDFLAGS + LDFLAGS="$LDFLAGS -L${ZLIB_HOME}/lib" + CPPFLAGS="$CPPFLAGS -I${ZLIB_HOME}/include" + AC_LANG_SAVE + AC_LANG_C + AC_CHECK_LIB(z, inflateEnd, [zlib_cv_libz=yes], [zlib_cv_libz=no]) + AC_CHECK_HEADER(zlib.h, [zlib_cv_zlib_h=yes], [zlib_cv_zlib_h=no]) + AC_LANG_RESTORE + if test "$zlib_cv_libz" = "yes" -a "$zlib_cv_zlib_h" = "yes" + then + # + # If both library and header were found, use them + # + AC_CHECK_LIB(z, inflateEnd) + AC_MSG_CHECKING(zlib in ${ZLIB_HOME}) + AC_MSG_RESULT(ok) + else + # + # If either header or library was not found, revert and bomb + # + AC_MSG_CHECKING(zlib in ${ZLIB_HOME}) + LDFLAGS="$ZLIB_OLD_LDFLAGS" + CPPFLAGS="$ZLIB_OLD_CPPFLAGS" + AC_MSG_RESULT(failed) + AC_MSG_ERROR(either specify a valid zlib installation with --with-zlib=DIR or disable zlib usage with --without-zlib) + fi +fi + +]) Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/Makefile.am =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/Makefile.am (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/Makefile.am (revision 41112) @@ -0,0 +1,42 @@ +bin_PROGRAMS = ppMops ppMonet + +# Force recompilation of ppMopsVersion.c, since it gets the version information +ppTranslateVersion.c: ppTranslateVersionDefinitions.h +ppTranslateVersionDefinitions.h: ppTranslateVersionDefinitions.h.in FORCE + pslib-setsvnversion.pl PPTRANSLATE ppTranslateVersionDefinitions.h.in ppTranslateVersionDefinitions.h +FORCE: ; + +ppMops_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(PPMOPS_CFLAGS) +ppMops_LDFLAGS = $(PSLIB_LIBS) $(PPMOPS_LIBS) + +ppMops_SOURCES = \ + ppMops.c \ + ppTranslateVersion.c \ + ppMopsArguments.c \ + ppMopsDetections.c \ + ppMopsRead.c \ + ppMopsWrite.c \ + ppMopsMerge.c \ + ppMopsGetSkyChipPsfVersion.c + +ppMonet_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(PPMONET_CFLAGS) +ppMonet_LDFLAGS = $(PSLIB_LIBS) $(PPMONET_LIBS) + +ppMonet_SOURCES = \ + ppMonet.c \ + ppTranslateVersion.c \ + ppMonetArguments.c + + +noinst_HEADERS = \ + ppTranslateVersion.h \ + ppMops.h \ + ppMonet.h + + +clean-local: + -rm -f TAGS + +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonet.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonet.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonet.c (revision 41112) @@ -0,0 +1,165 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppTranslateVersion.h" +#include "ppMonet.h" + +int main(int argc, char *argv[]) +{ + psLibInit(NULL); + + ppMonetArguments *args = ppMonetArgumentsParse(argc, argv); // Parsed arguments + if (!args) { + psErrorStackPrint(stderr, "Error parsing arguments"); + exit(PS_EXIT_CONFIG_ERROR); + } + + psTrace("ppMonet.read", 1, "Reading input detections\n"); + + psFits *fits = psFitsOpen(args->input, "r"); // FITS file + if (!fits) { + psError(PS_ERR_IO, false, "Unable to open input %s", args->input); + exit(PS_EXIT_SYS_ERROR); + } + + // Set up output file + FILE *file = fopen(args->output, "w"); // File handle + if (!file) { + psErrorStackPrint(stderr, "Error opening file %s", args->output); + exit(PS_EXIT_SYS_ERROR); + } + gzFile gz = gzdopen(fileno(file), "w"); // Gzip file handle + gzprintf(gz, "# Pan-STARRS IPP detections for Dave Monet\n"); + gzprintf(gz, "#\n"); + gzprintf(gz, "# Translated from %s\n", args->input); + psString source = ppTranslateSource(), version = ppTranslateVersion(); + gzprintf(gz, "# S/W source = %s\n", source); + gzprintf(gz, "# S/W version = %s\n", version); + gzprintf(gz, "#\n"); + psFree(source); + psFree(version); + gzprintf(gz, "# exp_name = %s\n", args->exp_name); + gzprintf(gz, "# exp_id = %" PRId64 "\n", args->exp_id); + gzprintf(gz, "# chip_id = %" PRId64 "\n", args->chip_id); + gzprintf(gz, "# cam_id = %" PRId64 "\n", args->cam_id); + gzprintf(gz, "# ZP = %f\n", args->zp); + gzprintf(gz, "# ZP error = %f\n", args->zpErr); + gzprintf(gz, "# Astrometry rms = %f\n", args->rmsAstrom); + + psFree(args); + + psMetadata *header = psFitsReadHeader(NULL, fits); // Primary header + if (!header) { + psError(PS_ERR_IO, false, "Unable to read header"); + exit(PS_EXIT_SYS_ERROR); + } + + double raBoresight = psMetadataLookupF64(NULL, header, "RA"); + double decBoresight = psMetadataLookupF64(NULL, header, "DEC"); + psString filter = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FILTERID")); + float airmass = psMetadataLookupF32(NULL, header, "AIRMASS"); + float exptime = psMetadataLookupF32(NULL, header, "EXPTIME"); + double posangle = RAD_TO_DEG(psMetadataLookupF64(NULL, header, "POSANGLE")); + double rotangle = psMetadataLookupF64(NULL, header, "ROTANGLE"); + double alt = psMetadataLookupF64(NULL, header, "ALT"); + double az = psMetadataLookupF64(NULL, header, "AZ"); + double mjd = psMetadataLookupF64(NULL, header, "MJD-OBS"); + + gzprintf(gz, "# RA = %.8lf\n", raBoresight); + gzprintf(gz, "# Dec = %.8lf\n", decBoresight); + gzprintf(gz, "# Filter = %s\n", filter); + gzprintf(gz, "# Airmass = %f\n", airmass); + gzprintf(gz, "# ExpTime = %f\n", exptime); + gzprintf(gz, "# PA = %lf\n", posangle); + gzprintf(gz, "# Rotator = %lf\n", rotangle); + gzprintf(gz, "# Alt = %.8lf\n", alt); + gzprintf(gz, "# Az = %.8lf\n", az); + gzprintf(gz, "# MJD = %.6lf\n", mjd); + gzprintf(gz, "#\n"); + gzprintf(gz, "# chip , x , y , RA , Dec , Mag , MagErr , Width , Flags\n"); + + psFree(header); + + int numHDU = psFitsGetSize(fits); // Number of HDUs + for (int i = 1; i < numHDU; i++) { + if (!psFitsMoveExtNum(fits, i, false)) { + psError(PS_ERR_IO, false, "Unable to move to HDU %d", i); + exit(PS_EXIT_SYS_ERROR); + } + + if (psFitsGetExtType(fits) != PS_FITS_TYPE_BINARY_TABLE) { + psTrace("ppMonet", 1, "Skipping extension %d: not a binary table", i); + continue; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); // Primary header + if (!header) { + psError(PS_ERR_IO, false, "Unable to read header %d\n", i); + exit(PS_EXIT_SYS_ERROR); + return false; + } + + const char *extname = psMetadataLookupStr(NULL, header, "EXTNAME"); + const char *exttype = extname + strlen(extname) - strlen(EXT_TYPE); + if (strcmp(exttype, EXT_TYPE) != 0) { + psTrace("ppMonet", 1, "Skipping extension %d (%s): not correct type", i, extname); + psFree(header); + continue; + } + + long size = psFitsTableSize(fits); // Size of table + if (size <= 0) { + psErrorStackPrint(stderr, "Unable to determine size of table %d", i); + psFree(header); + exit(PS_EXIT_SYS_ERROR); + } + + psString chipName = psStringNCopy(extname, strlen(extname) - strlen(EXT_TYPE)); // Name of chip + psFree(header); + + psTrace("ppMonet.read", 3, "Reading %ld rows from %s\n", size, extname); + psArray *table = psFitsReadTable(fits); // Table of interest + if (!table) { + psError(PS_ERR_IO, false, "Unable to read table %d", i); + psFree(chipName); + exit(PS_EXIT_SYS_ERROR); + } + + psTrace("ppMonet.read", 3, "Writing %ld rows for %s\n", size, chipName); + for (long j = 0; j < size; j++) { + psMetadata *row = table->data[j]; // Row of interest + + float x = psMetadataLookupF32(NULL, row, "X_PSF"); + float y = psMetadataLookupF32(NULL, row, "Y_PSF"); + double ra = psMetadataLookupF64(NULL, row, "RA_PSF"); + double dec = psMetadataLookupF64(NULL, row, "DEC_PSF"); + float mag = psMetadataLookupF32(NULL, row, "PSF_INST_MAG"); + float magErr = psMetadataLookupF32(NULL, row, "PSF_INST_MAG_SIG"); + float major = psMetadataLookupF32(NULL, row, "PSF_MAJOR"); + float minor = psMetadataLookupF32(NULL, row, "PSF_MINOR"); + psU32 flags = psMetadataLookupU32(NULL, row, "FLAGS"); + + float width = (float) (0.5 * (major + minor)); + + gzprintf(gz, "%s , %.2f , %.2f , %.8lf , %.8lf , %.3f , %.3f , %.2f , %#08x\n", + chipName, x, y, ra, dec, mag, magErr, width, flags); + } + psFree(table); + psFree(chipName); + } + gzclose(gz); + psFitsClose(fits); + psFree(filter); + + psTrace("ppMonet.read", 1, "Done reading input detections\n"); + + psLibFinalize(); + + return PS_EXIT_SUCCESS; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonet.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonet.h (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonet.h (revision 41112) @@ -0,0 +1,23 @@ +#ifndef PP_MOPS_H +#define PP_MOPS_H + +#include + +#define EXT_TYPE ".psf" // Extension type to consider + +// Configuration data +typedef struct { + psString input; // Input filenames + psString exp_name; // Exposure name + psS64 exp_id; // Exposure identifier + psS64 chip_id; // Chip stage identifier + psS64 cam_id; // Camera stage identifier + float zp, zpErr; // Magnitude zero point and error + float rmsAstrom; // Astrometric solution RMS + psString output; // Output filename +} ppMonetArguments; + +/// Parse arguments +ppMonetArguments *ppMonetArgumentsParse(int argc, char *argv[]); + +#endif Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonetArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonetArguments.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonetArguments.c (revision 41112) @@ -0,0 +1,88 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMonet.h" + +// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments // Command-line arguments + ) +{ + fprintf(stderr, "\nPan-STARRS IPP-Monet detection translator\n\n"); + fprintf(stderr, "Usage: %s INPUT_NAME OUTPUT_NAME\n", program); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); + psLibFinalize(); + exit(PS_EXIT_CONFIG_ERROR); +} + +static void monetArgumentsFree(ppMonetArguments *args) +{ + psFree(args->input); + psFree(args->exp_name); + psFree(args->output); + return; +} + +ppMonetArguments *ppMonetArgumentsAlloc(void) +{ + ppMonetArguments *args = psAlloc(sizeof(ppMonetArguments)); // Data to return + psMemSetDeallocator(args, (psFreeFunc)monetArgumentsFree); + + args->input = NULL; + args->exp_name = NULL; + args->exp_id = 0; + args->chip_id = 0; + args->cam_id = 0; + args->zp = NAN; + args->zpErr = NAN; + args->rmsAstrom = NAN; + args->output = NULL; + + return args; +} + + +ppMonetArguments *ppMonetArgumentsParse(int argc, char *argv[]) +{ + assert(argv); + + psTrace("ppMonet.args", 1, "Parsing command-line arguments\n"); + + psArgumentVerbosity(&argc, argv); + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-exp_name", 0, "Exposure name", NULL); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-exp_id", 0, "Exposure identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-chip_id", 0, "Chip stage identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-cam_id", 0, "Camera stage identifier", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Magnitude zero point", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp_error", 0, "Error in magnitude zero point", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-astrom_rms", 0, "Astrometric solution RMS", NAN); + + if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 3) { + usage(argv[0], arguments); + } + + ppMonetArguments *args = ppMonetArgumentsAlloc(); // Arguments, to return + + args->input = psStringCopy(argv[1]); + args->output = psStringCopy(argv[2]); + + args->exp_name = psMetadataLookupStr(NULL, arguments, "-exp_name"); + args->exp_id = psMetadataLookupS64(NULL, arguments, "-exp_id"); + args->chip_id = psMetadataLookupS64(NULL, arguments, "-chip_id"); + args->cam_id = psMetadataLookupS64(NULL, arguments, "-cam_id"); + args->zp = psMetadataLookupF32(NULL, arguments, "-zp"); + args->zpErr = psMetadataLookupF32(NULL, arguments, "-zp_error"); + args->rmsAstrom = psMetadataLookupF32(NULL, arguments, "-astrom_rms"); + + psTrace("ppMonet.args", 1, "Done parsing command-line arguments\n"); + + return args; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonetMerge.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonetMerge.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMonetMerge.c (revision 41112) @@ -0,0 +1,171 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMonet.h" + +#define LEAF_SIZE 4 // Size of leaf +#define MATCH_RADIUS SEC_TO_RAD(1.0) // Matching radius +#define MJD_TOL 1.0/3600.0/24.0 // Tolerance for MJD matching +#define BORESIGHT_TOL SEC_TO_RAD(1.0) // Tolerance for boresight matching +#define EXPTIME_TOL 1.0e-3 // Tolerance for exposure time matching +#define POSANGLE_TOL SEC_TO_RAD(1.0) // Tolerance for position angle matching +#define AIRMASS_TOL 1.0e-3 // Tolerance for airmass matching + +// Get distance from detection to centre of image +static float mergeDistance(const ppMonetDetections *detections, // Detections of interest + long index // Index for source of interest + ) +{ + float dx = detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0; + float dy = detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0; + return PS_SQR(dx) + PS_SQR(dy); +} + + +ppMonetDetections *ppMonetMerge(const psArray *detections) +{ + PS_ASSERT_ARRAY_NON_NULL(detections, NULL); + + psTrace("ppMonet.merge", 1, "Merging detections from %ld inputs\n", detections->n); + + ppMonetDetections *merged = NULL; // Merged list + int num = 1; // Number of merged files + for (int i = 0; i < detections->n; i++) { + ppMonetDetections *det = detections->data[i]; // Detections of interest + if (!det) { + psTrace("ppMonet.merge", 3, "Ignoring NULL input %d\n", i); + continue; + } else if (det->num == 0) { + psTrace("ppMonet.merge", 3, "Ignoring empty input %d\n", i); + continue; + } + num++; + if (!merged) { + psTrace("ppMonet.merge", 3, "Accepting %ld detections from input %d\n", det->num, i); + merged = psMemIncrRefCounter(det); + continue; + } + psTrace("ppMonet.merge", 3, "Merging %ld detections from input %d\n", det->num, i); + + // XXX compare exposure properties + if (strcmp(merged->raBoresight, det->raBoresight) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure RA values differ: %s vs %s", + merged->raBoresight, det->raBoresight); + return NULL; + } + if (strcmp(merged->decBoresight, det->decBoresight) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure Dec values differ: %s vs %s", + merged->decBoresight, det->decBoresight); + return NULL; + } + if (strcmp(merged->filter, det->filter) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure filter values differ: %s vs %s", + merged->filter, det->filter); + return NULL; + } + + if (fabsf(merged->airmass - det->airmass) > AIRMASS_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure airmass values differ: %f vs %f", + merged->airmass, det->airmass); + return NULL; + } + if (fabsf(merged->exptime - det->exptime) > EXPTIME_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure exposure time values differ: %f vs %f", + merged->exptime, det->exptime); + return NULL; + } + if (fabs(merged->posangle - det->posangle) > POSANGLE_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure position angle values differ: %f vs %f", + merged->posangle, det->posangle); + return NULL; + } + if (fabs(merged->alt - det->alt) > BORESIGHT_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure altitude values differ: %lf vs %lf", + merged->alt, det->alt); + return NULL; + } + if (fabs(merged->az - det->az) > BORESIGHT_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure azimuth values differ: %lf vs %lf", + merged->az, det->az); + return NULL; + } + if (fabs(merged->mjd - det->mjd) > MJD_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure MJD values differ: %lf vs %lf", + merged->mjd, det->mjd); + return NULL; + } + + merged->seeing += det->seeing; // Taking average + + psTree *tree = psTreePlant(2, LEAF_SIZE, PS_TREE_SPHERICAL, merged->ra, merged->dec); // kd tree + if (!tree) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kd tree"); + psFree(merged); + return NULL; + } + + psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of interest + for (int j = 0; j < det->num; j++) { + coords->data.F64[0] = det->ra->data.F64[j]; + coords->data.F64[1] = det->dec->data.F64[j]; + psVector *indices = psTreeAllWithin(tree, coords, MATCH_RADIUS); // Indices for matching sources + if (!indices) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to search for matches"); + psFree(coords); + psFree(tree); + psFree(merged); + return NULL; + } + if (indices->n == 0) { + psTrace("ppMonet.merge", 9, "No matches for source %d in input %d\n", j, i); + psFree(indices); + ppMonetDetectionsCopySingle(merged, det, j); + continue; + } + psTrace("ppMonet.merge", 5, "%ld matches for source %d from input %d\n", indices->n, j, i); + + // Which one do we keep? + float bestDistance = INFINITY; // Best distance to centre + long bestIndex = -1; // Index with best distance + for (int k = 0; k < indices->n; k++) { + long index = indices->data.S64[k]; // Index of point + float distance = mergeDistance(merged, index); // Distance to centre of image + if (distance < bestDistance) { + bestDistance = distance; + bestIndex = index; + } + } + + float distance = mergeDistance(det, j); // Distance to centre of image + if (distance < bestDistance) { + psTrace("ppMonet.merge", 6, "New source clobbers old sources\n"); + // Blow away existing sources + for (int k = 0; k < indices->n; k++) { + long index = indices->data.S64[k]; // Index of point + merged->mask->data.U8[index] = 0xFF; + } + ppMonetDetectionsCopySingle(merged, det, j); + } else { + psTrace("ppMonet.merge", 6, "Old sources clobber new source\n"); + } + psFree(indices); + } + + psTrace("ppMonet.merge", 3, "Done merging input %d, %ld merged sources\n", i, merged->num); + + psFree(tree); + ppMonetDetectionsPurge(merged); + } + + psTrace("ppMonet.merge", 2, "%ld sources in merged detections list\n", merged->num); + + merged->seeing /= num; + + return merged; +} + Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMops.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMops.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMops.c (revision 41112) @@ -0,0 +1,99 @@ +#include +#include + +#include "ppMops.h" + +void test() +{ + psMetadata *md = psMetadataAlloc(); + + psVector *vec = psVectorAlloc(42, PS_TYPE_S32); + + psMetadataAddVector(md, PS_LIST_TAIL, "TEST", 0, NULL, vec); + + psFree(vec); + psFree(md); + + psLibFinalize(); + + fprintf (stderr, "found %d leaks at %s\n", + psMemCheckLeaks2 (0, + NULL, stdout, false, 500), "ppMops"); + + exit(0); +} + +/* + Behavior: + + If the CMF input files have different versions, merging cannot be + performed. + + If -version option is not given: + the output version is the version of the input file(s) + otherwise + the output version is (possibly forced to) the version option + + If the input file(s) version is equals to the version option: + no change in version (neither data creation nor data loss) + If the input file(s) version is strictly less than the version option: + Data for version option are set tp default values: 0, NaN, NULL + If the input file(s) version is strictly greater than the version option: + Data are those of the lower version + + Example: + -> ppMops dv1_input_files_list output -version 1 + is the same as 'ppMops dv1_input_files_list output' + -> ppMops dv1_input_files_list output -version 1 + is the same as 'ppMops dv1_input_files_list output' + -> ppMops dv1_input_files_list output -version 2 + Aggregate DV1 values and add default DV2 values + Saved as DV2 file + -> ppMops dv2_input_files_list output -version 1 + Aggregate DV1 values and truncate DV2 values + Saved as DV1 file + */ + +int main(int argc, char *argv[]) +{ + psLibInit(NULL); + + // test(); + + ppMopsArguments *args = ppMopsArgumentsParse(argc, argv); // Parsed arguments + if (!args) { + psErrorStackPrint(stderr, "Error parsing arguments"); + exit(PS_EXIT_CONFIG_ERROR); + } + + psArray *detections = ppMopsRead(args); // Detections from each input + if (!detections) { + psErrorStackPrint(stderr, "Unable to read detections"); + exit(PS_EXIT_SYS_ERROR); + } + + if (!ppMopsPurgeDuplicates(detections)) { + psErrorStackPrint(stderr, "Unable to merge detections"); + exit(PS_EXIT_SYS_ERROR); + } + + if (!ppMopsWrite(detections, args)) { + psErrorStackPrint(stderr, "Unable to write detections"); + exit(PS_EXIT_SYS_ERROR); + } + + for (int i = 0; i < detections->n; i++) { + psFree(detections->data[i]); + } + psFree(detections); + psFree(args); + + psLibFinalize(); + +/* fprintf (stderr, "found %d leaks at %s\n", */ +/* psMemCheckLeaks2 (0, */ +/* NULL, stdout, false, 500), "ppMops"); */ + + return PS_EXIT_SUCCESS; +} + Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMops.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMops.h (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMops.h (revision 41112) @@ -0,0 +1,96 @@ +#ifndef PP_MOPS_H +#define PP_MOPS_H + +#include +#include + +#define IN_EXTNAME "SkyChip.psf" // Extension name for data in input +// #define OBSERVATORY_CODE "F51" // IAU Observatory Code +#define OUT_EXTNAME "MOPS_TRANSIENT_DETECTIONS" // Extension name for data in output +#define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_SATURATED | \ + PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_SKY_FAILURE | PM_SOURCE_MODE_DEFECT) // Flags to exclude +//#define SOURCE_MASK2 (PM_SOURCE_MODE2_DIFF_WITH_DOUBLE | PM_SOURCE_MODE2_ON_SPIKE | PM_SOURCE_MODE2_ON_STARCORE | PM_SOURCE_MODE2_ON_BURNTOOL | PM_SOURCE_MODE2_ON_CONVPOOR) // Flags2 to exclude +#define SOURCE_MASK2 (PM_SOURCE_MODE2_DEFAULT) //Flags2 to exclude + +#define PS1_DV_FORMAT "PS1_DV%d" + +// Configuration data +typedef struct { + psArray *input; // Input filenames + psString exp_name; // Exposure name + psS64 exp_id; // Exposure identifier + psS64 chip_id; // Chip stage identifier + psS64 cam_id; // Camera stage identifier + psS64 fake_id; // Fake stage identifier + psS64 warp_id; // Warp stage identifier + psS64 diff_id; // Diff stage identifier + psString camera; // Camera name + psString obscode; // observatory code (derived from camera) + bool positive; // Sense of subtraction, T=positive, F=negative + float zp, zpErr; // Magnitude zero point and error + float rmsAstrom; // Astrometric solution RMS + psString output; // Output filename + psU16 version; // Version (for parameters) + psString comment; // Comment associated with the first argument of the diff + psString obsMode; // Observation mode + psString difftype; // WW (Warp-Warp Diff) or WS (Warp-Stack Diff) or SW (Stack-Warp Diff) + float sky; // Exposure avg sky background + psString shutoutc; // Camera exposure shutter open (UTC) from DB +} ppMopsArguments; + +/// Parse arguments +ppMopsArguments *ppMopsArgumentsParse(int argc, char *argv[]); + +typedef struct { + psString component; // skycell_id for these detections + psString raBoresight, decBoresight; // RA,Dec of telescope boresight + psString filter; // Filter for exposure + float airmass; // Airmass of exposure + float exptime; // Exposure time + double posangle; // Position angle + double alt, az; // Telescope altitude and azimuth + double mjd; // Modified Julian Date + float seeing; // Seeing of exposure + int naxis1, naxis2; // size of the image + long num; // Number of detections + long numGood; // Number of "good" detections + psS64 diffSkyfileId; // unique id for input skyfile + psMetadata *table; // Columns from the input file (SkyChip.psf extension) + psVector *x, *y; // Image coordinates + psVector *ra, *dec; // Sky coordinates + psVector *raErr, *decErr; // Error in sky coordinates + psVector *raExt, *decExt; // Sky coordinates for RA_EXT and DEC_EXT + psVector *raExtErr, *decExtErr; // Error in sky coordinates for RA_EXT and DEC_EXT + psVector *mask; // Mask for detections + float platescale; // Plate scale at centroid + psString fpashutoutc; // FPA shutoutc + psString fpashutcutc; // FPA shutcutc + psString refcat; // Reference catalog used +} ppMopsDetections; + +ppMopsDetections *ppMopsDetectionsAlloc(void); + +/// Copy a detection +bool ppMopsDetectionsCopySingle(ppMopsDetections *target, const ppMopsDetections *source, long index); + +/// Purge the detections list of masked detections +bool ppMopsDetectionsPurge(ppMopsDetections *detections); + +/// Read detections +psArray *ppMopsRead(ppMopsArguments *args); + +/// Merge detections +// ppMopsDetections *ppMopsMerge(const psArray *detections); +bool ppMopsPurgeDuplicates(const psArray *detections); + +/// Write detections +bool ppMopsWrite(const psArray *detections, const ppMopsArguments *args); + +/// Get the version contained in EXTTYPE of the "SkyChip.psf" table: +/// @returns 1 if EXTTYPE of "SkyChip.psf" is PS1_DV1 +/// @returns 2 if EXTTYPE of "SkyChip.psf" is PS1_DV2 +/// @returns 3 if EXTTYPE of "SkyChip.psf" is PS1_DV3 +/// @returns 0 otherwise +int ppMopsGetSkyChipPsfVersion(const psFits* fits); + +#endif Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsArguments.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsArguments.c (revision 41112) @@ -0,0 +1,164 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMops.h" + +// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments // Command-line arguments + ) +{ + fprintf(stderr, "\nPan-STARRS IPP-MOPS detection translator\n\n"); + fprintf(stderr, "Usage: %s INPUT_LIST OUTPUT_NAME\n", program); + fprintf(stderr, "\n"); + fprintf(stderr, "Description: Merge detections from INPUT_LIST into the single OUTPUT_NAME file\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "\tIf the CMF input files have different versions, \n\ +\tmerging cannot be performed.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "\tIf -version option is not given, the output version is\n\ +\tthe version of the input file(s) otherwise the output version is (possibly forced to)\n\ +\tthe version option.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "\tIf the input file(s) version is equals to the version option:\n\ +\t No change in version (neither data creation nor data loss)\n\ +\tIf the input file(s) version is strictly less than the version option:\n\ +\t Data for version option are set to default values: 0, NaN, NULL\n\ +\tIf the input file(s) version is strictly greater than the version option:\n\ +\t Data are those of the lower version.\n"); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); + fprintf(stderr, "\t\tCMF file version can be set to either 1 for PS1_DV1, 2 for PS1_DV2, or 3 for PS1_DV3\n"); + fprintf(stderr, "\t\tSee IPP-MOPS ICD for details\n"); + psLibFinalize(); + exit(PS_EXIT_CONFIG_ERROR); +} + +static void mopsArgumentsFree(ppMopsArguments *args) +{ + psFree(args->camera); + psFree(args->obscode); + psFree(args->input); + psFree(args->exp_name); + psFree(args->comment); + psFree(args->obsMode); + psFree(args->difftype); + psFree(args->shutoutc); + return; +} + +ppMopsArguments *ppMopsArgumentsAlloc(void) +{ + ppMopsArguments *args = psAlloc(sizeof(ppMopsArguments)); // Data to return + psMemSetDeallocator(args, (psFreeFunc)mopsArgumentsFree); + args->input = NULL; + args->exp_name = NULL; + args->exp_id = 0; + args->chip_id = 0; + args->cam_id = 0; + args->fake_id = 0; + args->warp_id = 0; + args->diff_id = 0; + args->camera = NULL; + args->obscode = NULL; + args->zp = NAN; + args->positive = true; + args->zpErr = NAN; + args->rmsAstrom = NAN; + args->output = NULL; + args->version = 1; + args->comment = NULL; + args->obsMode = NULL; + args->difftype = NULL; + args->sky = NAN; + args->shutoutc = NULL; + return args; +} + + +ppMopsArguments *ppMopsArgumentsParse(int argc, char *argv[]) +{ + assert(argv); + + psTrace("ppMops.args", 1, "Parsing command-line arguments\n"); + + psArgumentVerbosity(&argc, argv); + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-exp_name", 0, "Exposure name", NULL); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-exp_id", 0, "Exposure identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-chip_id", 0, "Chip stage identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-cam_id", 0, "Camera stage identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-fake_id", 0, "Fake stage identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-warp_id", 0, "Warp stage identifier", 0); + psMetadataAddS64(arguments, PS_LIST_TAIL, "-diff_id", 0, "Diff stage identifier", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-camera", 0, "Camera name", "No comment provided"); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Inverse subtraction?", false); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Magnitude zero point", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp_error", 0, "Error in magnitude zero point", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-astrom_rms", 0, "Astrometric solution RMS", NAN); + psMetadataAddU16(arguments, PS_LIST_TAIL, "-version", 0, "CMF file version", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-comment", 0, "Exposure comment", "No comment provided"); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-obsmode", 0, "Observation mode", "No obsmode provided"); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-difftype", 0, "Either WW (Warp-Warp Diff) or WS (Warp-Stack Diff)", "No difftype provided"); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-sky", 0, "Exposure avg sky background", NAN); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-shutoutc", 0, "Camera exposure shutter open (UTC)", "No shutoutc provided"); + if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 3) { + usage(argv[0], arguments); + } + + ppMopsArguments *args = ppMopsArgumentsAlloc(); // Arguments, to return + + psString inList = psSlurpFilename(argv[1]); // List of filenames + args->input = psStringSplitArray(inList, "\n", false); + psFree(inList); + if (!args->input || args->input->n == 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No inputs provided."); + return NULL; + } + args->output = psStringCopy(argv[2]); + + args->exp_name = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-exp_name")); + args->exp_id = psMetadataLookupS64(NULL, arguments, "-exp_id"); + args->chip_id = psMetadataLookupS64(NULL, arguments, "-chip_id"); + args->cam_id = psMetadataLookupS64(NULL, arguments, "-cam_id"); + args->fake_id = psMetadataLookupS64(NULL, arguments, "-fake_id"); + args->warp_id = psMetadataLookupS64(NULL, arguments, "-warp_id"); + args->diff_id = psMetadataLookupS64(NULL, arguments, "-diff_id"); + args->camera = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-camera")); + args->positive = !psMetadataLookupBool(NULL, arguments, "-inverse"); // NOTE: negated + args->zp = psMetadataLookupF32(NULL, arguments, "-zp"); + args->zpErr = psMetadataLookupF32(NULL, arguments, "-zp_error"); + args->rmsAstrom = psMetadataLookupF32(NULL, arguments, "-astrom_rms"); + args->version = psMetadataLookupU16(NULL, arguments, "-version"); + args->comment = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-comment")); + args->obsMode = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-obsmode")); + args->difftype = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-difftype")); + args->sky = psMetadataLookupF32(NULL, arguments, "-sky"); + args->shutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-shutoutc")); + + if (!args->camera) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "camera name not provided."); + return NULL; + } + if (!strcasecmp(args->camera, "GPC1")) { + args->obscode = psStringCopy("F51"); + } + if (!strcasecmp(args->camera, "GPC2")) { + args->obscode = psStringCopy("F52"); + } + if (!args->obscode) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "unknown camera name."); + return NULL; + } + + psTrace("ppMops.args", 1, "Done parsing command-line arguments\n"); + + psFree(arguments); + return args; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsDetections.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsDetections.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsDetections.c (revision 41112) @@ -0,0 +1,55 @@ +#ifdef HAVE_CONFIG_H +#include +#endif +#include +#include +#include "ppMops.h" + +static void mopsDetectionsFree(ppMopsDetections *det) +{ + psFree(det->component); + psFree(det->raBoresight); + psFree(det->decBoresight); + psFree(det->filter); + psFree(det->table); + psFree(det->x); + psFree(det->y); + psFree(det->ra); + psFree(det->dec); + psFree(det->raErr); + psFree(det->decErr); + psFree(det->mask); + psFree(det->fpashutoutc); + psFree(det->fpashutcutc); + return; +} + +ppMopsDetections *ppMopsDetectionsAlloc(void) +{ + ppMopsDetections *det = psAlloc(sizeof(ppMopsDetections)); // Detections, to return + psMemSetDeallocator(det, (psFreeFunc)mopsDetectionsFree); + det->component = NULL; + det->raBoresight = NULL; + det->decBoresight = NULL; + det->filter = NULL; + det->airmass = NAN; + det->exptime = NAN; + det->posangle = NAN; + det->alt = NAN; + det->az = NAN; + det->mjd = NAN; + det->seeing = NAN; + det->num = 0; + det->table = NULL; + det->x = NULL; + det->y = NULL; + det->ra = NULL; + det->dec = NULL; + det->raErr = NULL; + det->decErr = NULL; + det->mask = NULL; + det->diffSkyfileId = 0; + det->fpashutoutc = NULL; + det->fpashutcutc = NULL; + return det; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsGetSkyChipPsfVersion.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsGetSkyChipPsfVersion.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsGetSkyChipPsfVersion.c (revision 41112) @@ -0,0 +1,29 @@ +# include "ppMops.h" + +int ppMopsGetSkyChipPsfVersion(const psFits* fits) { + // It is expected that the fits stream cursor was advanced + // till the "SkyChip.psf" + psMetadata *headerSkyChip = psFitsReadHeader(NULL, fits); + char* version = psMetadataLookupStr(NULL, headerSkyChip, "EXTTYPE"); + psTrace("ppMopsGetSkyChipPsfVersion", 1, + "EXTTYPE value: [%s]\n", version); + if (strcmp(version, "PS1_DV1") == 0) { + psFree(headerSkyChip); + return 1; + } else if (strcmp(version, "PS1_DV2") == 0) { + psFree(headerSkyChip); + return 2; + } else if (strcmp(version, "PS1_DV3") == 0) { + psFree(headerSkyChip); + return 3; + } else if (strcmp(version, "PS1_DV4") == 0) { + psFree(headerSkyChip); + return 4; + } else if (strcmp(version, "PS1_DV5") == 0) { + psFree(headerSkyChip); + return 5; + } + psWarning("Unsupported EXTTYPE in SkyChip.psf table: [%s]", version); + psFree(headerSkyChip); + return 0; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsMerge.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsMerge.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsMerge.c (revision 41112) @@ -0,0 +1,313 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppMops.h" + +#define LEAF_SIZE 4 // Size of leaf +#define MATCH_RADIUS SEC_TO_RAD(1.0) // Matching radius +#define MJD_TOL 1.0/3600.0/24.0 // Tolerance for MJD matching +#define BORESIGHT_TOL SEC_TO_RAD(1.0) // Tolerance for boresight matching +#define EXPTIME_TOL 1.0e-3 // Tolerance for exposure time matching +#define POSANGLE_TOL SEC_TO_RAD(1.0) // Tolerance for position angle matching +#define AIRMASS_TOL 1.0e-3 // Tolerance for airmass matching + +#if 0 +#undef psTrace +#define psTrace(facil, level, ...) \ + if (level <= 5) { \ + fprintf(stderr, __VA_ARGS__); \ + } +#endif + +// Get distance from detection to centre of image +static float mergeDistance(const ppMopsDetections *detections, // Detections of interest + long index // Index for source of interest + ) +{ + float dx = detections->x->data.F32[index] - detections->naxis1 / 2.0; + float dy = detections->y->data.F32[index] - detections->naxis2 / 2.0; + return PS_SQR(dx) + PS_SQR(dy); +} + + +bool ppMopsPurgeDuplicates(const psArray *detections) +{ + PS_ASSERT_ARRAY_NON_NULL(detections, NULL); + + int numInputs = detections->n; // Number of inputs + psTrace("ppMops.merge", 1, "Checking detections from %d inputs\n", numInputs); + + long total = 0; // Total number of sources + psArray *duplicates = psArrayAlloc(numInputs); // Vector of duplicate bits for each input + psVector *dupNum = psVectorAlloc(numInputs, PS_TYPE_U32); // Number of duplicates for each input + psVectorInit(dupNum, 0); + for (int i = 0; i < numInputs; i++) { + ppMopsDetections *det = detections->data[i]; // Detections from + if (!det || det->num == 0) { + continue; + } + psVector *dupes = duplicates->data[i] = psVectorAlloc(det->num, PS_TYPE_U8); + psVectorInit(dupes, 0); + total += det->num; + } + psTrace("ppMops.merge", 2, "Total detections: %ld\n", total); + + psVector *raMerged = psVectorAlloc(total, PS_TYPE_F64); // Merged RAs + psVector *decMerged = psVectorAlloc(total, PS_TYPE_F64); // Merged Decs + psVector *sourceMerged = psVectorAlloc(total, PS_TYPE_U16); // Source image of merged sources + psVector *indexMerged = psVectorAlloc(total, PS_TYPE_U32); // Index of merged sources + long num = 0; // Number of merged sources + + const char *raBoresight = NULL, *decBoresight = NULL; // Boresight coordinates + const char *filter = NULL; // Filter name + float airmass = NAN, exptime = NAN; // Exposure details + double posangle = NAN, alt = NAN, az = NAN; // Telescope pointing + double mjd = NAN; // Time of exposure + + for (int i = 0; i < numInputs; i++) { + psTrace("ppMops.merge", 1, "Processing batch %d\n", i); + ppMopsDetections *det = detections->data[i]; // Detections of interest + if (!det) { + psTrace("ppMops.merge", 3, "Ignoring NULL input %d\n", i); + continue; + } else if (det->num == 0) { + psTrace("ppMops.merge", 3, "Ignoring empty input %d\n", i); + continue; + } + + // Check exposure characteristics + if (num == 0) { + raBoresight = det->raBoresight; + decBoresight = det->decBoresight; + filter = det->filter; + airmass = det->airmass; + exptime = det->exptime; + posangle = det->posangle; + alt = det->alt; + az = det->az; + } else { + if (strcmp(raBoresight, det->raBoresight) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure RA values differ: %s vs %s", + raBoresight, det->raBoresight); + return false; + } + if (strcmp(decBoresight, det->decBoresight) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure Dec values differ: %s vs %s", + decBoresight, det->decBoresight); + return false; + } + if (strcmp(filter, det->filter) != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure filter values differ: %s vs %s", + filter, det->filter); + return false; + } + if (fabsf(airmass - det->airmass) > AIRMASS_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure airmass values differ: %f vs %f", + airmass, det->airmass); + return false; + } + if (fabsf(exptime - det->exptime) > EXPTIME_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure exposure time values differ: %f vs %f", + exptime, det->exptime); + return false; + } + if (fabs(posangle - det->posangle) > POSANGLE_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure position angle values differ: %f vs %f", + posangle, det->posangle); + return false; + } + if (fabs(alt - det->alt) > BORESIGHT_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure altitude values differ: %lf vs %lf", + alt, det->alt); + return false; + } + if (fabs(az - det->az) > BORESIGHT_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure azimuth values differ: %lf vs %lf", + az, det->az); + return false; + } + if (fabs(mjd - det->mjd) > MJD_TOL) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure MJD values differ: %lf vs %lf", + mjd, det->mjd); + return false; + } + } + + psTrace("ppMops.merge", 3, "Accepting %ld detections from input %d\n", det->num, i); + memcpy(&raMerged->data.F64[num], det->ra->data.F64, det->num * PSELEMTYPE_SIZEOF(PS_TYPE_F64)); + memcpy(&decMerged->data.F64[num], det->dec->data.F64, det->num * PSELEMTYPE_SIZEOF(PS_TYPE_F64)); + for (long j = 0, k = num; j < det->num; j++, k++) { + sourceMerged->data.U16[k] = i; + indexMerged->data.U32[k] = j; + } + num += det->num; + } + + psTrace("ppMops.merge", 3, "Generating kd-tree from %ld sources\n", num); + psTree *tree = psTreePlant(2, LEAF_SIZE, PS_TREE_SPHERICAL, raMerged, decMerged); // kd tree + if (!tree) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kd tree"); + return false; + } + + for (int i = 0; i < detections->n; i++) { + ppMopsDetections *det = detections->data[i]; // Detections of interest + if (!det) { + continue; + } else if (det->num == 0) { + continue; + } + psTrace("ppMops.merge", 3, "Checking %ld detections from input %d\n", det->num, i); + + psVector *dupes = duplicates->data[i]; // Duplicates list + psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of interest + for (int j = 0; j < det->num; j++) { + // XXX: added by Bill + if (det->mask->data.U8[j]) { + // we marked this source marked as bad when we read it, mark it as a duplicate + dupes->data.U8[j] = 0xFF; + continue; + } + + coords->data.F64[0] = det->ra->data.F64[j]; + coords->data.F64[1] = det->dec->data.F64[j]; + psVector *indices = psTreeAllWithin(tree, coords, MATCH_RADIUS); // Indices for matching sources + if (!indices) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to search for matches"); + psFree(coords); + psFree(tree); + return false; + } + psTrace("ppMops.merge", 5, "%ld matches for source %d from input %d\n", indices->n, j, i); + psAssert(indices->n > 0, "Expect at least one match for source %d in input %d", j, i); + + if (indices->n == 1 && sourceMerged->data.U16[indices->data.S64[0]] == i) { + // It's myself + psFree(indices); + continue; + } + + // Which one do we keep? + float bestDistance = INFINITY; // Best distance to centre + long bestIndex = -1; // Index with best distance + for (int k = 0; k < indices->n; k++) { + long mergeIndex = indices->data.S64[k]; // Index of point in merged list + int source = sourceMerged->data.U16[mergeIndex]; // Source image + if (source == i) { + continue; + } + long index = indexMerged->data.U32[mergeIndex]; // Index in source + psVector *dupes = duplicates->data[source]; // Duplicates list + if (dupes->data.U8[index]) { + continue; + } + + float distance = mergeDistance(detections->data[source], index); // Distance to centre of image + if (distance < bestDistance) { + bestDistance = distance; + bestIndex = index; + } + } + + if (bestIndex >= 0) { + float distance = mergeDistance(det, j); // Distance to centre of image + if (bestIndex >= 0 && distance < bestDistance) { + psTrace("ppMops.merge", 6, "New source clobbers old sources\n"); + for (int k = 0; k < indices->n; k++) { + long mergeIndex = indices->data.S64[k]; // Index of point + int source = sourceMerged->data.U16[mergeIndex]; // Source image + if (source == i) { + continue; + } + long index = indexMerged->data.U32[mergeIndex]; // Index in source + psVector *dupes = duplicates->data[source]; // Duplicates list + if (!dupes->data.U8[index]) { + dupes->data.U8[index] = 0xFF; + dupNum->data.U32[source]++; + } + } + } else { + psTrace("ppMops.merge", 6, "Old sources clobber new source\n"); + dupes->data.U8[j] = 0xFF; + dupNum->data.U32[i]++; + } + } + psFree(indices); + } + psFree(coords); + } + psFree(tree); + psFree(raMerged); + psFree(decMerged); + psFree(sourceMerged); + psFree(indexMerged); + + // Remove duplicates + for (int i = 0; i < detections->n; i++) { + ppMopsDetections *det = detections->data[i]; // Detections of interest + if (!det) { + continue; + } else if (det->num == 0) { + continue; + } + psTrace("ppMops.merge", 3, "Purging %d duplicates from input %d\n", dupNum->data.U32[i], i); + +#define VECTOR_PURGE_CASE(TYPE) \ + case PS_TYPE_##TYPE: { \ + long j = 0; \ + for (long i = 0; i < vector->n; i++) { \ + if (!dupes->data.U8[i]) { \ + if (i == j) { \ + j++; \ + continue; \ + } \ + vector->data.TYPE[j++] = vector->data.TYPE[i]; \ + } \ + } \ + vector->n = j; \ + } \ + break + + psVector *dupes = duplicates->data[i]; // Duplicates + psArray *table = psListToArray(det->table->list); // Table of data + long newLength = -1; + for (int t = 0; t < table->n; t++) { + psMetadataItem *item = table->data[t]; // Table item + psAssert(item->type == PS_DATA_VECTOR, "Table column is not a vector: %x", item->type); + psVector *vector = item->data.V; // Vector to purge + switch (vector->type.type) { + VECTOR_PURGE_CASE(U8); + VECTOR_PURGE_CASE(U16); + VECTOR_PURGE_CASE(U32); + VECTOR_PURGE_CASE(U64); + VECTOR_PURGE_CASE(S8); + VECTOR_PURGE_CASE(S16); + VECTOR_PURGE_CASE(S32); + VECTOR_PURGE_CASE(S64); + VECTOR_PURGE_CASE(F32); + VECTOR_PURGE_CASE(F64); + default: + psAbort("Unrecognised vector type: %x", vector->type.type); + } + if (newLength == -1) { + newLength = vector->n; + } else if (vector->n != newLength) { + psAbort("Unexpected new length found : %ld expected: %ld", + vector->n, newLength); + } + } + // XXX IS this safe? Perhaps save in numGood? + det->num = newLength; + psFree(table); + } + psFree(dupNum); + psFree(duplicates); + + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsRead.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsRead.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsRead.c (revision 41112) @@ -0,0 +1,495 @@ +#define DIRTY_BUG_FIX 0 + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppMops.h" + +static void addDummyValues(psMetadata* md, long size, int version); +static void replaceDummyValuesF32(const char* colName, psMetadata* source, psMetadata* target, psVector* indexes); +static void replaceDummyValuesS32(const char* colName, psMetadata* source, psMetadata* target, psVector* indexes); +static void renameDummyValuesF32(const char* colNameSource, psMetadata* source, const char* colNameTarget, psMetadata* target, psVector* indexes); + +/* + ppMopsRead possibly modifies the args->version if the user did not + set it explicitely. +*/ +psArray *ppMopsRead(ppMopsArguments *args) +{ + psTrace("ppMops.read", 1, "Reading input detections\n"); + + psArray *inNames = args->input; // Input names + long num = inNames->n; // Number of inputs + psArray *detections = psArrayAlloc(num); // Array of detections, to return + for (int i = 0; i < num; i++) { + const char *name = inNames->data[i]; + printf("Input filename:\n%s\n", name); + + psFits *fits = psFitsOpen(name, "r"); // FITS file + if (!fits) { + psError(PS_ERR_IO, false, "Unable to open input %d", i); + return false; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); // Primary header + if (!header) { + psError(PS_ERR_IO, false, "Unable to read header %d", i); + return false; + } + + psS64 diffSkyfileId = psMetadataLookupS64(NULL, header, "IMAGEID"); // Identifier for image + if (diffSkyfileId == 0) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find identifier for image %d", i); + return false; + } + + if (!psFitsMoveExtName(fits, "SkyChip.psf")) { + psError(PS_ERR_IO, false, "Unable to move to HDU with detections"); + return false; + } + int skyChipPsfVersion = ppMopsGetSkyChipPsfVersion(fits); + if (args->version == 0) { + psTrace("ppMops.read", 1, "Changing args->version to %d\n", skyChipPsfVersion); + args->version = (unsigned short) skyChipPsfVersion; + } + if (skyChipPsfVersion == 0) { + // Try to read with the user specified version? + skyChipPsfVersion = args->version; + } + /* Display a warning message if there are version + inconsistencies between the file and the flag (note that + those inconsistencies might be wanted) */ + if (skyChipPsfVersion != args->version) { + if (skyChipPsfVersion > args->version) { + psWarning("The FITS data will be downgraded from PS1_DV%d to PS1_DV%d\n", + skyChipPsfVersion, args->version); + } else { // Necessarily: skyChipPsfVersion > args->version + psWarning("The FITS data will be upgraded from PS1_DV%d to PS1_DV%d (new values set to default 0, NaN...)\n", + skyChipPsfVersion, args->version); + } + } + + long size = psFitsTableSize(fits); // Size of table + if (size <= 0) { + psErrorStackPrint(stderr, "Unable to determine size of table %d", i); + psErrorClear(); + psWarning("Ignoring input %d", i); + psFree(header); + psFitsClose(fits); + continue; + } + ppMopsDetections *det = ppMopsDetectionsAlloc(); + det->platescale = NAN; + detections->data[i] = det; + det->component = psStringNCopy(name, strrchr(name, '.') - name); // Strip off extension + det->num = size; + det->diffSkyfileId = diffSkyfileId; + + psTrace("ppMops.read", 3, "Reading %ld rows from %s\n", size, (const char*)inNames->data[i]); + + det->raBoresight = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.RA")); + det->decBoresight = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.DEC")); + det->filter = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.FILTER")); + det->airmass = psMetadataLookupF32(NULL, header, "AIRMASS"); + det->exptime = psMetadataLookupF32(NULL, header, "EXPTIME"); + det->posangle = psMetadataLookupF64(NULL, header, "FPA.POSANGLE"); + det->alt = psMetadataLookupF64(NULL, header, "FPA.ALT"); + det->az = psMetadataLookupF64(NULL, header, "FPA.AZ"); + det->mjd = psMetadataLookupF64(NULL, header, "MJD-OBS") + det->exptime / 2.0 / 3600 / 24; + //The global SEEING is the mean of all defined FWHM_MAJ + det->seeing = psMetadataLookupF32(NULL, header, "FWHM_MAJ"); + det->naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns + det->naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); // Number of rows + det->fpashutoutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.SHUTOUTC")); + det->fpashutcutc = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.SHUTCUTC")); + det->refcat = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "PSREFCAT")); + psFree(header); + + psMetadata *table = psFitsReadTableAllColumns(fits); // Table of interest + if (!table) { + psError(PS_ERR_IO, false, "Unable to read table %d", i); + return NULL; + } + det->table = table; + psTrace("ppMops.read", 19, "PSF table:\n%s\n", psMetadataConfigFormat(det->table)); + psTrace("ppMops.read", 19, "End of PSF table\n"); + + //fittedTrail extension (Supported for releases 2 and above) + if (skyChipPsfVersion >= 2) { + //First: append all the new columns that we want to the existing table + addDummyValues(det->table, size, args->version); + if (!psFitsMoveExtName(fits, "SkyChip.xfit")) { + psTrace("ppMops.read", 3, "No fitted trails extension"); + } else { + psTrace("ppMops.read", 3, "Fitted trails extension found\n"); + psTrace("ppMops.read", 10, "Getting size?\n"); + int fittedTrailsSize = psFitsTableSize(fits); + psTrace("ppMops.read", 10, "size = %d\n", fittedTrailsSize); + if (fittedTrailsSize <= 0) { + psErrorStackPrint(stderr, "Unable to determine size of fitted trails extension table %d", i); + psTrace("ppMops.read", 3, "No entry in fitted trails extension!!!!\n"); + psErrorClear(); + } else { + psTrace("ppMops.read", 10, "Reading table of interest?\n"); + psMetadata* fittedTrails = psFitsReadTableAllColumns(fits); // Table of interest + psTrace("ppMops.read", 10, "OK for table of interest\n"); + if (!fittedTrails) { + psError(PS_ERR_IO, false, "Unable to read fittedTrails table in file %d", i); + return NULL; + } + //Iterate on the different names and types expected in the fittedTrails parameters + psTrace("ppMops.read", 10, "Getting IPP_IDET\n"); + psVector* idet = psMetadataLookupVector(NULL, fittedTrails, "IPP_IDET"); + replaceDummyValuesF32("X_EXT", fittedTrails, det->table, idet); + replaceDummyValuesF32("Y_EXT", fittedTrails, det->table, idet); + replaceDummyValuesF32("X_EXT_SIG", fittedTrails, det->table, idet); + replaceDummyValuesF32("Y_EXT_SIG", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_INST_MAG", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_INST_MAG_SIG", fittedTrails, det->table, idet); + replaceDummyValuesS32("NPARAMS", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_WIDTH_MAJ", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_WIDTH_MIN", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_THETA", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_WIDTH_MAJ_ERR", fittedTrails, det->table, idet); + // EXT_WIDTH_MIN_ERR is actually undefined but set to 0. in CMF files + // We explicitely let it set to NaN, hence the commented out + // following line + //replaceDummyValuesF32("EXT_WIDTH_MIN_ERR", fittedTrails, det->table, idet); + replaceDummyValuesF32("EXT_THETA_ERR", fittedTrails, det->table, idet); + psTrace("ppMops.read", 10, "Got all version 2 values\n"); + if (skyChipPsfVersion >= 3) { + psTrace("ppMops.read", 10, "Getting PS1_DV3 data\n"); + replaceDummyValuesF32("RA_EXT", fittedTrails, det->table, idet); + // Note: RA_EXT_SIGMA is computed (see det->raExtErr) + replaceDummyValuesF32("DEC_EXT", fittedTrails, det->table, idet); + // Note: DEC_EXT_SIGMA is computed (see det->decExtErr) + //Note: POSANG_EXT is POSANGLE in xfit + renameDummyValuesF32("POSANGLE", fittedTrails, "POSANG_EXT", det->table, idet); + //Not written anymore but still used in computation + renameDummyValuesF32("PLTSCALE", fittedTrails, "PLTSCALE_EXT", det->table, idet); + renameDummyValuesF32("EXT_INST_FLUX", fittedTrails, "EXT_FLUX", det->table, idet); + replaceDummyValuesF32("EXT_CAL_MAG", fittedTrails, det->table, idet); + renameDummyValuesF32("EXT_INST_MAG_SIG", fittedTrails, "EXT_MAG_SIG", det->table, idet); + replaceDummyValuesF32("EXT_CHISQ", fittedTrails, det->table, idet); + replaceDummyValuesS32("EXT_NDOF", fittedTrails, det->table, idet); + psTrace("ppMops.read", 10, "Got all version 3 values\n"); + } + } + } + } + psFitsClose(fits); + + if (args->version == 0) { + if (skyChipPsfVersion < 2) { + // XXX: TODO: Do we need to add dummy vectors for the missing columns? + } + } + + psVector *ra = psMetadataLookupVector(NULL, table, "RA_PSF"); + psVector *dec = psMetadataLookupVector(NULL, table, "DEC_PSF"); + det->raErr = psVectorAlloc(size, PS_TYPE_F64); + det->decErr = psVectorAlloc(size, PS_TYPE_F64); + if (skyChipPsfVersion >= 3) { + det->raExtErr = psVectorAlloc(size, PS_TYPE_F64); + det->decExtErr = psVectorAlloc(size, PS_TYPE_F64); + } else { + det->raExtErr = NULL; + det->decExtErr = NULL; + } + det->mask = psVectorAlloc(size, PS_TYPE_U8); + + // convert ra and dec to radians for use in the purge duplicates function + det->ra = (psVector*)psBinaryOp(NULL, ra, "*", psScalarAlloc(DEG_TO_RAD(1.0), PS_TYPE_F64)); + det->dec = (psVector*)psBinaryOp(NULL, dec, "*", psScalarAlloc(DEG_TO_RAD(1.0), PS_TYPE_F64)); + det->x = psMemIncrRefCounter(psMetadataLookupVector(NULL, table, "X_PSF")); + det->y = psMemIncrRefCounter(psMetadataLookupVector(NULL, table, "Y_PSF")); + if (!det->ra || !det->dec || !det->x || !det->y) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find all of RA, Dec, X and Y columns"); + return NULL; + } + + // Add our new vectors to the table so that duplicates and masked items may be purged + psMetadataAddVector(table, PS_LIST_HEAD, "RA_ERR", 0, NULL, det->raErr); + psMetadataAddVector(table, PS_LIST_HEAD, "DEC_ERR", 0, NULL, det->decErr); + if (skyChipPsfVersion >= 3) { + psMetadataAddVector(table, PS_LIST_HEAD, "RA_EXT_SIGMA", 0, NULL, det->raExtErr); + psMetadataAddVector(table, PS_LIST_HEAD, "DEC_EXT_SIGMA", 0, NULL, det->decExtErr); + } + + psTrace("ppMops.read", 2, "Read %ld rows from %s\n", det->num, det->component); + + psVector *mag = psMetadataLookupVector(NULL, table, "PSF_INST_MAG"); + psVector *magErr = psMetadataLookupVector(NULL, table, "PSF_INST_MAG_SIG"); + psVector *xErrV = psMetadataLookupVector(NULL, table, "X_PSF_SIG"); + psVector *yErrV = psMetadataLookupVector(NULL, table, "Y_PSF_SIG"); + psVector *scaleV = psMetadataLookupVector(NULL, table, "PLTSCALE"); + psVector *angleV = psMetadataLookupVector(NULL, table, "POSANGLE"); + psVector *flagsV = psMetadataLookupVector(NULL, table, "FLAGS"); + psVector *flags2V = psMetadataLookupVector(NULL, table, "FLAGS2"); + psVector *xExtErrV = NULL; + psVector *yExtErrV = NULL; + psVector *scaleExtV = NULL; + psVector *angleExtV = NULL; + if (skyChipPsfVersion >= 3) { + xExtErrV = psMetadataLookupVector(NULL, table, "X_EXT_SIG"); + yExtErrV = psMetadataLookupVector(NULL, table, "Y_EXT_SIG"); + scaleExtV = psMetadataLookupVector(NULL, table, "PLTSCALE_EXT"); + angleExtV = psMetadataLookupVector(NULL, table, "POSANG_EXT"); + } + + //MEH -- diff params, may need skyChipPsfVersion + // -- unclear for npos, remove for now + //psVector *npos = psMetadataLookupVector(NULL, table, "DIFF_NPOS"); + psVector *fpos = psMetadataLookupVector(NULL, table, "DIFF_FRATIO"); + psVector *rbad = psMetadataLookupVector(NULL, table, "DIFF_NRATIO_BAD"); + psVector *rmask = psMetadataLookupVector(NULL, table, "DIFF_NRATIO_MASK"); + psVector *rall = psMetadataLookupVector(NULL, table, "DIFF_NRATIO_ALL"); + + psVector *rp = psMetadataLookupVector(NULL, table, "DIFF_R_P"); + psVector *rm = psMetadataLookupVector(NULL, table, "DIFF_R_M"); + psVector *snp = psMetadataLookupVector(NULL, table, "DIFF_SN_P"); + psVector *snm = psMetadataLookupVector(NULL, table, "DIFF_SN_M"); + + psVector *mxx = psMetadataLookupVector(NULL, table, "MOMENTS_XX"); + psVector *myy = psMetadataLookupVector(NULL, table, "MOMENTS_YY"); + //MEH + + double plateScale = 0.0; // Plate scale + long numGood = 0; // Number of good rows + for (long row = 0; row < size; row++) { + //psU32 flags = flagsV->data.U32[row]; // psFitsTableGetU32(NULL, table, row, "FLAGS"); + psU64 flags = flagsV->data.U64[row]; // table reads in as U64 + if (flags & SOURCE_MASK) { + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %lud", row, i, flags); + det->mask->data.U8[row] = 0xFF; + continue; + } + + //MEH -- section off flags2 and diffstats cuts to just be for WSdiffs -- + // -- need to test similarity/differences for WW/WS -- have separate block for WW + // -- may need to move block + // -- block should set vars and then run rej afterwards -- not clear want all for both -- + +// if (!strcasecmp(args->difftype, "WW")) { +// // cut non-physical moments for PS -- <2.0? -- need to modify w/ mask level for edges +// if (!isfinite(mxx->data.F32[row]) || !isfinite(myy->data.F32[row])) { +// psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN MOMENTS_XX/YY params: < 2 2: %f %f ", row, i, mxx->data.F32[row],myy->data.F32[row]); +// det->mask->data.U8[row] = 0xFF; +// continue; +// } +// if ( (mxx->data.F32[row]<2.0 || myy->data.F32[row]<2.0) && rmask->data.F32[row]>0.2 ) { +// psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of non-physical MOMENTS_XX/YY params: < 2 2: %f %f ", row, i, mxx->data.F32[row],myy->data.F32[row]); +// det->mask->data.U8[row] = 0xFF; +// continue; +// } +// } + + if (!strcasecmp(args->difftype, "WS")) { +// if (!strcasecmp(args->difftype, "WS") && (!strcasecmp(args->camera, "GPC2")) { + psU64 flags2 = flags2V->data.U64[row]; // table reads in as U64 + if (flags2 & SOURCE_MASK2) { + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags2: %lud", row, i, flags2); + det->mask->data.U8[row] = 0xFF; + continue; + } + + // MEH -- filter on same source P/M in diff (movers shouldn't have) -- WS deeper and may have M + // cut P/M sources at same position (stationary) -- NULL is issue when isn't one, lower s/n larger centroid error + if ( (isfinite(rp->data.F32[row]) && isfinite(snp->data.F32[row])) && (rp->data.F32[row]>4.0 && snp->data.F32[row]>5.0) ) { + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN R_P/SN_P or R_P/SN_P params: >4 >5 : %f %f ", row, i, rp->data.F32[row],snp->data.F32[row]); + det->mask->data.U8[row] = 0xFF; + continue; + } + + if ( (isfinite(rm->data.F32[row]) && isfinite(snm->data.F32[row])) && (rm->data.F32[row]<4.0 && snm->data.F32[row]>5.0) ) { + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN R_M/SN_M or R_M/SN_M params: <4 >5 : %f %f ", row, i, rm->data.F32[row],snm->data.F32[row]); + det->mask->data.U8[row] = 0xFF; + continue; + } + + // MEH -- add diff params to discard -- needs to be in config.. -- WSdiff only? + // -- needs to be finite to test but unclear if should reject (probably) + // -- using Napoli 2013 to start with + // DIFF_NPOS>3 + // DIFF_FRATIO>0.6 + // DIFF_NRATIO_BAD>0.4 + // DIFF_NRATIO_MASK>0.4 + // DIFF_NRATIO_ALL>0.3 + // -- increase the cuts for gpc2 to exclude the poor masked edges more + // DIFF_FRATIO>0.6 -- losing some fainter sources @0.7 + // DIFF_NRATIO_BAD>0.5 + // DIFF_NRATIO_MASK>0.6 + // DIFF_NRATIO_ALL>0.5 + // -- unclear about npos still -- so remove from check + //if (npos->data.S32[row]<=3 || + if (!isfinite(fpos->data.F32[row]) || !isfinite(rbad->data.F32[row]) || !isfinite(rmask->data.F32[row]) || !isfinite(rall->data.F32[row])) { + //psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN diff params: 3, 0.6, 0.4, 0.4, 0.3: %d %g %f %f %f ", row, i, npos->data.S32[row],fpos->data.F32[row],rbad->data.F32[row],rmask->data.F32[row],rall->data.F32[row]); + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN diff params: 3, 0.6, 0.4, 0.4, 0.3: %g %f %f %f ", row, i, fpos->data.F32[row],rbad->data.F32[row],rmask->data.F32[row],rall->data.F32[row]); + det->mask->data.U8[row] = 0xFF; + continue; + } + if (fpos->data.F32[row]<=0.6 || rbad->data.F32[row]<=0.5 || rmask->data.F32[row]<=0.6 || rall->data.F32[row]<=0.5) { + //psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of diff params: 3, 0.6, 0.4, 0.4, 0.3: %d %g %f %f %f ", row, i, npos->data.S32[row], fpos->data.F32[row], rbad->data.F32[row], rmask->data.F32[row], rall->data.F32[row]); + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of diff params: 3, 0.6, 0.4, 0.4, 0.3: %g %f %f %f ", row, i, fpos->data.F32[row], rbad->data.F32[row], rmask->data.F32[row], rall->data.F32[row]); + det->mask->data.U8[row] = 0xFF; + continue; + } + + // cut non-physical moments for PS -- <2.0? -- need to modify w/ mask level for edges + if (!isfinite(mxx->data.F32[row]) || !isfinite(myy->data.F32[row])) { + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of NAN MOMENTS_XX/YY params: < 2 2: %f %f ", row, i, mxx->data.F32[row],myy->data.F32[row]); + det->mask->data.U8[row] = 0xFF; + continue; + } + if ( (mxx->data.F32[row]<=2.0 || myy->data.F32[row]<=2.0) && rmask->data.F32[row]>0.2 ) { + psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of non-physical MOMENTS_XX/YY params: < 2 2: %f %f ", row, i, mxx->data.F32[row],myy->data.F32[row]); + det->mask->data.U8[row] = 0xFF; + continue; + } + + } + + // Calculate error in RA, Dec + double xErr = xErrV->data.F32[row]; + double yErr = yErrV->data.F32[row]; + double scale = scaleV->data.F32[row]; + double angle = angleV->data.F32[row]; + if (!isfinite(det->x->data.F32[row]) || !isfinite(det->y->data.F32[row]) || + !isfinite(det->ra->data.F64[row]) || !isfinite(det->dec->data.F64[row]) || + !isfinite(mag->data.F32[row]) || !isfinite(magErr->data.F32[row]) || + !isfinite(xErr) || !isfinite(yErr) || !isfinite(scale) || !isfinite(angle)) { + psTrace("ppMops.read", 10, + "Discarding row %ld from input %d because of non-finite values: " + "%f %f %lf %lf %f %f %f %f %f %f", + row, i, + det->x->data.F32[row], det->y->data.F32[row], + det->ra->data.F64[row], det->dec->data.F64[row], + mag->data.F32[row], magErr->data.F32[row], + xErr, yErr, scale, angle); + det->mask->data.U8[row] = 0xFF; + continue; + } + + // XXX Not at all sure I've got the angles around the right way here... + double cosAngle = cos(angle), sinAngle = sin(angle); + double cosAngle2 = PS_SQR(cosAngle), sinAngle2 = PS_SQR(sinAngle); + double xErr2 = PS_SQR(xErr), yErr2 = PS_SQR(yErr); + double errScale = scale / 3600.0; + det->raErr->data.F64[row] = errScale * sqrt(cosAngle2 * xErr2 + sinAngle2 * yErr2); + det->decErr->data.F64[row] = errScale * sqrt(sinAngle2 * xErr2 + cosAngle2 * yErr2); + det->mask->data.U8[row] = 0; + plateScale += scale; + //Update the platescale value (should be a constant) + if (isfinite(scale)) { + det->platescale = scale; + //printf("platescale = %g\n", scale); + } + numGood++; + + // Same for EXT data if version permits + if (skyChipPsfVersion >= 3) { + xErr = xExtErrV->data.F32[row]; + yErr = yExtErrV->data.F32[row]; + scale = scaleExtV->data.F32[row]/3600.; + angle = angleExtV->data.F32[row]; + cosAngle = cos(angle); + sinAngle = sin(angle); + cosAngle2 = PS_SQR(cosAngle); + sinAngle2 = PS_SQR(sinAngle); + xErr2 = PS_SQR(xErr); + yErr2 = PS_SQR(yErr); + errScale = scale / 3600.0; + det->raExtErr->data.F64[row] = errScale * sqrt(cosAngle2 * xErr2 + sinAngle2 * yErr2); + det->decExtErr->data.F64[row] = errScale * sqrt(sinAngle2 * xErr2 + cosAngle2 * yErr2); + } + } + det->numGood = numGood; + + if (isfinite(args->zp) && numGood > 0) { + psBinaryOp(mag, mag, "+", psScalarAlloc(args->zp, PS_TYPE_F32)); + } + + printf("Detection platescale = %g\n", det->platescale); + psTrace("ppMops.read", 2, "Read %ld good rows from %s\n", numGood, (const char*)name); + } + + psTrace("ppMops.read", 1, "Done reading input detections\n"); + + return detections; +} + +static psVector* createDummyF32(long size) { + psVector* dummy = psVectorAlloc(size, PS_TYPE_F32); + psVectorInit(dummy, NAN); + return dummy; +} + +static psVector* createDummyS32(long size) { + psVector* dummy = psVectorAlloc(size, PS_TYPE_S32); + psVectorInit(dummy, 0); + return dummy; +} + +static void addDummyValues(psMetadata* md, long size, int version) { + psMetadataAdd(md, PS_LIST_TAIL, "X_EXT", PS_DATA_VECTOR, "EXT model x coordinate", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "Y_EXT", PS_DATA_VECTOR, "EXT model y coordinate", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "X_EXT_SIG", PS_DATA_VECTOR, "Sigma in EXT x coordinate", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "Y_EXT_SIG", PS_DATA_VECTOR, "Sigma in EXT y coordinate", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_INST_MAG", PS_DATA_VECTOR, "EXT fit instrumental magnitude", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_INST_MAG_SIG", PS_DATA_VECTOR, "Sigma of PSF instrumental magnitude", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "NPARAMS", PS_DATA_VECTOR, "Number of model parameters", createDummyS32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_WIDTH_MAJ", PS_DATA_VECTOR, "EXT width (major axis), length for trail", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_WIDTH_MIN", PS_DATA_VECTOR, "EXT width (minor axis), sigma for trail", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_THETA", PS_DATA_VECTOR, "EXT orientation angle", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_WIDTH_MAJ_ERR", PS_DATA_VECTOR, "EXT width error (major axis)", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_WIDTH_MIN_ERR", PS_DATA_VECTOR, "EXT width error (minor axis)", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_THETA_ERR", PS_DATA_VECTOR, "EXT orientation angle (error)", createDummyF32(size)); + if (version >= 3) { + psTrace("ppMops.read", 10, "Adding columns for version PS1_DV%d\n", version); + psMetadataAdd(md, PS_LIST_TAIL, "RA_EXT", PS_DATA_VECTOR, "Fitted centroid RA", createDummyF32(size)); +/* psMetadataAdd(md, PS_LIST_TAIL, "RA_EXT_SIGMA", PS_DATA_VECTOR, "Fitted RA sigma", createDummyF32(size)); *\/ */ + psMetadataAdd(md, PS_LIST_TAIL, "DEC_EXT", PS_DATA_VECTOR, "Fitted centroid DEC", createDummyF32(size)); +/* psMetadataAdd(md, PS_LIST_TAIL, "DEC_EXT_SIGMA", PS_DATA_VECTOR, "Fitted DEC sigma", createDummyF32(size)); *\/ */ + psMetadataAdd(md, PS_LIST_TAIL, "POSANG_EXT", PS_DATA_VECTOR, "Fitted position angle", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "PLTSCALE_EXT", PS_DATA_VECTOR, "Plate scale at centroid", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_FLUX", PS_DATA_VECTOR, "Fitted flux", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_CAL_MAG", PS_DATA_VECTOR, "Calibrated mag", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_MAG_SIG", PS_DATA_VECTOR, "Mag sigma", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_CHISQ", PS_DATA_VECTOR, "Chi^2 of fit", createDummyF32(size)); + psMetadataAdd(md, PS_LIST_TAIL, "EXT_NDOF", PS_DATA_VECTOR, "Fit degrees of freedom", createDummyS32(size)); + } +} + +static void replaceDummyValuesF32(const char* colName, psMetadata* source, psMetadata* target, psVector* indexes) { + psTrace("ppMops.read", 10, "Trying to get column %s\n", colName); + psVector* source_vector = psMetadataLookupVector(NULL, source, colName); + psVector* target_vector = psMetadataLookupVector(NULL, target, colName); + for (long index = 0; indexn; ++index) { + target_vector->data.F32[indexes->data.S64[index]] = source_vector->data.F32[index]; + } + psTrace("ppMops.read", 10, "OK for %s\n", colName); +} + +static void replaceDummyValuesS32(const char* colName, psMetadata* source, psMetadata* target, psVector* indexes) { + psTrace("ppMops.read", 10, "Trying to get column %s\n", colName); + psVector* source_vector = psMetadataLookupVector(NULL, source, colName); + psVector* target_vector = psMetadataLookupVector(NULL, target, colName); + for (long index = 0; indexn; ++index) { + target_vector->data.S32[indexes->data.S64[index]] = source_vector->data.S32[index]; + } + psTrace("ppMops.read", 10, "OK for %s\n", colName); +} + +static void renameDummyValuesF32(const char* colNameSource, psMetadata* source, const char* colNameTarget, psMetadata* target, psVector* indexes) { + psTrace("ppMops.read", 10, "Trying to rename column %s to %s\n", colNameSource, colNameTarget); + psVector* source_vector = psMetadataLookupVector(NULL, source, colNameSource); + psVector* target_vector = psMetadataLookupVector(NULL, target, colNameTarget); + for (long index = 0; indexn; ++index) { + target_vector->data.F32[indexes->data.S64[index]] = source_vector->data.F32[index]; + } + psTrace("ppMops.read", 10, "OK for %s (renamed by %s)\n", colNameSource, colNameTarget); +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsWrite.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsWrite.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppMopsWrite.c (revision 41112) @@ -0,0 +1,484 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppMops.h" +#include "ppTranslateVersion.h" + +static bool addOutputColumn(psMetadata *table, const psArray *detections, long total, char *outColName, char *inColName, bool convertTo32); +static bool addSkyfileIDColumn(psMetadata *table, const psArray *detections, long total, char *colName); + +bool ppMopsWrite(const psArray *detections, const ppMopsArguments *args) +{ + psFits *fits = psFitsOpen(args->output, "w"); // FITS file + if (!fits) { + psError(PS_ERR_IO, false, "Unable to open output file."); + return false; + } + + psMetadata *header = psMetadataAlloc(); // Header to write + psString source = ppTranslateSource(), version = ppTranslateVersion(); + psMetadataAddStr(header, PS_LIST_TAIL, "SWSOURCE", 0, "Software source", source); + psMetadataAddStr(header, PS_LIST_TAIL, "SWVERSN", 0, "Software version", version); + ppTranslateVersionHeader(header); + psFree(source); + psFree(version); + + psMetadataAddStr(header, PS_LIST_TAIL, "EXP_NAME", 0, "Exposure name", args->exp_name); + psMetadataAddS64(header, PS_LIST_TAIL, "EXP_ID", 0, "Exposure identifier", args->exp_id); + psMetadataAddS64(header, PS_LIST_TAIL, "CHIP_ID", 0, "Chip stage identifier", args->chip_id); + psMetadataAddS64(header, PS_LIST_TAIL, "CAM_ID", 0, "Cam stage identifier", args->cam_id); + psMetadataAddS64(header, PS_LIST_TAIL, "FAKE_ID", 0, "Fake stage identifier", args->fake_id); + psMetadataAddS64(header, PS_LIST_TAIL, "WARP_ID", 0, "Warp stage identifier", args->warp_id); + psMetadataAddS64(header, PS_LIST_TAIL, "DIFF_ID", 0, "Diff stage identifier", args->diff_id); + psMetadataAddBool(header, PS_LIST_TAIL, "DIFF_POS", 0, "Positive subtraction?", args->positive); + + // Get these header words from the first input non null input + // but for SEEING which is the median of the (non-NULL) detections + // seeings + ppMopsDetections *det = NULL; + for (int d = 0; d < detections->n; d++) { + if (detections->data[d] != NULL) { + if (det == NULL) { + det = detections->data[d]; + } + } + } + //Get the SEEING weighted mean. Update the PLTSCALE value + float seeing = 0.; + int totalGood = 0; + int platescale_has_been_set = 0; + for (int d = 0; d < detections->n; d++) { + if ( (detections->data[d] != NULL) && (isfinite(((ppMopsDetections*) detections->data[d])->seeing)) ) { + seeing += ((ppMopsDetections*) detections->data[d])->seeing; + totalGood += 1; + if (isfinite(((ppMopsDetections*) detections->data[d])->platescale)) { + if (platescale_has_been_set == 0) { + det->platescale = ((ppMopsDetections*) detections->data[d])->platescale; + platescale_has_been_set = 1; + } else { + if (det->platescale != ((ppMopsDetections*) detections->data[d])->platescale) { + printf("Different values for platescale: %g - %g in detection %d\n", + det->platescale, + ((ppMopsDetections*) detections->data[d])->platescale, + d); + } + } + } + } + } + //printf("plate-scale = [%g]\n", det->platescale); + seeing /= (float) totalGood; + if (det != NULL) { + psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", det->mjd); + psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight); + psMetadataAddStr(header, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", det->decBoresight); + psMetadataAddF64(header, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", det->alt); + psMetadataAddF64(header, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", det->az); + psMetadataAddF64(header, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", det->exptime); + psMetadataAddF64(header, PS_LIST_TAIL, "ROTANGLE", 0, "Rotator position angle", det->posangle); + psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter); + psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", det->airmass); + psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", seeing); + //MOPS want the name FWHM for SEEING + psMetadataAddF32(header, PS_LIST_TAIL, "FWHM", 0, "Mean seeing", seeing); + } else { + psWarning("no inputs with surviving detections. output header will be incomplete"); + } + psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", args->obscode); + psMetadataAddF32(header, PS_LIST_TAIL, "MAGZP", 0, "Magnitude zero point", args->zp); + //MOPS want the name ZEROPOINT for MAGZP + psMetadataAddF32(header, PS_LIST_TAIL, "ZEROPOINT", 0, "Magnitude zero point", args->zp); + psMetadataAddF32(header, PS_LIST_TAIL, "MAGZPERR", 0, "Error in magnitude zero point", args->zpErr); + psMetadataAddF32(header, PS_LIST_TAIL, "ASTRORMS", 0, "RMS of astrometric fit", args->rmsAstrom); + psMetadataAddStr(header, PS_LIST_TAIL, "CMMTOBS", 0, "Exposure comment", args->comment); + psMetadataAddStr(header, PS_LIST_TAIL, "OBS_MODE", 0, "Exposure observation mode", args->obsMode); + psMetadataAddStr(header, PS_LIST_TAIL, "DIFFTYPE", 0, "WW: Warp-Warp diff / WS: Warp-Stack diff / SW: Stack-Warp diff", args->difftype); + psMetadataAddF32(header, PS_LIST_TAIL, "SKY", 0, "Exposure avg sky background", args->sky); + psMetadataAddStr(header, PS_LIST_TAIL, "SHUTOUTC", 0, "Camera exposure shutter open (UTC, DB arg)", args->shutoutc); + psMetadataAddF32(header, PS_LIST_TAIL, "PLTSCALE_EXT", 0, "Plate scale at centroid", det->platescale); + psMetadataAddF32(header, PS_LIST_TAIL, "PLTSCALE", 0, "Plate scale at centroid", det->platescale); + psMetadataAddStr(header, PS_LIST_TAIL, "FPA.SHUTOUTC", 0, "Time of exposure open (CHIP mod)", det->fpashutoutc); + psMetadataAddStr(header, PS_LIST_TAIL, "FPA.SHUTCUTC", 0, "Time of exposure close (RAW)", det->fpashutcutc); + psMetadataAddStr(header, PS_LIST_TAIL, "PSREFCAT", 0, "Reference catalog used", det->refcat); + + //field in header that tells about the CMF version + char cmfVersion[16]; + ps_snprintf_nowarn(cmfVersion, 16, PS1_DV_FORMAT, args->version); + psMetadataAddStr(header, PS_LIST_TAIL, "CMFVERSION", 0, "CMF version", cmfVersion); + + // Find the total number of detections + long total = 0; + for (long i=0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det) { + continue; + } + total += det->num; + } + + psTrace("ppMops.write", 1, "Writing %ld rows to %s", total, args->output); + + if (total == 0) { + // Write dummy table + psMetadata *row = psMetadataAlloc(); // Output row + psMetadataAddF64(row, PS_LIST_TAIL, "RA", 0, "Right ascension (degrees)", NAN); + psMetadataAddF64(row, PS_LIST_TAIL, "RA_ERR", 0, "Right ascension error (degrees)", NAN); + psMetadataAddF64(row, PS_LIST_TAIL, "DEC", 0, "Declination (degrees)", NAN); + psMetadataAddF64(row, PS_LIST_TAIL, "DEC_ERR", 0, "Declination error (degrees)", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "MAG", 0, "Magnitude", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "MAG_ERR", 0, "Magnitude error", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "PSF_CHI2", 0, "chi^2 of PSF fit", NAN); + psMetadataAddS32(row, PS_LIST_TAIL, "PSF_DOF", 0, "Degrees of freedom of PSF fit", 0); + psMetadataAddF32(row, PS_LIST_TAIL, "CR_SIGNIFICANCE", 0, "Significance of CR", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "EXT_SIGNIFICANCE", 0, "Significance of extendedness", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MAJOR", 0, "PSF major axis (pixels)", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MINOR", 0, "PSF minor axis (pixels)", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "PSF_THETA", 0, "PSF position angle (deg on chip)", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "PSF_QUALITY", 0, "PSF quality factor", NAN); + psMetadataAddS32(row, PS_LIST_TAIL, "PSF_NPIX", 0, "Number of pixels in PSF", 0); + psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XX", 0, "xx moment", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XY", 0, "xy moment", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_YY", 0, "yy moment", NAN); + psMetadataAddU32(row, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", 0); + psMetadataAddS64(row, PS_LIST_TAIL, "DIFF_SKYFILE_ID", 0, "Identifier for diff skyfile", 0); + + psMetadataAddS32(row, PS_LIST_TAIL, "N_POS", 0, "Number of positive pixels", 0); + psMetadataAddF32(row, PS_LIST_TAIL, "F_POS", 0, "Fraction of positive pixels", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_BAD", 0, "Ratio of positive pixels to negative", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_MASK", 0, "Ratio of positive pixels to masked", NAN); + psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_ALL", 0, "Ratio of positive pixels to all", NAN); + + if (args->version == 2) { + // Write data of version 2 (see ICD) + psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET", PS_DATA_U32, "IPP detection identifier index", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX", PS_DATA_F32, "PSF fit instrumental magnitude", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_FLUX_SIG",PS_DATA_F32, "Sigma of PSF instrumental magnitude", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RAW", PS_DATA_F32, "magnitude in real aperture", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX", PS_DATA_F32, "instrumental flux in standard aperture", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "AP_FLUX_SIG", PS_DATA_F32, "aperture flux error", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG", PS_DATA_F32, "PSF Magnitude using supplied calibration", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG_SIG", PS_DATA_F32, "measured scatter of zero point calibration", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "SKY", PS_DATA_F32, "Sky level", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "SKY_SIGMA", PS_DATA_F32, "Sigma of sky level", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_QF_PERFECT", PS_DATA_F32, "PSF coverage/quality factor (poor)", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_R1", PS_DATA_F32, "first radial moment", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_RH", PS_DATA_F32, "half radial moment", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX", PS_DATA_F32, "Kron Flux (in 2.5 R1)", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_ERR", PS_DATA_F32, "Kron Flux Error", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_INNER", PS_DATA_F32, "Kron Flux (in 1.0 R1)", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_FLUX_OUTER", PS_DATA_F32, "Kron Flux (in 4.0 R1)", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_P", PS_DATA_F32, "distance to positive match source", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_P", PS_DATA_F32, "signal-to-noise of pos match src", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "DIFF_R_M", PS_DATA_F32, "distance to negative match source", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "DIFF_SN_M", PS_DATA_F32, "signal-to-noise of neg match src", + NAN); + psMetadataAdd (row, PS_LIST_TAIL, "FLAGS2", PS_DATA_U32, "psphot analysis flags (group 2)", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES", PS_DATA_U16, "Number of frames overlapping source center", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "PADDING", PS_DATA_S16, "padding", + 0); + if (args->version == 3) { +// Write data of version 3 (see ICD) + psMetadataAdd (row, PS_LIST_TAIL, "RA_EXT", PS_DATA_F32, "Fitted centroid RA", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "RA_EXT_SIGMA", PS_DATA_F32, "Fitted RA sigma", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "DEC_EXT", PS_DATA_F32, "Fitted centroid DEC", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "DEC_EXT_SIGMA", PS_DATA_F32, "Fitted DEC sigma", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "POSANG_EXT", PS_DATA_F32, "Fitted position angle", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "EXT_FLUX", PS_DATA_F32, "Fitted flux", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "EXT_CAL_MAG", PS_DATA_F32, "Calibrated mag", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "EXT_MAG_SIG", PS_DATA_F32, "Mag sigma", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "EXT_CHISQ", PS_DATA_F32, "Chi^2 of fit", + 0); + psMetadataAdd (row, PS_LIST_TAIL, "EXT_NDOF", PS_DATA_F32, "Fit degrees of freedom", + 0); + } + } + + if (!psFitsWriteTableEmpty(fits, header, row, OUT_EXTNAME)) { + psErrorStackPrint(stderr, "Unable to write empty table."); + psFree(header); + psFree(row); + return false; + } + psFree(row); + } else { + +#define addColumn(_outName, _inName, _convertTo32) \ + if (!addOutputColumn(table, detections, total, _outName, _inName, _convertTo32)) { \ + psError(PS_ERR_UNKNOWN, false, "Failed to add column %s", _outName); \ + return false; \ + } + + // Allocate the output table + psMetadata *table = psMetadataAlloc(); + addColumn("RA", "RA_PSF", 0); + addColumn("RA_ERR", NULL, 0); // calculated from various parameters including X_PSF_SIG Y_PSF_SIG and POSANG + addColumn("DEC", "DEC_PSF", 0); + addColumn("DEC_ERR", NULL, 0); // calculated from various parameters including X_PSF_SIG Y_PSF_SIG and POSANG + addColumn("MAG", "PSF_INST_MAG", 0); + addColumn("MAG_ERR", "PSF_INST_MAG_SIG", 0); + addColumn("PSF_CHI2", "PSF_CHISQ", 0); + addColumn("PSF_DOF", "PSF_NDOF", 1); + addColumn("CR_SIGNIFICANCE", "CR_NSIGMA", 0); + addColumn("EXT_SIGNIFICANCE", "EXT_NSIGMA", 0); + addColumn("PSF_MAJOR", NULL, 0); + addColumn("PSF_MINOR", NULL, 0); + addColumn("PSF_THETA", NULL, 0); + addColumn("PSF_QUALITY", "PSF_QF", 0); + addColumn("PSF_NPIX", NULL, 1); + addColumn("MOMENTS_XX", NULL, 0); + addColumn("MOMENTS_XY", NULL, 0); + addColumn("MOMENTS_YY", NULL, 0); + addColumn("N_POS", "DIFF_NPOS", 1); + addColumn("F_POS", "DIFF_FRATIO", 0); + addColumn("RATIO_BAD", "DIFF_NRATIO_BAD", 0); + addColumn("RATIO_MASK", "DIFF_NRATIO_MASK", 0); + addColumn("RATIO_ALL", "DIFF_NRATIO_ALL", 0); + addColumn("FLAGS", "FLAGS", 1); + addSkyfileIDColumn(table, detections, total, "DIFF_SKYFILE_ID"); + if (args->version >= 2) { + addColumn("IPP_IDET", NULL, 1); + addColumn("PSF_INST_FLUX", NULL, 0); + addColumn("PSF_INST_FLUX_SIG", NULL, 0); + addColumn("AP_MAG", NULL, 0); + addColumn("AP_MAG_RAW", NULL, 0); + addColumn("AP_MAG_RADIUS", NULL, 0); + addColumn("AP_FLUX", NULL, 0); + addColumn("AP_FLUX_SIG", NULL, 0); + addColumn("PEAK_FLUX_AS_MAG", NULL, 0); + addColumn("CAL_PSF_MAG", NULL, 0); + addColumn("CAL_PSF_MAG_SIG", NULL, 0); + addColumn("SKY", NULL, 0); + addColumn("SKY_SIGMA", NULL, 0); + addColumn("PSF_QF_PERFECT", NULL, 0); + addColumn("MOMENTS_R1", NULL, 0); + addColumn("MOMENTS_RH", NULL, 0); + addColumn("KRON_FLUX", NULL, 0); + addColumn("KRON_FLUX_ERR", NULL, 0); + addColumn("KRON_FLUX_INNER", NULL, 0); + addColumn("KRON_FLUX_OUTER", NULL, 0); + addColumn("DIFF_R_P", NULL, 0); + addColumn("DIFF_SN_P", NULL, 0); + addColumn("DIFF_R_M", NULL, 0); + addColumn("DIFF_SN_M", NULL, 0); + addColumn("FLAGS2", NULL, 1); + addColumn("IPP_IDET", NULL, 0); + addColumn("N_FRAMES", NULL, 0); + addColumn("PADDING", NULL, 0); + addColumn("X_EXT", NULL, 0); + addColumn("Y_EXT", NULL, 0); + addColumn("X_EXT_SIG", NULL, 0); + addColumn("Y_EXT_SIG", NULL, 0); + addColumn("EXT_INST_MAG", NULL, 0); + addColumn("EXT_INST_MAG_SIG", NULL, 0); + addColumn("NPARAMS", NULL, 0); + addColumn("EXT_WIDTH_MAJ", NULL, 0); + addColumn("EXT_WIDTH_MIN", NULL, 0); + addColumn("EXT_THETA", NULL, 0); + addColumn("EXT_WIDTH_MAJ_ERR", NULL, 0); + addColumn("EXT_WIDTH_MIN_ERR", NULL, 0); + addColumn("EXT_THETA_ERR", NULL, 0); + } + if (args->version >= 3) { + addColumn("RA_EXT", NULL, 0); + addColumn("RA_EXT_SIGMA", NULL, 0); + addColumn("DEC_EXT", NULL, 0); + addColumn("DEC_EXT_SIGMA", NULL, 0); + addColumn("POSANG_EXT", NULL, 0); + addColumn("EXT_FLUX", NULL, 0); + addColumn("EXT_CAL_MAG", NULL, 0); + addColumn("EXT_MAG_SIG", NULL, 0); + addColumn("EXT_CHISQ", NULL, 0); + addColumn("EXT_NDOF", NULL, 0); + } + + if (!psFitsWriteTableAllColumns(fits, header, table, OUT_EXTNAME)) { + psError(psErrorCodeLast(), false, "Unable to write table"); + return false; + } + psFree(table); + } + + psFree(header); + psFitsClose(fits); + + psTrace("ppMops.write", 1, "Done writing %ld rows to %s", det->num, args->output); + + return true; +} + +// extension parameter values: +// 0: SkyChip.psf +// 1: SkyChip.xfit +// Any other value is ignored +static bool addOutputColumn(psMetadata *table, const psArray *detections, long outputSize, char *outColumnName, char *inColumnName, bool convertTo32) +{ + if (inColumnName == NULL) { + inColumnName = outColumnName; + } + psVector *out = NULL; + if (convertTo32) { + // psFitsReadTableAllColumns reads columns of cfitsio type LONG and ULONG into a 64 bit integers + // We want to write 32 bits to the output. + int next = 0; + for (long i=0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det || det->num == 0) { + // no detections survived for this input + continue; + } + psVector *in = NULL; + in = psMetadataLookupVector(NULL, det->table, inColumnName); + if (!in) { + psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is true)", inColumnName); + return false; + } + if (in->type.type != PS_TYPE_S64 && in->type.type != PS_TYPE_U64) { + psError(PS_ERR_PROGRAMMING, true, "input column to convert is not S64 or U64: %s %d", + inColumnName, in->type.type); + return false; + } + if (out == NULL) { + // First time through set up the output vector and the copy parameters + if (in->type.type == PS_TYPE_S64) { + out = psVectorAlloc(outputSize, PS_TYPE_S32); + } else { + out = psVectorAlloc(outputSize, PS_TYPE_U32); + } + } + for (long d=0; d < det->num; d++) { + if (in->type.type == PS_TYPE_S64) { + out->data.S32[next++] = in->data.S64[d]; + } else { + out->data.U32[next++] = in->data.U64[d]; + } + } + } + } else { + void *next = NULL; + int elementSize = 0; // size of elements in vector... We are making assumptions here about the organization of primitives in memory so we can use memcopy + for (long i=0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det || det->num == 0) { + // no detections survived for this input + continue; + } + psVector *in = NULL; + in = psMetadataLookupVector(NULL, det->table, inColumnName); + if (!in) { + psError(PS_ERR_PROGRAMMING, true, "failed to find input column: %s (convertTo32 is false)", inColumnName); + out = psVectorAlloc(outputSize, PS_TYPE_F32); + psVectorInit(out, NAN); + psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out); + return false; + } + if (out == NULL) { + // First time through set up the output vector and the copy parameters + out = psVectorAlloc(outputSize, in->type.type); + next = (void *) out->data.U8; + switch (in->type.type) { + case PS_TYPE_S8: + elementSize = sizeof(psS8); + break; + case PS_TYPE_U8: + elementSize = sizeof(psU8); + break; + case PS_TYPE_S16: + elementSize = sizeof(psS16); + break; + case PS_TYPE_U16: + elementSize = sizeof(psU16); + break; + case PS_TYPE_S32: + elementSize = sizeof(psS32); + break; + case PS_TYPE_U32: + elementSize = sizeof(psU32); + break; + case PS_TYPE_S64: + elementSize = sizeof(psS64); + break; + case PS_TYPE_U64: + elementSize = sizeof(psU64); + break; + case PS_TYPE_F32: + elementSize = sizeof(psF32); + break; + case PS_TYPE_F64: + elementSize = sizeof(psF64); + break; + default: + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Unknown vector type %d", in->type.type); + return false; + } + } + // We are doing nasty things here so we can use memcpy. + // It would be safer to do a proper loop over the elements. + long toCopy = det->num * elementSize; + memcpy(next, in->data.U8, toCopy); + next += toCopy; + } + } + // Finally add the new column to the output table + psMetadataAddVector(table, PS_LIST_TAIL, outColumnName, 0, NULL, out); + psFree(out); // drop reference + return true; +} + +static bool addSkyfileIDColumn(psMetadata *table, const psArray *detections, long total, char *colName) +{ + psVector *out = psVectorAlloc(total, PS_TYPE_S64); + long next = 0; + for (long i = 0; in; i++) { + ppMopsDetections *det = detections->data[i]; + if (!det) { + continue; + } + psS64 diffSkyfileId = det->diffSkyfileId; + for (long j = 0; j < det->num; j++) { + out->data.S64[next++] = diffSkyfileId; + } + } + psMetadataAddVector(table, PS_LIST_TAIL, colName, 0, NULL, out); + psFree(out); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersion.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersion.c (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersion.c (revision 41112) @@ -0,0 +1,102 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppTranslateVersionDefinitions.h" + +#ifndef PPTRANSLATE_VERSION +#error "PPTRANSLATE_VERSION is not set" +#endif +#ifndef PPTRANSLATE_BRANCH +#error "PPTRANSLATE_BRANCH is not set" +#endif +#ifndef PPTRANSLATE_SOURCE +#error "PPTRANSLATE_SOURCE is not set" +#endif + +psString ppTranslateVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PPTRANSLATE_BRANCH, PPTRANSLATE_VERSION); + return value; +} + +psString ppTranslateSource(void) +{ + return psStringCopy(PPTRANSLATE_SOURCE); +} + +psString ppTranslateVersionLong(void) +{ + psString version = ppTranslateVersion(); // Version, to return + psString source = ppTranslateSource(); // Source + + psStringPrepend(&version, "ppTranslate "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + + return version; +}; + + +bool ppTranslateVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "ppTranslate at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + + psString version = ppTranslateVersion(); // Software version + psString source = ppTranslateSource(); // Software source + + psMetadataAddStr(header, PS_LIST_TAIL, "TRANSL_V", PS_META_REPLACE, NULL, PPTRANSLATE_VERSION); + + psStringPrepend(&version, "ppTranslate version: "); + psStringPrepend(&source, "ppTranslate source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + psFree(time); + + return true; +} + +void ppTranslateVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("ppTranslate", PS_LOG_INFO, "ppTranslate at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString ppTranslate = ppTranslateVersionLong(); // ppTranslate version + + psLogMsg("ppTranslate", PS_LOG_INFO, "%s", pslib); + psLogMsg("ppTranslate", PS_LOG_INFO, "%s", ppTranslate); + + psFree(pslib); + psFree(ppTranslate); + + return; +} Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersion.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersion.h (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersion.h (revision 41112) @@ -0,0 +1,21 @@ +#ifndef PP_TRANSLATE_VERSION +#define PP_TRANSLATE_VERSION + +#include + +/// Return version +psString ppTranslateVersion(void); + +/// Return source +psString ppTranslateSource(void); + +/// Return detailed version information +psString ppTranslateVersionLong(void); + +/// Put version into header +bool ppTranslateVersionHeader(psMetadata *header); + +/// Print version information +void ppTranslateVersionPrint(void); + +#endif Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersionDefinitions.h.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersionDefinitions.h.in (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/src/ppTranslateVersionDefinitions.h.in (revision 41112) @@ -0,0 +1,8 @@ +#ifndef PPTRANSLATE_VERSION_DEFINITIONS_H +#define PPTRANSLATE_VERSION_DEFINITIONS_H + +#define PPTRANSLATE_VERSION @PPTRANSLATE_VERSION@ // SVN version +#define PPTRANSLATE_BRANCH @PPTRANSLATE_BRANCH@ // SVN branch +#define PPTRANSLATE_SOURCE @PPTRANSLATE_SOURCE@ // SVN source + +#endif Index: /branches/ccl_branches/ipponly-20191108/ppTranslate/tests/integration/build_inputs_for_ppMops.sh =================================================================== --- /branches/ccl_branches/ipponly-20191108/ppTranslate/tests/integration/build_inputs_for_ppMops.sh (revision 41112) +++ /branches/ccl_branches/ipponly-20191108/ppTranslate/tests/integration/build_inputs_for_ppMops.sh (revision 41112) @@ -0,0 +1,28 @@ +#!/bin/bash + +# This script creates the pos and neg files that the ppMops program +# needs as input. +# +# Only one argument is required: the diff_id of the exposure +# +# e.g.: +# $0 308413 +# +DIFFID=$1 +if [ "$DIFFID" == "" ]; then + echo "$0 " + echo " Create the pos and neg files that the ppMops program" + echo " needs as input." + exit 1 +fi + +OUTPUTROOT=test.$DIFFID + +for f in `difftool -dbname gpc1 -diffskyfile -diff_id $DIFFID | grep path_base | sed 's/path_base//g' | sed 's/ STR //g' | tr -d ' '`; do + neb-ls -p $f.cmf >> $OUTPUTROOT.pos +done + +for f in `difftool -dbname gpc1 -diffskyfile -diff_id $DIFFID | grep path_base | sed 's/path_base//g' | sed 's/ STR //g' | tr -d ' '`; do + neb-ls -p $f.inv.cmf >> $OUTPUTROOT.neg +done +