Index: /branches/eam_branches/ppStack.20211015/Doxyfile.in =================================================================== --- /branches/eam_branches/ppStack.20211015/Doxyfile.in (revision 41840) +++ /branches/eam_branches/ppStack.20211015/Doxyfile.in (revision 41840) @@ -0,0 +1,1310 @@ +# Doxyfile 1.5.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file that +# follow. The default is UTF-8 which is also the encoding used for all text before +# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into +# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of +# possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = ppStack + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = ipp-2.7.dev + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, +# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, +# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, +# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is +# documented as struct with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code where the coding convention is that all structs are +# typedef'ed and only the typedef is referenced never the struct's name. + +TYPEDEF_HIDES_STRUCT = NO + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be extracted +# and appear in the documentation as a namespace called 'anonymous_namespace{file}', +# where file will be replaced with the base name of the file that contains the anonymous +# namespace. By default anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text " + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ./src + +# This tag can be used to specify the character encoding of the source files that +# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default +# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. +# See http://www.gnu.org/software/libiconv for the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = *.h + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the output. +# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, +# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH +# then you must also enable this option. If you don't then doxygen will produce +# a warning and turn it on anyway + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to +# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to +# specify the directory where the mscgen tool resides. If left empty the tool is assumed to +# be found in the default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the number +# of direct children of the root node in a graph is already larger than +# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO Index: /branches/eam_branches/ppStack.20211015/Makefile.am =================================================================== --- /branches/eam_branches/ppStack.20211015/Makefile.am (revision 41840) +++ /branches/eam_branches/ppStack.20211015/Makefile.am (revision 41840) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *~ core core.* Index: /branches/eam_branches/ppStack.20211015/autogen.sh =================================================================== --- /branches/eam_branches/ppStack.20211015/autogen.sh (revision 41840) +++ /branches/eam_branches/ppStack.20211015/autogen.sh (revision 41840) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppStack +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOLIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branches/ppStack.20211015/configure.ac =================================================================== --- /branches/eam_branches/ppStack.20211015/configure.ac (revision 41840) +++ /branches/eam_branches/ppStack.20211015/configure.ac (revision 41840) @@ -0,0 +1,42 @@ +AC_PREREQ(2.61) + +AC_INIT([ppStack], [0.1.1], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDLDFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) +PKG_CHECK_MODULES([PSPHOT], [psphot >= 0.8.0]) +PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +IPP_STDOPTS +IPP_STDCFLAGS + +IPP_VERSION + +AC_SUBST([PPSTACK_CFLAGS]) +AC_SUBST([PPSTACK_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile + Doxyfile +]) +AC_OUTPUT Index: /branches/eam_branches/ppStack.20211015/src/Makefile.am =================================================================== --- /branches/eam_branches/ppStack.20211015/src/Makefile.am (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/Makefile.am (revision 41840) @@ -0,0 +1,97 @@ +bin_PROGRAMS = ppStack ppStackMedian + +# Force recompilation of ppStackVersion.c, since it gets the version information +ppStackVersion.c: ppStackVersionDefinitions.h +ppStackVersionDefinitions.h: ppStackVersionDefinitions.h.in FORCE + pslib-setsvnversion.pl PPSTACK ppStackVersionDefinitions.h.in ppStackVersionDefinitions.h +FORCE: ; + +ppStack_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS) +ppStack_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PSPHOT_LIBS) $(PPSTATS_LIBS) $(PPSTACK_LIBS) + +ppStackMedian_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS) +ppStackMedian_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PSPHOT_LIBS) $(PPSTATS_LIBS) $(PPSTACK_LIBS) + +ppStack_SOURCES = \ + ppStack.c \ + ppStackArguments.c \ + ppStackCamera.c \ + ppStackFiles.c \ + ppStackLoop.c \ + ppStackPSF.c \ + ppStackReadout.c \ + ppStackVersion.c \ + ppStackMatch.c \ + ppStackSources.c \ + ppStackThread.c \ + ppStackOptions.c \ + ppStackSetup.c \ + ppStackPrepare.c \ + ppStackConvolve.c \ + ppStackCombinePrepare.c \ + ppStackCombineInitial.c \ + ppStackCombineAlternate.c \ + ppStackReject.c \ + ppStackCombineFinal.c \ + ppStackCleanup.c \ + ppStackPhotometry.c \ + ppStackFinish.c \ + ppStackTarget.c \ + ppStackUpdateHeader.c \ + ppStackJPEGs.c \ + ppStackStats.c \ + ppStackErrorCodes.c + +ppStackMedian_SOURCES = \ + ppStackMedian.c \ + ppStackArguments.c \ + ppStackCamera.c \ + ppStackFiles.c \ + ppStackMedianLoop.c \ + ppStackPSF.c \ + ppStackReadout.c \ + ppStackVersion.c \ + ppStackMatch.c \ + ppStackSources.c \ + ppStackThread.c \ + ppStackOptions.c \ + ppStackSetup.c \ + ppStackPrepare.c \ + ppStackConvolve.c \ + ppStackCombinePrepare.c \ + ppStackCombineInitial.c \ + ppStackCombineAlternate.c \ + ppStackReject.c \ + ppStackCombineFinal.c \ + ppStackCleanup.c \ + ppStackPhotometry.c \ + ppStackFinish.c \ + ppStackTarget.c \ + ppStackUpdateHeader.c \ + ppStackJPEGs.c \ + ppStackStats.c \ + ppStackErrorCodes.c + +noinst_HEADERS = \ + ppStack.h \ + ppStackLoop.h \ + ppStackOptions.h \ + ppStackThread.h \ + ppStackErrorCodes.h + +### Error codes. +BUILT_SOURCES = ppStackErrorCodes.h ppStackErrorCodes.c +CLEANFILES = ppStackErrorCodes.h ppStackErrorCodes.c + +ppStackErrorCodes.h : ppStackErrorCodes.dat ppStackErrorCodes.h.in + $(ERRORCODES) --data=ppStackErrorCodes.dat --outdir=. ppStackErrorCodes.h + +ppStackErrorCodes.c : ppStackErrorCodes.dat ppStackErrorCodes.c.in ppStackErrorCodes.h + $(ERRORCODES) --data=ppStackErrorCodes.dat --outdir=. ppStackErrorCodes.c + + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: /branches/eam_branches/ppStack.20211015/src/ppStack.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStack.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStack.c (revision 41840) @@ -0,0 +1,51 @@ +#include "ppStack.h" + +int main(int argc, char *argv[]) +{ + psLibInit(NULL); + psTimerStart("PPSTACK"); + psTimerStart("PPSTACK_STEPS"); + + pmErrorRegister(); + ppStackErrorRegister(); + psphotErrorRegister(); + + ppStackOptions *options = NULL; // Options for stacking + + pmConfig *config = pmConfigRead(&argc, argv, PPSTACK_RECIPE); // Configuration + if (!config) { + ppStackCleanup(config, options); + } + + ppStackVersionPrint(); + + if (!pmModelClassInit()) { + psError(PPSTACK_ERR_PROG, false, "Unable to initialise model classes."); + ppStackCleanup(config, options); + } + + if (!psphotInit()) { + psError(PPSTACK_ERR_PROG, false, "Error initialising psphot."); + ppStackCleanup(config, options); + } + + if (!ppStackArgumentsSetup(argc, argv, config)) { + ppStackCleanup(config, options); + } + + if (!ppStackCamera(config)) { + ppStackCleanup(config, options); + } + + if (!ppStackArgumentsParse(config)) { + ppStackCleanup(config, options); + } + + options = ppStackOptionsAlloc(); + if (!ppStackLoop(config, options)) { + ppStackCleanup(config, options); + } + + ppStackCleanup(config, options); +} + Index: /branches/eam_branches/ppStack.20211015/src/ppStack.h =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStack.h (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStack.h (revision 41840) @@ -0,0 +1,214 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#ifndef PPSTACK_H +#define PPSTACK_H + +#define PPSTACK_RECIPE "PPSTACK" // Name of the recipe +#define PPSTACK_INSPECT_PIXELS "PPSTACK.PIXELS" // Name of rejected pixels metadata items + +#include +#include + +// Mask values for inputs +typedef enum { + PPSTACK_MASK_NONE = 0x00, // Nothing wrong + PPSTACK_MASK_CAL = 0x01, // Photometric calibration failed + PPSTACK_MASK_PSF = 0x02, // PSF measurement failed + PPSTACK_MASK_MATCH = 0x04, // PSF-matching failed + PPSTACK_MASK_CHI2 = 0x08, // Chi^2 too deviant + PPSTACK_MASK_REJECT = 0x10, // Rejection failed + PPSTACK_MASK_BAD = 0x20, // Bad image (too many pixels rejected) + PPSTACK_MASK_ALL = 0xff // All errors +} ppStackMask; + +// List of files +typedef enum { + PPSTACK_FILES_NONE, // NOP list + PPSTACK_FILES_PREPARE, // Files for preparation + PPSTACK_FILES_TARGET, // Files for target generation + PPSTACK_FILES_CONVOLVE, // Files for convolution + PPSTACK_FILES_STACK, // Stack files + PPSTACK_FILES_UNCONV, // Unconvolved stack files + PPSTACK_FILES_PHOT, // Files for photometry + PPSTACK_FILES_BKG, // Files for bkg + PPSTACK_FILES_MEDIAN_IN, // Files for median only stacks. + PPSTACK_FILES_MEDIAN_OUT // Files for median only stacks. +} ppStackFileList; + +#include "ppStackOptions.h" +#include "ppStackThread.h" +#include "ppStackLoop.h" +#include "ppStackErrorCodes.h" + +// Setup command-line arguments +bool ppStackArgumentsSetup(int argc, char *argv[], // Command-line arguments + pmConfig *config // Configuration + ); + +// Parse command-line arguments +bool ppStackArgumentsParse(pmConfig *config // Configuration + ); + +// Parse cameras +bool ppStackCamera(pmConfig *config // Configuration + ); + +// Determine target PSF for input images +pmPSF *ppStackPSF(const pmConfig *config, // Configuration + int numCols, int numRows, // Size of image + const psArray *psfs, // List of input PSFs + ppStackOptions *options // full options including Mask for inputs + ); + +// Re-do photometry on input sources +// +// Photometry for the sources is replaced by what is measured +bool ppStackInputPhotometry(const pmReadout *ro, // Readout + const psArray *sources, // Sources to photometer + const pmConfig *config // Configuration + ); + +// Perform stacking on a readout +// +// Returns two arrays: pixels to inspect for each input image, and pixels to reject for each input image. +psArray *ppStackReadoutInitial(const pmConfig *config, // Configuration + pmReadout *outRO, // Output readout + const psArray *readouts, // Input readouts + const psVector *mask, // Mask for input readouts + const psVector *weightings, // Weighting factors for each image + const psVector *exposures, // Exposure time for each image + const psVector *addVariance // Additional variance for rejection + ); + +// Thread entry point for ppStackReadoutInitial +bool ppStackReadoutInitialThread(psThreadJob *job // Job to process + ); + +// Concatenate inspection lists for each input image +bool ppStackInspect(psThreadJob *job // Job to process + ); + +// Perform stacking on a readout +bool ppStackReadoutFinal(const pmConfig *config, // Configuration + pmReadout *outRO, // Output readout + pmReadout *expRO, // Exposure readout + const psArray *readouts, // Input readouts + const psVector *mask, // Mask for input readouts + const psArray *rejected, // Array with pixels rejected in each image + const psVector *weightings, // Weighting factors for each image + const psVector *exposures, // Exposure times for each image + const psVector *addVariance, // Additional variance for rejection + bool safety, // Enable safety switch? + const psVector *norm, // Normalisations to apply + const psVector *bscaleApplyOffset // hack for offset based on bscale + ); + +// Thread entry point for ppStackReadoutFinal +bool ppStackReadoutFinalThread(psThreadJob *job // Job to process + ); +// Perform median stacking for background +bool ppStackCombineBackground(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); +bool ppStackCombineMedian(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); +bool ppStackLinearScale(psArray *inputs, pmConfig *config); + +// Return software version +psString ppStackVersion(void); + +/// Return software source +psString ppStackSource(void); + +// Return long description of software version +psString ppStackVersionLong(void); + +// Supplement header with software version +bool ppStackVersionHeader(psMetadata *header // Header to supplement + ); + +/// Print version information +void ppStackVersionPrint(void); + +/// Generate target PSF image +psImage *ppStackTarget(ppStackOptions *options, // Options for stacking + pmConfig *config // Configuration + ); + +/// Convolve image to match specified seeing +bool ppStackMatch(pmReadout *readout, // Readout to be convolved; replaced with output + const psImage *target, // Target PSF image + ppStackOptions *options, // Options for stacking + int index, // Index of image to match + const pmConfig *config // Configuration + ); + + +/// Calculate transparency differences between images +/// +/// Corrects the source PSF photometry to a common system. Return the sum of the exposure times. +bool ppStackSourcesTransparency(ppStackOptions *options, // Stacking options + const pmFPAview *view, // View to readout + const pmConfig *config // Configuration + ); + +/// Dump memory debugging information +void ppStackMemDump( + const char *name ///< Stage name, for inclusion in the output file name + ); + +/// Activate/deactivate a list of files +void ppStackFileActivation( + pmConfig *config, // Configuration + ppStackFileList list, // Files to turn on/off + bool state // Activation state + ); + +// Activate/deactivate a single element for a list +void ppStackFileActivationSingle( + pmConfig *config, // Configuration + ppStackFileList list, // Files to turn on/off + bool state, // Activation state + int num // Number of file in sequence + ); + +/// Iterate down the hierarchy, loading files +/// +/// We can get away with this simplistic treatment of the FPA hierarchy because we're working on skycells. +pmFPAview *ppStackFilesIterateDown( + pmConfig *config // Configuration + ); + +/// Iterate up the hierarchy, writing files +/// +/// We can get away with this simplistic treatment of the FPA hierarchy because we're working on skycells. +bool ppStackFilesIterateUp( + pmConfig *config // Configuration + ); + +/// Write an image to a FITS file +bool ppStackWriteImage( + const char *name, // Name of image + psMetadata *header, // Header + const psImage *image, // Image + pmConfig *config // Configuration + ); + +bool ppStackWriteVariance(const char *name, // Name of image + psMetadata *header, // Header + const psImage *variance, // Variance + const psImage *covariance, // Variance + pmConfig *config // Configuration + ); + +/// Return an appropriate exit code based on the error code +psExit ppStackExitCode(psExit exitValue); + +bool ppStackCleanup(pmConfig *config, ppStackOptions *options) PS_ATTR_NORETURN; + +#endif Index: /branches/eam_branches/ppStack.20211015/src/ppStackArguments.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackArguments.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackArguments.c (revision 41840) @@ -0,0 +1,336 @@ +#include "ppStack.h" + +// XXX add in the version info as in ppImage + +// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config // Configuration + ) +{ + fprintf(stderr, "\nPan-STARRS Image combination\n\n"); + fprintf(stderr, + "Usage: %s -input INPUTS.mdc OUTPUT_ROOT [-sources STAMPS.cmf | -stamps STAMPS.dat]\n" + "where INPUTS.mdc contains various METADATAs, each with:\n" + "\tIMAGE(STR): Image filename\n" + "\tMASK(STR): Mask filename\n" + "\tVARIANCE(STR): Variance map filename\n" + "\tPSF(STR): PSF filename\n" + "\tSOURCES(STR): Sources filename\n", + program); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_CONFIG_ERROR); +} + +// Get a float-point value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_FLOAT(ARGNAME, RECIPENAME, TYPE) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, config->arguments, ARGNAME); \ + if (isnan(value)) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PPSTACK_ERR_CONFIG, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(recipe, PS_LIST_TAIL, RECIPENAME, PS_META_REPLACE, NULL, value); \ +} + +// Get an integer value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_INT(ARGNAME, RECIPENAME, TYPE, UNSET) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, config->arguments, ARGNAME); \ + if (value == UNSET) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PPSTACK_ERR_CONFIG, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(recipe, PS_LIST_TAIL, RECIPENAME, PS_META_REPLACE, NULL, value); \ +} + +// Get a mask value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_MASK(ARGNAME, RECIPENAME) { \ + bool mdok; \ + const char *name = psMetadataLookupStr(&mdok, config->arguments, ARGNAME); \ + if (!mdok || !name || strlen(name) == 0) { \ + name = psMetadataLookupStr(NULL, recipe, RECIPENAME); \ + if (!name) { \ + psError(PPSTACK_ERR_CONFIG, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psImageMaskType value = pmConfigMaskGet(name, config); \ + psMetadataAddImageMask(recipe, PS_LIST_TAIL, RECIPENAME, PS_META_REPLACE, NULL, value); \ +} + +// Get a string value from the command-line and add it to the target +static bool valueArgStr(psMetadata *arguments, // Command-line arguments + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata + psMetadata *target // Target metadata to which to add value + ) +{ + psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest + if (value && strlen(value) > 0) { + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, PS_META_REPLACE, NULL, value); + } + return false; +} + +// Get a string value from the command-line or recipe and add it to the target +static bool valueArgRecipeStr(psMetadata *arguments, // Command-line arguments + psMetadata *recipe, // Recipe + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata and recipe + psMetadata *target // Target metadata to which to add value + ) +{ + psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest + if (!value) { + value = psMetadataLookupStr(NULL, recipe, mdName); + if (!value) { + psError(PPSTACK_ERR_CONFIG, true, "Unable to find %s in recipe %s", + mdName, PPSTACK_RECIPE); + return false; + } + } + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, PS_META_REPLACE, NULL, value); +} + +bool ppStackArgumentsSetup(int argc, char *argv[], pmConfig *config) +{ + assert(config); + + // generic arguments (version, dumpconfig) + PS_ARGUMENTS_GENERIC( ppStack, config, argc, argv ); + + // thread arguments + PS_ARGUMENTS_THREADS( ppStack, config, argc, argv ) + + { + int argNum = psArgumentGet(argc, argv, "-debug"); // Debugging argument number + if (argNum) { + psArgumentRemove(argNum, &argc, argv); + pmSubtractionRegions(true); + } + } + + // This capability makes things much faster when debugging + bool debugStack = false; // Read old convolutions to debug the stacking? + int argNum = psArgumentGet(argc, argv, "-debug-stack"); // Argument number + if (argNum > 0) { + debugStack = true; + psArgumentRemove(argNum, &argc, argv); + } + + pmConfigFileSetsMD(config->arguments, &argc, argv, "PPSTACK.SOURCES", "-sources", NULL); + + if ((argNum = psArgumentGet(argc, argv, "-dumpconfig"))) { + psArgumentRemove(argNum, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "DUMP_CONFIG", PS_META_REPLACE, + "Filename for configuration dump", argv[argNum]); + psArgumentRemove(argNum, &argc, argv); + } + + psMetadata *arguments = config->arguments; // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stamps", 0, "Stamps file with x,y,flux per line", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Statistics file", NULL); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-combine-iter", 0, "Number of rejection iterations per input", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-combine-rej", 0, + "Combination rejection thresold (sigma)", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-combine-sys", 0, + "Relative systematic error in combination", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-combine-discard", 0, + "Discard fraction for Olympic weighted mean", NAN); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-val", 0, "Mask value of input bad pixels", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-bad", 0, "Mask value to give bad pixels", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-poor", 0, "Mask value to give poor pixels", NULL); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-threshold-mask", 0, "Threshold for mask deconvolution", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-poor-frac", 0, "Fraction of variance for poor pixels", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-deconv-limit", 0, "Maximum deconvolution fraction limit", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-image-rej", 0, + "Pixel rejection fraction threshold for rejecting entire image", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-rows", 0, "Rows to read at once", 0); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Do photometry on stacked image?", false); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-psf-instances", 0, + "Number of instances for PSF generation", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-psf-radius", 0, "Radius for PSF generation", NAN); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-psf-model", 0, "Model name for PSF generation", NULL); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-psf-order", 0, "Spatial order for PSF generation", 0); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-variance", 0, "Use variance for rejection?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-safe", 0, + "Play safe with small numbers of pixels to combine?", false); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-radius", 0, "Radius (pixels) for matching sources", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-iter-1", 0, "Maximum iterations for zero point; pass 1", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-iter-2", 0, "Maximum iterations for zero point; pass 2", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-tol", 0, "Tolerance for zero point iterations", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-trans-iter", 0, "Iterations for transparency determination", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-trans-rej", 0, "Rejection threshold for transparency determination", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-trans-thresh", 0, "Threshold for transparency determination", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-rej-1", 0, "Rejection threshold for stars; pass 1", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-rej-2", 0, "Rejection threshold for stars; pass 2", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-limit", 0, "Limit on star rejection fraction for successful iteration", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-sys-1", 0, "Estimated systematic error; pass 1", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-sys-2", 0, "Estimated systematic error; pass 2", NAN); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-dir", 0, "Directory for temporary images", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-image", 0, "Suffix for temporary images", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-mask", 0, "Suffix for temporary masks", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-variance", 0, "Suffix for temporary variance maps", NULL); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-temp-delete", 0, "Delete temporary files on completion?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "visualisation", false); + + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stack_id", 0, "stack ID", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-tess_id", 0, "tessellation ID", NULL); + + if (argc == 1) { + usage(argv[0], arguments, config); + } + + // stack-type : used to define the stack for PSPS + if ((argNum = psArgumentGet (argc, argv, "-stack-type"))) { + if (argc <= argNum+1) { + psErrorStackPrint(stderr, "Expected to see an argument for -stack-type"); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (argNum, &argc, argv); + if (strcasecmp(argv[argNum], "NIGHTLY_STACK") && strcasecmp(argv[argNum], "DEEP_STACK") && strcasecmp(argv[argNum], "IQ_STACK")) { + psErrorStackPrint(stderr, "Invalid option for -stack-type %s (must be one of NIGHTLY_STACK, DEEP_STACK, IQ_STACK)", argv[argNum]); + exit(PS_EXIT_CONFIG_ERROR); + } + psMetadataAddStr (arguments, PS_LIST_TAIL, "STACK_TYPE", PS_META_REPLACE, "Stack Type", argv[argNum]); + psArgumentRemove (argNum, &argc, argv); + } else { + psMetadataAddStr (arguments, PS_LIST_TAIL, "STACK_TYPE", PS_META_REPLACE, "Stack Type", "DEEP_STACK"); + } + + if ((argNum = psArgumentGet(argc, argv, "-input"))) { + psArgumentRemove(argNum, &argc, argv); + if (argNum >= argc) { + usage(argv[0], arguments, config); + } + + unsigned int numBad = 0; // Number of bad lines + psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[argNum], false); // Input file info + if (!inputs || numBad > 0) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Unable to cleanly read MDC file with inputs."); + return false; + } + psMetadataAddMetadata(arguments, PS_LIST_TAIL, "INPUTS", 0, "Metadata with input details", inputs); + psFree(inputs); + + psArgumentRemove(argNum, &argc, argv); + } + + if (!psArgumentParse(arguments, &argc, argv) || argc != 2) { + usage(argv[0], arguments, config); + } + + psMetadataAddStr(arguments, PS_LIST_TAIL, "OUTPUT", 0, "Root name of the output image list", argv[1]); + + const char *stampsName = psMetadataLookupStr(NULL, arguments, "-stamps"); // Name of stamps file + psMetadataAddStr(arguments, PS_LIST_TAIL, "STAMPS", 0, "Stamps file", stampsName); + + valueArgStr(arguments, "-stats", "STATS", arguments); + + psMetadataAddBool(arguments, PS_LIST_TAIL, "PPSTACK.DEBUG.STACK", 0, + "Read old convolved images to debug stack?", debugStack); + + return true; +} + +bool ppStackArgumentsParse(pmConfig *config) +{ + assert(config); + + psMetadata *arguments = config->arguments; // Command-line arguments + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe + if (!recipe) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find recipe %s", PPSTACK_RECIPE); + goto ERROR; + } + + VALUE_ARG_RECIPE_FLOAT("-combine-iter", "COMBINE.ITER", F32); + VALUE_ARG_RECIPE_FLOAT("-combine-rej", "COMBINE.REJ", F32); + VALUE_ARG_RECIPE_FLOAT("-combine-sys", "COMBINE.SYS", F32); + VALUE_ARG_RECIPE_FLOAT("-combine-discard", "COMBINE.DISCARD", F32); + VALUE_ARG_RECIPE_FLOAT("-threshold-mask", "THRESHOLD.MASK", F32); + VALUE_ARG_RECIPE_FLOAT("-image-rej", "IMAGE.REJ", F32); + VALUE_ARG_RECIPE_FLOAT("-deconv-limit", "DECONV.LIMIT", F32); + VALUE_ARG_RECIPE_INT("-rows", "ROWS", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-poor-frac", "POOR.FRACTION", F32); + + valueArgRecipeStr(arguments, recipe, "-mask-val", "MASK.VAL", recipe); + valueArgRecipeStr(arguments, recipe, "-mask-bad", "MASK.BAD", recipe); + valueArgRecipeStr(arguments, recipe, "-mask-poor", "MASK.POOR", recipe); + + VALUE_ARG_RECIPE_FLOAT("-zp-radius", "ZP.RADIUS", F32); + VALUE_ARG_RECIPE_INT("-zp-iter-1", "ZP.ITER.1", S32, 0); + VALUE_ARG_RECIPE_INT("-zp-iter-2", "ZP.ITER.2", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-zp-tol", "ZP.TOL", F32); + VALUE_ARG_RECIPE_INT("-zp-trans-iter", "ZP.TRANS.ITER", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-zp-trans-rej", "ZP.TRANS.REJ", F32); + VALUE_ARG_RECIPE_FLOAT("-zp-trans-thresh", "ZP.TRANS.THRESH", F32); + VALUE_ARG_RECIPE_FLOAT("-zp-star-rej-1", "ZP.STAR.REJ.1", F32); + VALUE_ARG_RECIPE_FLOAT("-zp-star-rej-2", "ZP.STAR.REJ.2", F32); + VALUE_ARG_RECIPE_FLOAT("-zp-star-limit", "ZP.STAR.LIMIT", F32); + VALUE_ARG_RECIPE_FLOAT("-zp-star-sys-1", "ZP.STAR.SYS.1", F32); + VALUE_ARG_RECIPE_FLOAT("-zp-star-sys-2", "ZP.STAR.SYS.2", F32); + + VALUE_ARG_RECIPE_INT("-psf-instances", "PSF.INSTANCES", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-psf-radius", "PSF.RADIUS", F32); + VALUE_ARG_RECIPE_INT("-psf-order", "PSF.ORDER", S32, 0); + valueArgRecipeStr(arguments, recipe, "-psf-model", "PSF.MODEL", recipe); + + if (psMetadataLookupBool(NULL, arguments, "-visual")) { + pmVisualSetVisual(true); + } + + if (psMetadataLookupBool(NULL, arguments, "-photometry") || + psMetadataLookupBool(NULL, recipe, "PHOTOMETRY")) { + psMetadataAddBool(recipe, PS_LIST_TAIL, "PHOTOMETRY", PS_META_REPLACE, + "Do photometry on stacked image?", true); + } + + if (psMetadataLookupBool(NULL, arguments, "-variance") || + psMetadataLookupBool(NULL, recipe, "VARIANCE")) { + psMetadataAddBool(arguments, PS_LIST_TAIL, "VARIANCE", 0, "Use variance for rejection?", true); + } + + if (psMetadataLookupBool(NULL, arguments, "-safe") || + psMetadataLookupBool(NULL, recipe, "SAFE")) { + psMetadataAddBool(arguments, PS_LIST_TAIL, "SAFE", 0, + "Play safe with small number of pixels to combine?", true); + } + + valueArgRecipeStr(arguments, recipe, "-temp-image", "TEMP.IMAGE", recipe); + valueArgRecipeStr(arguments, recipe, "-temp-mask", "TEMP.MASK", recipe); + valueArgRecipeStr(arguments, recipe, "-temp-variance", "TEMP.VARIANCE", recipe); + + if (psMetadataLookupBool(NULL, arguments, "-temp-delete") || + psMetadataLookupBool(NULL, recipe, "TEMP.DELETE")) { + psMetadataAddBool(arguments, PS_LIST_TAIL, "TEMP.DELETE", 0, + "Delete temporary files on completion?", true); + } + + + psTrace("ppStack", 1, "Done parsing arguments\n"); + + pmConfigCamerasCull(config, NULL); + pmConfigRecipesCull(config, "PPSTACK,PPSUB,PPSTATS,PSPHOT,PSASTRO,MASKS,JPEG"); + + return true; + +ERROR: + return false; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackCamera.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackCamera.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackCamera.c (revision 41840) @@ -0,0 +1,560 @@ +#include "ppStack.h" + +// Define a file +static pmFPAfile *defineFile(pmConfig *config, // Configuration + pmFPAfile *bind, // File to which to bind + const char *name, // Name of file rule + const char *filename, // Name of file + pmFPAfileType type // Type of file + ) +{ + + psArray *dummy = psArrayAlloc(1); // Dummy array of filenames for this FPA + dummy->data[0] = psStringCopy(filename); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "FILENAMES", PS_META_REPLACE, + "Filenames for file rule definition", dummy); + psFree(dummy); + + bool found = false; // Found the file? + pmFPAfile *file = bind ? pmFPAfileBindFromArgs(&found, bind, config, name, "FILENAMES") : + pmFPAfileDefineFromArgs(&found, config, name, "FILENAMES"); + if (!file || !found) { + psError(psErrorCodeLast(), false, "Unable to define file %s from %s", name, filename); + return NULL; + } + if (file->type != type) { + psError(PS_ERR_IO, PPSTACK_ERR_CONFIG, "%s is not of type %s", name, pmFPAfileStringFromType(type)); + return NULL; + } + + return file; +} + +pmConfig *pmConfigMakeTemp (pmConfig *config) { + pmConfig *altconfig = pmConfigAlloc(); + + // these are NULL on pmConfigAlloc + altconfig->user = psMemIncrRefCounter(config->user); // inherit from primary camera + altconfig->site = psMemIncrRefCounter(config->site); // inherit from primary camera + altconfig->system = psMemIncrRefCounter(config->system); // inherit from primary camera + + psFree (altconfig->files); + altconfig->files = psMemIncrRefCounter(config->files); // inherit from primary camera + + psFree (altconfig->arguments); + altconfig->arguments = psMemIncrRefCounter(config->arguments); // inherit from primary camera + + psFree (altconfig->recipes); + altconfig->recipes = psMetadataCopy(NULL, config->recipes); // container for camera-specific recipe values (to be dropped) + + return (altconfig); +} + +bool ppStackCamera(pmConfig *config) +{ + int num = 0; // Number of inputs + bool haveVariances = false; // Do we have variance maps? + bool havePSFs = false; // Do we have PSFs? + + bool status = false; // Status of file definition + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe for ppSim + if (!recipe) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find recipe %s", PPSTACK_RECIPE); + return false; + } + bool convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images before stack? + + psArray *runImages = pmFPAfileDefineMultipleFromRun(&status, NULL, config, "PPSTACK.INPUT"); // Input images from previous run + if (runImages) { + // Defining files from the RUN metadata + num = runImages->n; + + psArray *runMasks = pmFPAfileDefineMultipleFromRun(&status, runImages, config, "PPSTACK.INPUT.MASK"); // Input masks + if (!status) { + psError(psErrorCodeLast(), false, "Unable to define input masks from RUN metadata."); + psFree(runImages); + return false; + } + psFree(runMasks); + + psArray *runVars = pmFPAfileDefineMultipleFromRun(&status, runImages, config, "PPSTACK.INPUT.VARIANCE"); // Input variances + if (!status) { + psError(psErrorCodeLast(), false, "Unable to define input variances from RUN metadata."); + psFree(runImages); + return false; + } + if (runVars) { + haveVariances = true; + } + psFree(runVars); + + psArray *runSrc = pmFPAfileDefineMultipleFromRun(&status, runImages, config, "PPSTACK.INPUT.SOURCES"); // Input sources + if (!status) { + psError(psErrorCodeLast(), false, "Unable to define input sources from RUN metadata."); + psFree(runImages); + return false; + } + if (!runSrc) { + psError(PPSTACK_ERR_CONFIG, true, "Unable to define input sources from RUN metadata."); + psFree(runImages); + return false; + } + psFree(runSrc); + + if (convolve) { + { + psArray *runPSF = pmFPAfileDefineMultipleFromRun(&status, runImages, config, "PPSTACK.INPUT.PSF"); // Input PSFs + if (!status) { + psError(psErrorCodeLast(), false, "Unable to define input PSFs from RUN metadata."); + psFree(runImages); + return false; + } + if (runPSF) { + havePSFs = true; + } + psFree(runPSF); + } + { + + psArray *runKernel = pmFPAfileDefineMultipleFromRun(&status, runImages, config, "PPSTACK.CONV.KERNEL"); // Conv'n kernels + if (!status) { + psError(psErrorCodeLast(), false, + "Unable to define convolution kernels from RUN metadata."); + psFree(runImages); + return false; + } + if (!runKernel) { + psError(PPSTACK_ERR_CONFIG, true, + "Unable to define convolution kernels from RUN metadata."); + psFree(runImages); + return false; + } + psFree(runKernel); + } + } + + psFree(runImages); + } else { + // Defining files from the input metadata + psMetadata *inputs = psMetadataLookupMetadata(NULL, config->arguments, "INPUTS"); // The inputs info + if (!inputs) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Unable to find inputs."); + return false; + } + psMetadataIterator *iter = psMetadataIteratorAlloc(inputs, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + int i = 0; // Counter + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_METADATA) { + psError(PPSTACK_ERR_ARGUMENTS, true, + "Component %s of the input metadata is not of type METADATA", item->name); + psFree(iter); + return false; + } + + psMetadata *input = item->data.md; // The input metadata of interest + + psString image = psMetadataLookupStr(NULL, input, "IMAGE"); // Name of image + if (!image || strlen(image) == 0) { + psError(PPSTACK_ERR_ARGUMENTS, true, "Component %s lacks IMAGE of type STR", item->name); + psFree(iter); + return false; + } + + bool mdok; + psString mask = psMetadataLookupStr(&mdok, input, "MASK"); // Name of mask + psString variance = psMetadataLookupStr(&mdok, input, "VARIANCE"); // Name of variance map + psString psf = psMetadataLookupStr(&mdok, input, "PSF"); // Name of PSF + psString sources = psMetadataLookupStr(&mdok, input, "SOURCES"); // Name of sources + psString bkgmodel = psMetadataLookupStr(&mdok, input, "BKGMODEL"); // Name of warped background model + + // Use a temporary config for all but the first image, keeping the main user, + // site, system, files, arguments entries loaded for the first image. This + // allows the images to be from different cameras than the first input image. + // NOTE: there is no check that these images match in terms of size, pixel + // scale, etc. That is up to the user. + + pmConfig *tempConfig = (i == 0) ? psMemIncrRefCounter(config) : pmConfigMakeTemp(config); + + pmFPAfile *imageFile = defineFile(tempConfig, NULL, "PPSTACK.INPUT", + image, PM_FPA_FILE_IMAGE); // File for image + if (!imageFile) { + psError(psErrorCodeLast(), false, "Unable to define file from image %d (%s)", i, image); + return false; + } + + if (mask && strlen(mask) > 0 && + !defineFile(tempConfig, imageFile, "PPSTACK.INPUT.MASK", mask, PM_FPA_FILE_MASK)) { + psError(psErrorCodeLast(), false, "Unable to define file from mask %d (%s)", i, mask); + return false; + } + + if (variance && strlen(variance) > 0) { + haveVariances = true; + if (!defineFile(tempConfig, imageFile, "PPSTACK.INPUT.VARIANCE", variance, + PM_FPA_FILE_VARIANCE)) { + psError(psErrorCodeLast(), false, + "Unable to define file from variance %d (%s)", i, variance); + return false; + } + } + + if (psf && strlen(psf) > 0) { + if (i != 0 && !havePSFs) { + psWarning("PSF not provided for all inputs --- ignoring."); + } else { + havePSFs = true; + if (!defineFile(tempConfig, imageFile, "PPSTACK.INPUT.PSF", psf, PM_FPA_FILE_PSF)) { + psError(psErrorCodeLast(), false, "Unable to define file from psf %d (%s)", i, psf); + return false; + } + } + } else if (havePSFs) { + psError(PPSTACK_ERR_CONFIG, true, "Unable to find PSF %d", i); + return false; + } + + if (!sources || strlen(sources) == 0) { + psError(PPSTACK_ERR_CONFIG, true, "SOURCES not provided for file %d", i); + return false; + } + if (!defineFile(tempConfig, imageFile, "PPSTACK.INPUT.SOURCES", sources, PM_FPA_FILE_CMF)) { + psError(psErrorCodeLast(), false, "Unable to define file from sources %d (%s)", + i, sources); + return false; + } + + if (convolve) { + pmFPAfile *kernel = pmFPAfileDefineOutput(tempConfig, imageFile->fpa, "PPSTACK.CONV.KERNEL"); + if (!kernel) { + psError(psErrorCodeLast(), false, + "Unable to generate output file from PPSTACK.CONV.KERNEL"); + return false; + } + kernel->save = true; + } + + // Grab bkgmodel information here + if ((!bkgmodel) || (strlen(bkgmodel) == 0)) { + // We have no background models. + psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE, "Do photometry on stacked image?", false); + } else { + pmFPAfile *inputBKG = defineFile(tempConfig,NULL,"PPSTACK.INPUT.BKGMODEL",bkgmodel, PM_FPA_FILE_IMAGE); + if (!inputBKG) { + psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE, "Do photometry on stacked image?", false); + +#if (0) + psError(psErrorCodeLast(), false, + "Unable to define file from bkgmodel %d (%s)",i,bkgmodel); + return(false); +#endif + } + } // End bkgmodel + psFree(tempConfig); + i++; + } + psFree(iter); + psMetadataRemoveKey(config->arguments, "FILENAMES"); + num = i; + } + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "INPUTS.NUM", 0, "Number of input files", num); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "HAVE.PSF", 0, "Have PSFs available?", havePSFs); + + // Output image + pmFPA *outFPA = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain the output + if (!outFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *output = pmFPAfileDefineOutput(config, outFPA, "PPSTACK.OUTPUT"); + psFree(outFPA); // Drop reference + if (!output) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT")); + return false; + } + if (output->type != PM_FPA_FILE_IMAGE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT is not of type IMAGE"); + return false; + } + output->save = true; + + if (!pmFPAAddSourceFromFormat(outFPA, output->format)) { + psError(psErrorCodeLast(), false, "Unable to generate output FPA."); + return false; + } + + // Output mask + pmFPAfile *outMask = pmFPAfileDefineOutput(config, output->fpa, "PPSTACK.OUTPUT.MASK"); + if (!outMask) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.MASK")); + return false; + } + if (outMask->type != PM_FPA_FILE_MASK) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.MASK is not of type MASK"); + return false; + } + outMask->save = true; + + // Output variance + if (haveVariances) { + pmFPAfile *outVariance = pmFPAfileDefineOutput(config, output->fpa, "PPSTACK.OUTPUT.VARIANCE"); + if (!outVariance) { + psError(psErrorCodeLast(), false, "Unable to generate output file from PPSTACK.OUTPUT.VARIANCE"); + return false; + } + if (outVariance->type != PM_FPA_FILE_VARIANCE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.VARIANCE is not of type VARIANCE"); + return false; + } + outVariance->save = true; + } + + + // Exposure image + pmFPA *expFPA = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain the output + if (!expFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *exp = pmFPAfileDefineOutput(config, expFPA, "PPSTACK.OUTPUT.EXP"); + psFree(expFPA); // Drop reference + if (!exp) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.EXP")); + return false; + } + if (exp->type != PM_FPA_FILE_IMAGE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.EXP is not of type IMAGE"); + return false; + } + exp->save = true; + + if (!pmFPAAddSourceFromFormat(expFPA, exp->format)) { + psError(psErrorCodeLast(), false, "Unable to generate output FPA."); + return false; + } + + // Exposure numbers + pmFPAfile *expNum = pmFPAfileDefineOutput(config, exp->fpa, "PPSTACK.OUTPUT.EXPNUM"); + if (!expNum) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.EXPNUM")); + return false; + } + if (expNum->type != PM_FPA_FILE_MASK) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.EXPNUM is not of type MASK"); + return false; + } + expNum->save = true; + + // Weighted exposure + pmFPAfile *expWt = pmFPAfileDefineOutput(config, exp->fpa, "PPSTACK.OUTPUT.EXPWT"); + if (!expWt) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.EXPWT")); + return false; + } + if (expWt->type != PM_FPA_FILE_VARIANCE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.EXPWT is not of type VARIANCE"); + return false; + } + expWt->save = true; + + + if (havePSFs) { + pmFPA *psfFPA = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain PSF + if (!psfFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *targetPSF = pmFPAfileDefineOutput(config, psfFPA, "PPSTACK.TARGET.PSF"); + if (!targetPSF) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.TARGET.PSF")); + return false; + } + psFree(psfFPA); + if (targetPSF->type != PM_FPA_FILE_PSF) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.TARGET.PSF is not of type PSF"); + return false; + } + targetPSF->save = true; + } + + // Unconvolved stack + pmFPA *unconvFPA = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain unconvolved output + if (!unconvFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *unConv = pmFPAfileDefineOutput(config, unconvFPA, "PPSTACK.UNCONV"); + psFree(unconvFPA); // Drop reference + if (!unConv) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.UNCONV")); + return false; + } + if (unConv->type != PM_FPA_FILE_IMAGE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.UNCONV is not of type IMAGE"); + return false; + } + unConv->save = true; + + if (!pmFPAAddSourceFromFormat(unconvFPA, unConv->format)) { + psError(psErrorCodeLast(), false, "Unable to generate output FPA."); + return false; + } + + // Unconvolved mask + pmFPAfile *unconvMask = pmFPAfileDefineOutput(config, unconvFPA, "PPSTACK.UNCONV.MASK"); + if (!unconvMask) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.UNCONV.MASK")); + return false; + } + if (unconvMask->type != PM_FPA_FILE_MASK) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.UNCONV.MASK is not of type MASK"); + return false; + } + unconvMask->save = true; + + // Unconvolved variance + if (haveVariances) { + pmFPAfile *unconvVariance = pmFPAfileDefineOutput(config, unconvFPA, "PPSTACK.UNCONV.VARIANCE"); + if (!unconvVariance) { + psError(psErrorCodeLast(), false, "Unable to generate output file from PPSTACK.UNCONV.VARIANCE"); + return false; + } + if (unconvVariance->type != PM_FPA_FILE_VARIANCE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.UNCONV.VARIANCE is not of type VARIANCE"); + return false; + } + unconvVariance->save = true; + } + + + // Exposure image + pmFPA *unconvExpFPA = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain the output + if (!unconvExpFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *unconvExp = pmFPAfileDefineOutput(config, unconvExpFPA, "PPSTACK.UNCONV.EXP"); + psFree(unconvExpFPA); // Drop reference + if (!unconvExp) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.UNCONV.EXP")); + return false; + } + if (unconvExp->type != PM_FPA_FILE_IMAGE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.UNCONV.EXP is not of type IMAGE"); + return false; + } + unconvExp->save = true; + + if (!pmFPAAddSourceFromFormat(unconvExpFPA, unconvExp->format)) { + psError(psErrorCodeLast(), false, "Unable to generate output FPA."); + return false; + } + + // Exposure numbers + pmFPAfile *unconvExpNum = pmFPAfileDefineOutput(config, unconvExp->fpa, "PPSTACK.UNCONV.EXPNUM"); + if (!unconvExpNum) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.UNCONV.MASK")); + return false; + } + if (unconvExpNum->type != PM_FPA_FILE_MASK) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.UNCONV.EXPNUM is not of type MASK"); + return false; + } + unconvExpNum->save = true; + + // Weighted exposure + pmFPAfile *unconvExpWt = pmFPAfileDefineOutput(config, unconvExp->fpa, "PPSTACK.UNCONV.EXPWT"); + if (!unconvExpWt) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.UNCONV.EXPWT")); + return false; + } + if (unconvExpWt->type != PM_FPA_FILE_VARIANCE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.UNCONV.EXPWT is not of type VARIANCE"); + return false; + } + unconvExpWt->save = true; + + // Output JPEGs + pmFPAfile *jpeg1 = pmFPAfileDefineOutput(config, NULL, "PPSTACK.OUTPUT.JPEG1"); + if (!jpeg1) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.JPEG1")); + return false; + } + if (jpeg1->type != PM_FPA_FILE_JPEG) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.JPEG1 is not of type JPEG"); + return false; + } + jpeg1->save = true; + pmFPAfile *jpeg2 = pmFPAfileDefineOutput(config, NULL, "PPSTACK.OUTPUT.JPEG2"); + if (!jpeg2) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.JPEG2")); + return false; + } + if (jpeg2->type != PM_FPA_FILE_JPEG) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.JPEG2 is not of type JPEG"); + return false; + } + jpeg2->save = true; + + // Output background + pmFPAfile *outBkg = pmFPAfileDefineOutput(config,NULL,"PPSTACK.OUTPUT.BKGMODEL"); + if (!outBkg) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.BKGMODEL")); + return(false); + } + if (outBkg->type != PM_FPA_FILE_IMAGE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.BKGMODEL is not of type IMAGE"); + return(false); + } + outBkg->save = true; + pmFPAfile *outBkgRest = pmFPAfileDefineOutput(config,NULL,"PPSTACK.OUTPUT.BKGREST"); + if (!outBkgRest) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PPSTACK.OUTPUT.BKGREST")); + return(false); + } + if (outBkgRest->type != PM_FPA_FILE_IMAGE) { + psError(PPSTACK_ERR_CONFIG, true, "PPSTACK.OUTPUT.BKGREST is not of type IMAGE"); + return(false); + } + + // For photometry, we operate on the chip-mosaicked image + // we create a copy of the mosaicked image for psphot so we can write out a clean image + bool mdok = false; + bool doPhotom = psMetadataLookupBool(&mdok, recipe, "PHOTOMETRY") || + psMetadataLookupBool(&mdok, config->arguments, "-photometry"); // perform photometry + if (doPhotom) { + // This pmFPAfile, PSPHOT.INPUT, is just used as a carrier; output files (eg, + // PSPHOT.RESID) are defined by psphotDefineFiles + pmFPAfile *psphotInput = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT"); + if (!psphotInput) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PSPHOT.INPUT")); + return false; + } + // specify the number of psphot input images + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", 1); + + // Define associated psphot input/output files + if (!psphotDefineFiles(config, psphotInput)) { + psError(psErrorCodeLast(), false, + "Trouble defining the additional input/output files for psphot"); + return false; + } + } else { + // Output PSF --- only required if photometry is not being performed + pmFPAfile *outPSF = pmFPAfileDefineOutputFromFile(config, output, "PSPHOT.PSF.SAVE"); + if (!outPSF) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PSPHOT.PSF.SAVE")); + return false; + } + if (outPSF->type != PM_FPA_FILE_PSF) { + psError(PPSTACK_ERR_CONFIG, true, "PSPHOT.PSF.SAVE is not of type PSF"); + return false; + } + outPSF->save = true; + } + + // Define output file here. + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackCleanup.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackCleanup.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackCleanup.c (revision 41840) @@ -0,0 +1,145 @@ +#include "ppStack.h" + +// ppStackCleanupFiles +bool ppStackCleanupFiles(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config, ppStackFileList stackFiles, ppStackFileList photFiles, bool closeJPEGs) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + if (!ppStackFilesIterateUp(config)) { + psError(psErrorCodeLast(), false, "Unable to close files."); + return false; + } + ppStackFileActivation(config, stackFiles, false); + ppStackFileActivation(config, photFiles, false); + + // Ensure files are freed + options->outRO->data_exists = false; + options->outRO->parent->data_exists = false; + options->outRO->parent->parent->data_exists = false; + psFree(options->outRO); + options->outRO = NULL; + + if (options->expRO) { + options->expRO->data_exists = false; + if (options->expRO->parent) { + options->expRO->parent->data_exists = false; + options->expRO->parent->parent->data_exists = false; + } + psFree(options->expRO); + options->expRO = NULL; + } + + if (options->bkgRO) { + options->bkgRO->data_exists = false; + if (options->bkgRO->parent) { + options->bkgRO->parent->data_exists = false; + options->bkgRO->parent->parent->data_exists = false; + } + psFree(options->bkgRO); + options->bkgRO = NULL; + } + + for (int i = 0; i < options->num; i++) { + pmCellFreeData(options->cells->data[i]); + } + + if (closeJPEGs) { + // XXX move these close / free operations to the jpeg creation function + pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy + view->chip = view->cell = 0; // pmFPAviewFreeData doesn't want to deal with readouts + pmFPAfile *jpeg1 = pmFPAfileSelectSingle(config->files, "PPSTACK.OUTPUT.JPEG1", 0); // JPEG file + pmFPAviewFreeData(view, jpeg1); + pmFPAfile *jpeg2 = pmFPAfileSelectSingle(config->files, "PPSTACK.OUTPUT.JPEG2", 0); // JPEG file + pmFPAviewFreeData(view, jpeg2); + pmFPAfile *phot = NULL; // Photometry file + if (options->photometry) { + phot = pmFPAfileSelectSingle(config->files, "PSPHOT.INPUT", 0); // Photometry file + pmFPAviewFreeData(view, phot); + } + + view->readout = 0; + pmReadout *ro1 = pmFPAviewThisReadout(view, jpeg1->fpa); // JPEG readout + ro1->data_exists = ro1->parent->data_exists = ro1->parent->parent->data_exists = false; + pmReadout *ro2 = pmFPAviewThisReadout(view, jpeg2->fpa); // JPEG readout + ro2->data_exists = ro2->parent->data_exists = ro2->parent->parent->data_exists = false; + if (options->photometry) { + pmReadout *ro = pmFPAviewThisReadout(view, phot->fpa); // Photometry readout + ro->data_exists = ro->parent->data_exists = ro->parent->parent->data_exists = false; + } + psFree(view); + } + + return true; +} + +bool ppStackCleanup (pmConfig *config, ppStackOptions *options) { + psExit exitValue = ppStackExitCode(PS_EXIT_SUCCESS); // Exit code + + // Ensure everything closes + if (config) { + ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true); + ppStackFileActivation(config, PPSTACK_FILES_CONVOLVE, true); + ppStackFileActivation(config, PPSTACK_FILES_STACK, true); + ppStackFileActivation(config, PPSTACK_FILES_UNCONV, true); + ppStackFileActivation(config, PPSTACK_FILES_PHOT, true); + ppStackFileActivation(config, PPSTACK_FILES_MEDIAN_IN, true); + ppStackFileActivation(config, PPSTACK_FILES_MEDIAN_OUT, true); + if (!ppStackFilesIterateUp(config)) { + psError(psErrorCodeLast(), false, "Unable to close files."); + exitValue = ppStackExitCode(exitValue); + pmFPAfileFreeSetStrict(false); + } + } + + // Write out summary statistics + if (options && options->stats) { + + psMetadataAddS32(options->stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "Bad data quality flag", options->quality); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_STACK", 0, "Total time", psTimerClear("PPSTACK_TOTAL")); + + const char *statsMDC = psMetadataConfigFormat(options->stats); + if (!statsMDC || strlen(statsMDC) == 0) { + psError(PS_ERR_IO, false, "Unable to get statistics MDC file."); + exitValue = ppStackExitCode(exitValue); + exit(exitValue); + } + if (fprintf(options->statsFile, "%s", statsMDC) != strlen(statsMDC)) { + psError(PS_ERR_IO, false, "Unable to write statistics MDC file."); + exitValue = ppStackExitCode(exitValue); + exit(exitValue); + } + psFree(statsMDC); + if (fclose(options->statsFile) == EOF) { + psError(PS_ERR_IO, false, "Unable to close statistics MDC file."); + exitValue = ppStackExitCode(exitValue); + exit(exitValue); + } + options->statsFile = NULL; + pmConfigRunFilenameAddWrite(config, "STATS", psMetadataLookupStr(NULL, config->arguments, "STATS")); + } + psFree(options); + + // Dump configuration + bool mdok; // Status of MD lookup + psString dump = psMetadataLookupStr(&mdok, config->arguments, "DUMP_CONFIG"); // File for config + if (dump && !pmConfigDump(config, dump)) { + psError(psErrorCodeLast(), false, "Unable to dump configuration."); + exitValue = ppStackExitCode(exitValue); + } + + psTrace("ppStack", 1, "Finished at %f sec\n", psTimerMark("PPSTACK")); + psLogMsg("ppStack", PS_LOG_INFO, "Complete ppStack run: %f sec\n", psTimerMark("PPSTACK")); + psTimerStop(); + + psFree(config); + pmModelClassCleanup(); + pmConfigDone(); + psLibFinalize(); + pmVisualClose(); + pmVisualCleanup (); + + exitValue = ppStackExitCode(exitValue); + exit(exitValue); +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackCombineAlternate.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackCombineAlternate.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackCombineAlternate.c (revision 41840) @@ -0,0 +1,216 @@ +#include "ppStack.h" + +// This is the doomsday switch. +// #define TESTING // Enable test output +bool ppStackCombineMedian(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psTimerStart("PPSTACK_BKGMED"); + + + pmReadout *outRO = options->outRO; + + psArray *inputs = psArrayAlloc(options->num); + for (int i = 0; i < options->num; i++) { + ppStackFileActivationSingle(config, PPSTACK_FILES_MEDIAN_IN, true, i); + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); + pmFPAview *view = ppStackFilesIterateDown(config); + pmReadout *ro = pmFPAviewThisReadout(view,file->fpa); + inputs->data[i] = ro; + pmFPAfileClose(file,view); + } + + // Scaling block + if (!ppStackLinearScale(inputs, config)) { + psFree(inputs); + return(false); + } + + if (!pmStackSimpleMedianCombine(outRO,inputs)) { + psFree(inputs); + return(false); + } + +#if 0 + if (!ppStackWriteImage("/tmp/test_forced.median.fits", + outRO->parent->parent->parent->hdu->header, + outRO->image, + config)) { + fprintf(stderr,"Failed to write image because fail.\n"); + } +#endif + for (int i = 0; i < options->num; i++) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); + pmFPAview *view = ppStackFilesIterateDown(config); + bool success = pmFPAfileClose(file,view); + if (!success) { + psTrace("ppStack",5,"I failed at closing a file.\n"); + } + + psFitsClose(file->fits); + file->fits = NULL; + file->header = NULL; + file->state = PM_FPA_STATE_CLOSED; + file->wrote_phu = false; + ppStackFileActivationSingle(config, PPSTACK_FILES_MEDIAN_IN, false, i); + } + psFree(inputs); + outRO->data_exists = true; + outRO->parent->data_exists = true; + outRO->parent->parent->data_exists = true; + + return(true); +} + + +bool ppStackCombineBackground(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psTimerStart("PPSTACK_BKGMED"); + + + pmReadout *bkgRO = options->bkgRO; + + psArray *inputs = psArrayAlloc(options->num); + for (int i = 0; i < options->num; i++) { + ppStackFileActivationSingle(config, PPSTACK_FILES_BKG, true, i); + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.BKGMODEL", i); + pmFPAview *view = ppStackFilesIterateDown(config); + pmReadout *ro = pmFPAviewThisReadout(view,file->fpa); + inputs->data[i] = ro; + pmFPAfileClose(file,view); + } + + if (!ppStackLinearScale(inputs, config)) { + psFree(inputs); + return(false); + } + + // Do combination + if (!pmStackSimpleMedianCombine(bkgRO,inputs)) { + psFree(inputs); + return(false); + } +#if 0 + if (!ppStackWriteImage("/tmp/test_forced.bkgmdl.fits", + bkgRO->parent->parent->parent->hdu->header, + bkgRO->image, + config)) { + fprintf(stderr,"Failed to write image because fail.\n"); + } +#endif + for (int i = 0; i < options->num; i++) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.BKGMODEL", i); + pmFPAview *view = ppStackFilesIterateDown(config); + bool success = pmFPAfileClose(file,view); + if (!success) { + psTrace("ppStack",5,"I failed at closing a file.\n"); + } + + psFitsClose(file->fits); + file->fits = NULL; + file->header = NULL; + file->state = PM_FPA_STATE_CLOSED; + file->wrote_phu = false; + ppStackFileActivationSingle(config, PPSTACK_FILES_BKG, false, i); + } + psFree(inputs); + bkgRO->data_exists = true; + bkgRO->parent->data_exists = true; + bkgRO->parent->parent->data_exists = true; + + return(true); +} + +bool ppStackLinearScale (psArray *inputs, pmConfig *config) { + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); + bool doLinearScaling = psMetadataLookupBool(NULL, recipe, "DO.LINEAR.INPUT.SCALING"); + if (doLinearScaling) { + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize,inputs)) { + psError(psErrorCodeLast(), false, "Input stack is not valid."); + psFree(inputs); + return false; + } + + // Determine best input. + int ref = 0; + int Nref = 0; + double refScale = 0; + + for (int i = 0; i < inputs->n; i++) { + pmReadout *ro = inputs->data[i]; + psImage *roImage = ro->image; + int Nvalid = 0; + double scale = 0; + + for (int y = minInputRows; y < maxInputRows; y++) { + for (int x = minInputCols; x < maxInputCols; x++) { + if ((isfinite(roImage->data.F32[y][x]))) { + Nvalid += 1; + scale += roImage->data.F32[y][x]; + } + } + } + if ((scale > refScale)&&(Nvalid > 0.9 * Nref)) { + ref = i; + refScale = scale; + Nref = Nvalid; + } + } + fprintf(stderr,"ref: %d %d %f\n",ref,Nref,refScale); + // Calculate scaling factors + pmReadout *refReadout = inputs->data[ref]; + psImage *refImage = refReadout->image; + for (int i = 0; i < inputs->n; i++) { + pmReadout *ro = inputs->data[i]; + psImage *roImage = ro->image; + double S = 0.0; + double Sx = 0.0; + double Sy = 0.0; + double Sxx = 0.0; + double Syy = 0.0; + double Sxy = 0.0; + double D = 0.0; + double offset = 0.0; + double scale = 0.0; + + for (int y = minInputRows; y < maxInputRows; y++) { + for (int x = minInputCols; x < maxInputCols; x++) { + if ((isfinite(refImage->data.F32[y][x]))&& + (isfinite(roImage->data.F32[y][x]))) { + S += 1.0; + Sx += roImage->data.F32[y][x]; + Sy += refImage->data.F32[y][x]; + + Sxx += pow(roImage->data.F32[y][x],2); + Syy += pow(refImage->data.F32[y][x],2); + Sxy += roImage->data.F32[y][x] * refImage->data.F32[y][x]; + } + } + } + + D = S * Sxx - Sx * Sx; + offset = (Sy * Sxx - Sx * Sxy) / D; + scale = (S * Sxy - Sx * Sy) / D; + fprintf(stderr,"Scales: %d %g %g %g %g %g %g %g %g\n",i,offset,scale,D,Sx,Sy,Sxx,Syy,Sxy); + // Apply scaling factors + for (int y = minInputRows; y < maxInputRows; y++) { + for (int x = minInputCols; x < maxInputCols; x++) { + roImage->data.F32[y][x] = offset + scale * roImage->data.F32[y][x]; + } + } + } + } + return(true); +} + + Index: /branches/eam_branches/ppStack.20211015/src/ppStackCombineFinal.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackCombineFinal.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackCombineFinal.c (revision 41840) @@ -0,0 +1,170 @@ +#include "ppStack.h" + +// This is the doomsday switch. +// #define TESTING // Enable test output + +//MEH -- adhoc addition to blank mask border of final stack since rejection different/none on order of KERNEL.SIZE with overlap in CombineInitial +static void stackBorderMask(psImage *image, // Image to mark as blank + psImage *mask, // Mask to mark as blank (or NULL) + psImage *variance, // Weight map to mark as blank (or NULL) + int numCols, int numRows, // Size of image + int size, // Size to mark blank + psImageMaskType blank // Blank mask value + ) +{ + for (int y = size; y < numRows - size; y++) { + for (int x = 0; x < size; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + for (int x = numCols - size; x < numCols; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + } + for (int y = 0; y < size; y++) { + for (int x = 0; x < numCols; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + } + for (int y = numRows - size; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = blank; + variance->data.F32[y][x] = NAN; + } + } + return; +} + + +bool ppStackCombineFinal(ppStackThreadData *stack, psArray *covariances, ppStackOptions *options, + pmConfig *config, bool safe, bool normalise, bool grow, bool bscaleoffset) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psTimerStart("PPSTACK_FINAL"); + + pmReadout *outRO = options->outRO; // Output readout + pmReadout *expRO = options->expRO; // Exposure readout + int numCols = outRO->image->numCols, numRows = outRO->image->numRows; // Size of image + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + float poorFrac = psMetadataLookupF32(NULL, recipe, "POOR.FRACTION"); // Fraction for "poor" + + int sizeBlank = psMetadataLookupS32(NULL, recipe, "MASK.BLANKBORDER"); // Pixels to mask BLANK from edge + psImageMaskType maskBlank = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels + + // Grow the list of rejected pixels, if desired + psArray *reject = psArrayAlloc(options->num); // Pixels rejected for each image + for (int i = 0; i < options->num; i++) { + if (options->inputMask->data.U8[i]) { + continue; + } + if (grow) { + reject->data[i] = pmStackRejectGrow(options->rejected->data[i], numCols, numRows, poorFrac, + options->regions->data[i], options->kernels->data[i]); + } else { + reject->data[i] = psMemIncrRefCounter(options->rejected->data[i]); + } + } + + if (!outRO->mask) { + outRO->mask = psImageAlloc(numCols, numRows, PS_TYPE_IMAGE_MASK); + } + if (!expRO->mask) { + expRO->mask = psImageAlloc(numCols, numRows, PS_TYPE_IMAGE_MASK); + } + + stack->lastScan = 0; // Reset read + bool status; // Status of read + for (int numChunk = 0; true; numChunk++) { + ppStackThread *thread = ppStackThreadRead(&status, stack, config, numChunk, 0); + if (!status) { + // Something went wrong + psError(psErrorCodeLast(), false, "Unable to read chunk %d", numChunk); + psFree(reject); + return false; + } + if (!thread) { + // Nothing more to read + break; + } + + // calls ppStackReadoutFinal(config, outRO, readouts, rejected) in ppStackReadout.c + psThreadJob *job = psThreadJobAlloc("PPSTACK_FINAL_COMBINE"); // Job to start + psArrayAdd(job->args, 1, thread); + psArrayAdd(job->args, 1, reject); + psArrayAdd(job->args, 1, options); + psArrayAdd(job->args, 1, config); + PS_ARRAY_ADD_SCALAR(job->args, safe, PS_TYPE_U8); + PS_ARRAY_ADD_SCALAR(job->args, normalise, PS_TYPE_U8); + PS_ARRAY_ADD_SCALAR(job->args, bscaleoffset, PS_TYPE_U8); + if (!psThreadJobAddPending(job)) { + psFree(reject); + return false; + } + } + + if (!psThreadPoolWait(true, true)) { + psError(psErrorCodeLast(), false, "Unable to do final combination."); + psFree(reject); + return false; + } + + psFree(reject); + + // Sum covariance matrices + // the array may be defined, but no covariances actually supplied. + bool haveCovariances = false; + if (covariances) { + for (int i = 0; i < covariances->n; i++) { + haveCovariances |= (covariances->data[i] != NULL); + } + } + + if (haveCovariances) { + outRO->covariance = psImageCovarianceAverageWeighted(covariances, options->weightings); + } else { + outRO->covariance = psImageCovarianceNone(); + } + +#ifdef TESTING + static int pass = 0; // Pass through + psString name = NULL; // Name of file + psStringAppend(&name, "combined_image_final_%d.fits", pass); + pass++; + ppStackWriteImage(name, NULL, outRO->image, config); + psStringSubstitute(&name, "mask", "image"); + ppStackWriteImage(name, NULL, outRO->mask, config); + psStringSubstitute(&name, "variance", "mask"); + ppStackWriteImage(name, NULL, outRO->variance, config); + psFree(name); + + pmStackVisualPlotTestImage(outRO->image, "combined_image_final.fits"); +#endif + + //MEH blank mask/manual reject border on final stack -- + if (sizeBlank > 0) { + stackBorderMask(outRO->image,outRO->mask,outRO->variance,numCols,numRows,sizeBlank,maskBlank); + stackBorderMask(expRO->image,expRO->mask,expRO->variance,numCols,numRows,sizeBlank,0); + } + + if (options->stats) { + // only add the timer if it has not been set (convolved stack) + // XXX this is weak: we should be distinguishing explicitly between convolved and unconvolved stacks + psMetadataLookupF32 (&status, options->stats, "TIME_FINAL"); + if (!status) { + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_FINAL", PS_META_REPLACE, "Time to make final stack", psTimerMark("PPSTACK_FINAL")); + } + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackCombineInitial.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackCombineInitial.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackCombineInitial.c (revision 41840) @@ -0,0 +1,96 @@ +#include "ppStack.h" + +// This is the doomsday switch. +// #define TESTING // Enable test output + +bool ppStackCombineInitial(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + +/* if (!options->convolve) { */ +/* // No need to do initial combination when we haven't convolved */ +/* // XXX either allocate inspect and rejected here, or do not require them downstream */ +/* return true; */ +/* } */ + + psTimerStart("PPSTACK_INITIAL"); + + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + int overlap = 2 * psMetadataLookupS32(NULL, ppsub, + "KERNEL.SIZE"); // Overlap by kernel size between consecutive scans + + + bool status; // Status of read + int numChunk; // Number of chunks + for (numChunk = 0; true; numChunk++) { + ppStackThread *thread = ppStackThreadRead(&status, stack, config, numChunk, overlap); + if (!status) { + // Something went wrong + psError(psErrorCodeLast(), false, "Unable to read chunk %d", numChunk); + return false; + } + if (!thread) { + // Nothing more to read + break; + } + + psThreadJob *job = psThreadJobAlloc("PPSTACK_INITIAL_COMBINE"); // Job to start + psArrayAdd(job->args, 1, thread); + psArrayAdd(job->args, 1, options); + psArrayAdd(job->args, 1, config); + if (!psThreadJobAddPending(job)) { + return false; + } + } + + if (!psThreadPoolWait(false, true)) { + psError(psErrorCodeLast(), false, "Unable to do initial combination."); + return false; + } + + // Harvest the jobs, gathering the inspection lists + options->inspect = psArrayAlloc(options->num); + options->rejected = psArrayAlloc(options->num); + for (int i = 0; i < options->num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + options->inspect->data[i] = psArrayAllocEmpty(numChunk); + options->rejected->data[i] = psArrayAllocEmpty(numChunk); + } + psThreadJob *job; // Completed job + while ((job = psThreadJobGetDone())) { + psAssert(strcmp(job->type, "PPSTACK_INITIAL_COMBINE") == 0, + "Job has incorrect type: %s", job->type); + psArray *results = job->results; // Results of job + psAssert(results->n == 2, "Results array has wrong size!"); + psArray *inspect = results->data[0]; // Pixels to inspect + psArray *reject = results->data[1]; // Pixels to reject + for (int i = 0; i < options->num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + options->inspect->data[i] = psArrayAdd(options->inspect->data[i], 1, inspect->data[i]); + options->rejected->data[i] = psArrayAdd(options->rejected->data[i], 1, reject->data[i]); + } + psFree(job); + } + + ppStackMemDump("initial"); + +#ifdef TESTING + ppStackWriteImage("combined_image_initial.fits", NULL, options->outRO->image, config); + ppStackWriteImage("combined_mask_initial.fits", NULL, options->outRO->mask, config); + ppStackWriteImage("combined_variance_initial.fits", NULL, options->outRO->variance, config); + + pmStackVisualPlotTestImage(options->outRO->image, "combined_image_initial.fits"); +#endif + + if (options->stats) { + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_INITIAL", PS_META_REPLACE, "Time to make initial stack", psTimerMark("PPSTACK_INITIAL")); + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackCombinePrepare.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackCombinePrepare.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackCombinePrepare.c (revision 41840) @@ -0,0 +1,105 @@ +#include "ppStack.h" + +bool ppStackCombinePrepare(const char *outName, const char *expName, const char *bkgName, + ppStackFileList files, ppStackThreadData *stack, + ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + int row0, col0; // Offset for readout + int numCols, numRows; // Size of readout + ppStackThread *thread = stack->threads->data[0]; // Representative thread + if (!pmReadoutStackSetOutputSize(&col0, &row0, &numCols, &numRows, thread->readouts)) { + psError(PPSTACK_ERR_ARGUMENTS, false, "problem setting output readout size."); + return false; + } + + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivation(config, files, true); + pmFPAview *view = ppStackFilesIterateDown(config); // View to readout + if (!view) { + return false; + } + + pmCell *cell = pmFPAfileThisCell(config->files, view, outName); // Output cell + options->outRO = pmReadoutAlloc(cell); // Output readout + + if (expName) { + pmCell *expCell = pmFPAfileThisCell(config->files, view, expName); // Exposure cell + options->expRO = pmReadoutAlloc(expCell); //Output readout + } +/* else { */ +/* options->expRO = NULL; */ +/* } */ + int bkg_r0 = 0,bkg_c0 = 0; + int bkg_nC = 0,bkg_nR = 0; + if (bkgName) { + ppStackFileActivationSingle(config, PPSTACK_FILES_BKG, true, 0); + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.BKGMODEL", 0); + if (!file) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Output background model selected but no inputs provided."); + return false; + } + pmFPAview *view = ppStackFilesIterateDown(config); + pmReadout *ro = pmFPAviewThisReadout(view,file->fpa); + + bkg_r0 = ro->image->row0; + bkg_c0 = ro->image->col0; + bkg_nC = ro->image->numCols; + bkg_nR = ro->image->numRows; + pmCell *bkgCell = pmFPAfileThisCell(config->files, view, bkgName); // Bkg cell + + options->bkgRO = pmReadoutAlloc(bkgCell); // BKG readout + // if (!pmHDUGenerateForFPA(options->bkgRO->parent->parent->parent)) { + options->bkgRO->parent->parent->parent->hdu = pmHDUAlloc(NULL); + if (!options->bkgRO->parent->parent->parent->hdu) { + fprintf(stderr,"failed to generate a HDU for this thing.\n"); + } + options->bkgRO->parent->parent->parent->hdu->header = psMetadataCopy(options->bkgRO->parent->parent->parent->hdu->header, + ro->parent->parent->parent->hdu->header); + + options->bkgRO->parent->concepts = psMetadataCopy(options->bkgRO->parent->concepts, + ro->parent->concepts); + options->bkgRO->parent->parent->concepts = psMetadataCopy(options->bkgRO->parent->parent->concepts, + ro->parent->parent->concepts); + options->bkgRO->parent->parent->parent->concepts = psMetadataCopy(options->bkgRO->parent->parent->parent->concepts, + ro->parent->parent->parent->concepts); + + } + else { + options->bkgRO = NULL; + } + + psFree(view); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits to mask for bad + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + if (!pmReadoutStackDefineOutput(options->outRO, col0, row0, numCols, numRows, true, true, maskBad)) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Unable to prepare output."); + return false; + } + + if (expName) { + if (!pmReadoutStackDefineOutput(options->expRO, col0, row0, numCols, numRows, true, true, 0)) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Unable to prepare output."); + return false; + } + } + + if (bkgName) { + if (!pmReadoutStackDefineOutput(options->bkgRO, bkg_c0, bkg_r0, bkg_nC, bkg_nR, false, false, 0)) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Unable to prepare output."); + return false; + } + + + + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackConvolve.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackConvolve.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackConvolve.c (revision 41840) @@ -0,0 +1,445 @@ +#include "ppStack.h" + +//#define TESTING + +// Update the value of a concept +#define UPDATE_CONCEPT(SOURCE, NAME, VALUE) { \ + psMetadataItem *item = psMetadataLookup(SOURCE->concepts, NAME); \ + psAssert(item, "Concept should be present"); \ + psAssert(item->type == PS_DATA_F32, "Concept should be F32"); \ + item->data.F32 = VALUE; \ + } + +bool ppStackConvolve(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psTimerStart("PPSTACK_CONVOLVE"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + int num = options->num; // Number of inputs + options->cells = psArrayAlloc(num); // Cells for convolved images --- a handle for reading again + options->kernels = psArrayAlloc(num); // PSF-matching kernels --- required in the stacking + options->regions = psArrayAlloc(num); // PSF-matching regions --- required in the stacking + int numGood = 0; // Number of good frames + options->numCols = 0; + options->numRows = 0; + options->matchChi2 = psVectorAlloc(num, PS_TYPE_F32); // chi^2 for stamps when matching + psVectorInit(options->matchChi2, NAN); + options->weightings = psVectorAlloc(num, PS_TYPE_F32); // Combination weightings for images (1/noise^2) + psVectorInit(options->weightings, 0.0); + options->origCovars = psArrayAlloc(num); + options->convCovars = psArrayAlloc(num); // Covariance matrices + + psImage *target = NULL; // Target PSF image + if (options->convolve) { + target = ppStackTarget(options, config); + if (!target) { + psError(psErrorCodeLast(), false, "Unable to produce stack target image"); + return false; + } + } + + psVector *renorms = psVectorAlloc(num, PS_TYPE_F32); // Renormalisation values for variances + psVectorInit(renorms, NAN); + + psVector *satValues = psVectorAllocEmpty(num, PS_TYPE_F32); // Renormalisation values for variances + + psList *fpaList = psListAlloc(NULL); // List of input FPAs, for concept averaging + psList *cellList = psListAlloc(NULL); // List of input cells, for concept averaging + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + + for (int i = 0; i < num; i++) { + if (options->inputMask->data.U8[i]) { + continue; + } + psTrace("ppStack", 2, "Convolving input %d of %d to target PSF....\n", i, num); + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivationSingle(config, PPSTACK_FILES_CONVOLVE, true, i); + if (options->convolve) { + // PPSTACK.CONV.KERNEL not defined unless convolve + pmFPAfileActivateSingle(config->files, true, "PPSTACK.CONV.KERNEL", i); // Activated file + } + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest + pmFPAview *view = ppStackFilesIterateDown(config); + if (!view) { + psFree(rng); + psFree(fpaList); + psFree(cellList); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); // Input readout + psFree(view); + + if (options->numCols == 0 && options->numRows == 0) { + options->numCols = readout->image->numCols; + options->numRows = readout->image->numRows; + } else if (options->numCols != readout->image->numCols || + options->numRows != readout->image->numRows) { + psError(PPSTACK_ERR_ARGUMENTS, true, "Sizes of input images don't match: %dx%d vs %dx%d", + readout->image->numCols, readout->image->numRows, options->numCols, options->numRows); + psFree(rng); + psFree(fpaList); + psFree(cellList); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } + + // Background subtraction, scaling and normalisation is performed automatically by the image matching + psTimerStart("PPSTACK_MATCH"); + options->origCovars->data[i] = psMemIncrRefCounter(readout->covariance); + if (!ppStackMatch(readout, target, options, i, config)) { + // XXX many things can cause a failure of ppStackMatch -- should some be handled differently? + + // gcc -Wswitch complains here if err is declared as type psErrorCode + // the collection of ps*ErrorCode values are enums defined separately for + // each module (psphot, pswarp, etc). the lowest type, psErrorCode is only the base set and does + // not include the possible psphot values + + // for now, to get around this, we just use an int for the switch + + // psErrorCode error = psErrorCodeLast(); // Error code + int error = psErrorCodeLast(); // Error code + switch (error) { + // Fatal errors + case PM_ERR_CONFIG: + case PPSTACK_ERR_CONFIG: + case PPSTACK_ERR_IO: + psError(error, false, "Unable to match image %d due to fatal error.", i); + psFree(rng); + psFree(fpaList); + psFree(cellList); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + // Non-fatal errors + case PM_ERR_STAMPS: + case PM_ERR_SMALL_AREA: + case PPSTACK_ERR_DATA: + default: + psErrorStackPrint(stderr, "Unable to match image %d --- ignoring.", i); + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_MATCH; + psErrorClear(); + if (!ppStackFilesIterateUp(config)) { + psFree(rng); + psFree(fpaList); + psFree(cellList); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } + continue; + } + } + options->convCovars->data[i] = psMemIncrRefCounter(readout->covariance); + + float renorm = psMetadataLookupF32(NULL, readout->analysis, PM_READOUT_ANALYSIS_RENORM); + if (!isfinite(renorm)) { + renorm = 1.0; + } + renorms->data.F32[i] = renorm; + + if (options->stats) { + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_MATCH", PS_META_DUPLICATE_OK, + "Time to match PSF", psTimerMark("PPSTACK_MATCH")); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "PPSTACK.WEIGHTING", PS_META_DUPLICATE_OK, + "Weighting for image", options->weightings->data.F32[i]); + + if (options->convolve) { + // Pull parameters out of convolution kernel + pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, readout->analysis, + PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.MEAN", PS_META_DUPLICATE_OK, + "Mean deviation for stamps", kernels->mean); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.RMS", PS_META_DUPLICATE_OK, + "RMS deviation for stamps", kernels->rms); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.NUM", PS_META_DUPLICATE_OK, + "Number of stamps", kernels->numStamps); + float deconv = psMetadataLookupF32(NULL, readout->analysis, + PM_SUBTRACTION_ANALYSIS_DECONV_MAX); + psMetadataAddF32(options->stats, PS_LIST_TAIL, "KERNEL.DECONV", PS_META_DUPLICATE_OK, + "Deconvolution fraction for kernel", deconv); + } + } + psLogMsg("ppStack", PS_LOG_INFO, "Time to match image %d: %f sec", i, psTimerClear("PPSTACK_MATCH")); + + // Write the temporary convolved files + pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image + assert(hdu); + if (!ppStackWriteImage(options->convImages->data[i], hdu->header, readout->image, config)) { + psError(PPSTACK_ERR_IO, false, "Unable to write convolved image %d", i); + psFree(fpaList); + psFree(cellList); + psFree(rng); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } + psMetadata *maskHeader = psMetadataCopy(NULL, hdu->header); // Copy of header, for mask + pmConfigMaskWriteHeader(config, maskHeader); + if (!ppStackWriteImage(options->convMasks->data[i], maskHeader, readout->mask, config)) { + psError(PPSTACK_ERR_IO, false, "Unable to write convolved mask %d", i); + psFree(fpaList); + psFree(cellList); + psFree(rng); + psFree(maskHeader); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } + psFree(maskHeader); + if (!ppStackWriteVariance(options->convVariances->data[i], hdu->header, readout->variance, readout->covariance->image, config)) { + psError(PPSTACK_ERR_IO, false, "Unable to write convolved variance %d", i); + psFree(fpaList); + psFree(cellList); + psFree(rng); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } +#ifdef TESTING + { + psString name = NULL; + psStringAppend(&name, "covariance_%d.fits", i); + ppStackWriteImage(name, hdu->header, readout->covariance->image, config); + pmStackVisualPlotTestImage(readout->covariance->image, name); + psFree(name); + } + { + int numCols = readout->image->numCols, numRows = readout->image->numRows; + psImage *sn = psImageAlloc(numCols, numRows, PS_TYPE_F32); + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + sn->data.F32[y][x] = readout->image->data.F32[y][x] / + sqrtf(readout->variance->data.F32[y][x]); + } + } + psString name = NULL; + psStringAppend(&name, "signoise_%d.fits", i); + ppStackWriteImage(name, hdu->header, sn, config); + psFree(name); + psFree(sn); + } +#endif + + pmCell *inCell = readout->parent; // Input cell + + psListAdd(cellList, PS_LIST_TAIL, inCell); + psListAdd(fpaList, PS_LIST_TAIL, inCell->parent->parent); + + // Correct ZP + if (options->matchZPs) { + // I think I need to take off the exposure time because we're going to set the new exposure time + // Clarification: the zero point (ZP) in the header should be set such that: + // M_app = m_inst + ZP + 2.5*log(exptime), where exptime in the output is sumExposure + psMetadataItem *zpItem = psMetadataLookup(inCell->parent->parent->concepts, "FPA.ZP"); + float inZP = zpItem->data.F32; + zpItem->data.F32 += options->norm->data.F32[i] + 2.5*log10(options->sumExposure); + + psMetadataItem *expItem = psMetadataLookup(inCell->parent->parent->concepts, "FPA.EXPOSURE"); + expItem->data.F32 = options->sumExposure; + + expItem = psMetadataLookup(inCell->concepts, "CELL.EXPOSURE"); + float inExptime = expItem->data.F32; + expItem->data.F32 = options->sumExposure; + + // flux_out = flux_in * ten(-0.4*norm) -- save the individual saturation values + psMetadataItem *satItem = psMetadataLookup(inCell->concepts, "CELL.SATURATION"); + float inSat = satItem->data.F32; + satItem->data.F32 *= pow(10.0, -0.4*options->norm->data.F32[i]); + psVectorAppend (satValues, satItem->data.F32); + + psLogMsg("ppStack", PS_LOG_INFO, "image %d mods : zp %f -> %f, exptime %f -> %f, sat %f -> %f", + i, inZP, zpItem->data.F32, inExptime, expItem->data.F32, inSat, satItem->data.F32); + } + + options->cells->data[i] = psMemIncrRefCounter(inCell); + if (!ppStackFilesIterateUp(config)) { + psFree(fpaList); + psFree(cellList); + psFree(rng); + psFree(target); + psFree(renorms); + psFree(satValues); + return false; + } + numGood++; + + ppStackMemDump("match"); + } + psFree(rng); + psFree(target); + + psFree(options->sourceLists); options->sourceLists = NULL; + psFree(options->psf); options->psf = NULL; + + + if (numGood == 0) { + options->quality = PPSTACK_ERR_REJECTED; + psErrorStackPrint(stderr, "No good images survived convolution stage."); + psErrorClear(); + psWarning("No good images survived convolution stage."); + psFree(fpaList); + psFree(cellList); + psFree(renorms); + psFree(satValues); + return true; + } + + // Update concepts for output + // XXX This should probably be placed later, so that it's not influenced + // by images that get rejected later. + { + pmFPAview view; // View for output + view.chip = view.cell = view.readout = 0; + + pmCell *outCell = pmFPAfileThisCell(config->files, &view, "PPSTACK.OUTPUT"); // Output cell + pmFPA *outFPA = outCell->parent->parent; // Output FPA + + pmCell *unconvCell = pmFPAfileThisCell(config->files, &view, "PPSTACK.UNCONV"); // Unconvolved cell + pmFPA *unconvFPA = unconvCell->parent->parent; // Unconvolved FPA + + pmConceptsAverageFPAs(outFPA, fpaList); + pmConceptsAverageCells(outCell, cellList, NULL, NULL, true); + + pmConceptsAverageFPAs(unconvFPA, fpaList); + pmConceptsAverageCells(unconvCell, cellList, NULL, NULL, true); + + psFree(fpaList); + psFree(cellList); + + // The best guess for an output saturation value depends on the recipe. If we have + // 'safe' on, the we require at least 2 pixels to generate a valid output pixel. In + // this case, the best value for CELL.SATURATION is the 2nd highest value in the list. + // If not, it should be the higest value in the list + bool mdok = false; + bool safe = psMetadataLookupBool(&mdok, recipe, "SAFE"); // Be safe when combining small numbers of pixels + psVectorSortInPlace(satValues); + float satBest = safe && satValues->n > 1 ? satValues->data.F32[1] : satValues->data.F32[0]; + + // UPDATE CELL.SATURATION here + UPDATE_CONCEPT(outFPA, "FPA.EXPOSURE", options->sumExposure); + UPDATE_CONCEPT(outCell, "CELL.EXPOSURE", options->sumExposure); + UPDATE_CONCEPT(outCell, "CELL.DARKTIME", NAN); + UPDATE_CONCEPT(outCell, "CELL.SATURATION", satBest); + UPDATE_CONCEPT(outFPA, "FPA.ZP", options->zp); + UPDATE_CONCEPT(outFPA, "FPA.AIRMASS", options->airmass); + + UPDATE_CONCEPT(unconvFPA, "FPA.EXPOSURE", options->sumExposure); + UPDATE_CONCEPT(unconvCell, "CELL.EXPOSURE", options->sumExposure); + UPDATE_CONCEPT(unconvCell, "CELL.DARKTIME", NAN); + UPDATE_CONCEPT(unconvCell, "CELL.SATURATION", satBest); + UPDATE_CONCEPT(unconvFPA, "FPA.ZP", options->zp); + UPDATE_CONCEPT(unconvFPA, "FPA.AIRMASS", options->airmass); + + psLogMsg("ppStack", PS_LOG_INFO, "stack adjust metadata values : zp %f, exptime %f, sat %f", options->zp, options->sumExposure, satBest); + + if (options->stats) { + psTime *fpaTime = psMetadataLookupPtr(NULL, outFPA->concepts, "FPA.TIME"); + psTimeConvert(fpaTime, PS_TIME_TAI); // is this necessary?? + double time = psTimeToMJD(fpaTime); + psMetadataAddF64(options->stats, PS_LIST_TAIL, "MJD_OBS", PS_META_DUPLICATE_OK, + "Average MJD_OBS of inputs", time); + + } + } + + // XXX EAM : this may be overly harsh -- or at least it would be if I (EAM) hadn't changed + // the values of matchChi2 I modified pmSubtraction.c to fit a 2nd order polynomial to the + // star chisq distribution (because of systematic errors in the model being matched). This + // fit forces the mean value to be 0.0. Perhaps we can / should exclude images which have + // an excessively high value for the rms? + + // Reject images out-of-hand on the basis of their match chi^2 + if (options->convolve) { + psVector *values = psVectorAllocEmpty(num, PS_TYPE_F32); // Values to sort + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_ALL) { + continue; + } + values->data.F32[values->n++] = options->matchChi2->data.F32[i]; + } + assert(values->n == numGood); + if (!psVectorSortInPlace(values)) { + psError(PPSTACK_ERR_PROG, false, "Unable to sort vector."); + psFree(values); + psFree(renorms); + psFree(satValues); + return false; + } + float median = numGood % 2 ? values->data.F32[numGood / 2] : + 0.5 * (values->data.F32[numGood / 2 - 1] + values->data.F32[numGood / 2]); + + // EAM/MEH ensure RMS is positive + float rms = 0.74 * fabsf(values->data.F32[numGood * 3 / 4] - + values->data.F32[numGood / 4]); // Estimated RMS from interquartile range + psFree(values); + + float rej = psMetadataLookupF32(NULL, recipe, "MATCH.REJ"); // Rejection threshold (stdevs) + if (isfinite(rej)) { + float thresh = median + rej * rms; // Threshold for rejection + psLogMsg("ppStack", PS_LOG_INFO, "chi^2 rejection threshold = %f + %f * %f = %f", + median, rej, rms, thresh); + + int numRej = 0; // Number rejected + numGood = 0; // Number of good images + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_ALL) { + continue; + } + if ((options->matchChi2->data.F32[i] > thresh) || + ! isfinite(options->matchChi2->data.F32[i])) { + numRej++; + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_CHI2; + psLogMsg("ppStack", PS_LOG_INFO, "Rejecting image %d because of large matching chi^2: %f", + i, options->matchChi2->data.F32[i]); + } else { + psLogMsg("ppStack", PS_LOG_INFO, "Image %d has matching chi^2: %f", + i, options->matchChi2->data.F32[i]); + numGood++; + } + } + } + } + + if (numGood == 0) { + options->quality = PPSTACK_ERR_REJECTED; + psErrorStackPrint(stderr, "No good images survived convolution stage."); + psErrorClear(); + psWarning("No good images survived convolution stage."); + psFree(renorms); + psFree(satValues); + return true; + } + + // Correct chi^2 for renormalisation + psBinaryOp(options->matchChi2, options->matchChi2, "/", renorms); + for (int i = 0; i < num; i++) { + psLogMsg("ppStack", PS_LOG_INFO, "Additional variance for image %d: %f\n", + i, options->matchChi2->data.F32[i]); + } + psFree(renorms); + psFree(satValues); + + if (options->stats) { + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_CONVOLVE", PS_META_REPLACE, + "Time to convolve input images", psTimerMark("PPSTACK_CONVOLVE")); + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.c.in =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.c.in (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.c.in (revision 41840) @@ -0,0 +1,36 @@ +#include "ppStack.h" + +/** @file ppStackErrorCodes.c.in + * + * @brief + * + * @ingroup ppStack + * + * @author IfA + * @version $Revision: 1.3 $ $Name: not supported by cvs2svn $ + * @date $Date: 2009-02-05 20:44:04 $ + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +/* + * The line + { PPSTACK_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +void ppStackErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PPSTACK_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PPSTACK_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PPSTACK_ERR_NERROR - PPSTACK_ERR_BASE; ///< number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.dat =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.dat (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.dat (revision 41840) @@ -0,0 +1,13 @@ +# +# This file is used to generate ppStackErrorClasses.h +# +BASE = 13000 First value we use; lower values belong to psLib +UNKNOWN Unknown ppSub error code +NOT_IMPLEMENTED Desired feature is not yet implemented +ARGUMENTS Incorrect arguments +CONFIG Problem in configure files +IO Problem in FITS I/O +DATA Problem in data values +PSF Problem determining PSF +REJECTED All images rejected +PROG Programming error Index: /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.h.in =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.h.in (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackErrorCodes.h.in (revision 41840) @@ -0,0 +1,30 @@ +/** @file ppStackErrorCodes.h.in + * + * @brief + * + * @ingroup ppStack + * + * @author IfA + * @version $Revision: 1.2 $ $Name: not supported by cvs2svn $ + * @date $Date: 2009-02-05 20:44:04 $ + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#if !defined(PPSTACK_ERROR_CODES_H) +#define PPSTACK_ERROR_CODES_H +/* + * The line + * PPSTACK_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PPSTACK_ERR_BASE = 13000, + PPSTACK_ERR_${ErrorCode}, + PPSTACK_ERR_NERROR +} ppStackErrorCode; + +void ppStackErrorRegister(void); + +#endif Index: /branches/eam_branches/ppStack.20211015/src/ppStackFiles.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackFiles.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackFiles.c (revision 41840) @@ -0,0 +1,256 @@ +#include "ppStack.h" + +// Here follows lists of files for activation/deactivation at various stages. Each must be NULL-terminated. + +/// NOP list +static char *filesNOP[] = { NULL }; + +/// Files required in preparation for convolution +static char *filesPrepare[] = { "PPSTACK.INPUT.PSF", "PPSTACK.INPUT.SOURCES", "PPSTACK.TARGET.PSF", NULL }; + +/// Files required for generating convolution target +static char *filesTarget[] = { "PPSTACK.INPUT.VARIANCE", "PPSTACK.INPUT.MASK", NULL }; + +/// Files required for the convolution +static char *filesConvolve[] = { "PPSTACK.INPUT", "PPSTACK.INPUT.MASK", "PPSTACK.INPUT.VARIANCE", NULL }; + +/// Files required for the background +static char *filesBkg[] = { "PPSTACK.INPUT.BKGMODEL", NULL }; + +/// Files required for median only stacking +static char *filesMedianIn[] = { "PPSTACK.INPUT", + NULL }; +/// Files required for median only stacking +static char *filesMedianOut[] = { "PPSTACK.OUTPUT", + NULL }; + +/// Regular (convolved) stack files +static char *filesStack[] = { "PPSTACK.OUTPUT", "PPSTACK.OUTPUT.MASK", "PPSTACK.OUTPUT.VARIANCE", + "PPSTACK.OUTPUT.EXP", "PPSTACK.OUTPUT.EXPNUM", "PPSTACK.OUTPUT.EXPWT", + "PPSTACK.OUTPUT.JPEG1", "PPSTACK.OUTPUT.JPEG2", + "PPSTACK.OUTPUT.BKGMODEL", + NULL }; +/// Unconvolved stack files +static char *filesUnconv[] = { "PPSTACK.UNCONV", "PPSTACK.UNCONV.MASK", "PPSTACK.UNCONV.VARIANCE", + "PPSTACK.UNCONV.EXP", "PPSTACK.UNCONV.EXPNUM", "PPSTACK.UNCONV.EXPWT", + NULL }; + +/// Files for photometry +static char *filesPhot[] = { "PSPHOT.INPUT", "PSPHOT.OUTPUT", "PSPHOT.RESID", "PSPHOT.BACKMDL", + "PSPHOT.BACKMDL.STDEV", "PSPHOT.BACKGND", "PSPHOT.BACKSUB", + "SOURCE.PLOT.MOMENTS", "SOURCE.PLOT.PSFMODEL", "SOURCE.PLOT.APRESID", + "PSPHOT.INPUT.CMF", NULL }; + +static char **stackFiles(ppStackFileList list) +{ + switch (list) { + case PPSTACK_FILES_NONE: return filesNOP; + case PPSTACK_FILES_PREPARE: return filesPrepare; + case PPSTACK_FILES_TARGET: return filesTarget; + case PPSTACK_FILES_CONVOLVE: return filesConvolve; + case PPSTACK_FILES_STACK: return filesStack; + case PPSTACK_FILES_UNCONV: return filesUnconv; + case PPSTACK_FILES_PHOT: return filesPhot; + case PPSTACK_FILES_BKG: return filesBkg; + case PPSTACK_FILES_MEDIAN_IN: return filesMedianIn; + case PPSTACK_FILES_MEDIAN_OUT: return filesMedianOut; + default: + psAbort("Unrecognised file list: %x", list); + } + return NULL; +} + + +void ppStackMemDump(const char *name) +{ + return; + + static int num = 0; // Counter, to make files unique and give an idea of sequence + + psString filename = NULL; // Name of file + psStringAppend(&filename, "memdump_%s_%03d.txt", name, num); + FILE *memFile = fopen(filename, "w"); + psFree(filename); + + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true); + fprintf(memFile, "# MemBlock Size Source\n"); + unsigned long total = 0; // Total memory used + for (int i = 0; i < numLeaks; i++) { + psMemBlock *mb = leaks[i]; + fprintf(memFile, "%12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize, mb->file, mb->lineno); + total += mb->userMemorySize; + } + fclose(memFile); + psFree(leaks); + + fprintf(stderr, "Memdump %s %d: Memory use: %ld, sbrk: %p\n", name, num, total, sbrk(0)); + num++; +} + +// Activate/deactivate a list of files +void ppStackFileActivation(pmConfig *config, // Configuration + ppStackFileList list, // Files to turn on/off + bool state // Activation state + ) +{ + assert(config); + + char **files = stackFiles(list); // Files to turn on/off + for (int i = 0; files[i] != NULL; i++) { + pmFPAfileActivate(config->files, state, files[i]); + } + return; +} + +// Activate/deactivate a single element for a list +void ppStackFileActivationSingle(pmConfig *config, // Configuration + ppStackFileList list, // Files to turn on/off + bool state, // Activation state + int num // Number of file in sequence + ) +{ + assert(config); + + char **files = stackFiles(list); // Files to turn on/off + for (int i = 0; files[i] != NULL; i++) { + pmFPAfileActivateSingle(config->files, state, files[i], num); // Activated file + } + return; +} + +// Iterate down the hierarchy, loading files; we can get away with this because we're working on skycells +pmFPAview *ppStackFilesIterateDown(pmConfig *config // Configuration + ) +{ + assert(config); + + pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return NULL; + } + view->chip = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return NULL; + } + view->cell = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return NULL; + } + view->readout = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return NULL; + } + return view; +} + +// Iterate up the hierarchy, writing files; we can get away with this because we're working on skycells +bool ppStackFilesIterateUp(pmConfig *config // Configuration + ) +{ + assert(config); + + pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy + view->chip = view->cell = view->readout = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return false; + } + view->readout = -1; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return false; + } + view->cell = -1; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return false; + } + view->chip = -1; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PPSTACK_ERR_IO, false, "File checks failed."); + return false; + } + psFree(view); + return true; +} + +// Write an image to a FITS file +bool ppStackWriteImage(const char *name, // Name of image + psMetadata *header, // Header + const psImage *image, // Image + pmConfig *config // Configuration + ) +{ + assert(name); + assert(image); + + psString resolved = pmConfigConvertFilename(name, config, true, true); // Resolved file name + psFits *fits = psFitsOpen(resolved, "w"); + if (!fits) { + psError(PPSTACK_ERR_IO, false, "Unable to open FITS file %s to write image.", resolved); + psFree(resolved); + return false; + } + if (!psFitsWriteImage(fits, header, image, 0, NULL)) { + psError(PPSTACK_ERR_IO, false, "Unable to write FITS image %s.", resolved); + psFitsClose(fits); + psFree(resolved); + return false; + } + if (!psFitsClose(fits)) { + psError(PPSTACK_ERR_IO, false, "Unable to close FITS image %s.", resolved); + psFree(resolved); + return false; + } + psFree(resolved); + return true; +} + +// Write an image to a FITS file +bool ppStackWriteVariance(const char *name, // Name of image + psMetadata *header, // Header + const psImage *variance, // Variance + const psImage *covariance, // Variance + pmConfig *config // Configuration + ) +{ + assert(name); + assert(variance); + + psString resolved = pmConfigConvertFilename(name, config, true, true); // Resolved file name + psFits *fits = psFitsOpen(resolved, "w"); + if (!fits) { + psError(PPSTACK_ERR_IO, false, "Unable to open FITS file %s to write image.", resolved); + psFree(resolved); + return false; + } + if (!psFitsWriteImage(fits, header, variance, 0, NULL)) { + psError(PPSTACK_ERR_IO, false, "Unable to write FITS image %s.", resolved); + psFitsClose(fits); + psFree(resolved); + return false; + } + if (covariance) { + psMetadata *tmphead = psMetadataAlloc(); + psMetadataAddS32(tmphead, PS_LIST_TAIL, "COVARIANCE.CENTRE.X", PS_META_REPLACE, "center", (int)(covariance->numCols / 2)); + psMetadataAddS32(tmphead, PS_LIST_TAIL, "COVARIANCE.CENTRE.Y", PS_META_REPLACE, "center", (int)(covariance->numRows / 2)); + if (!psFitsWriteImage(fits, tmphead, covariance, 0, "COVAR_SkyChip_SkyCell")) { + psError(PPSTACK_ERR_IO, false, "Unable to write FITS image %s.", resolved); + psFitsClose(fits); + psFree(resolved); + return false; + } + } + if (!psFitsClose(fits)) { + psError(PPSTACK_ERR_IO, false, "Unable to close FITS image %s.", resolved); + psFree(resolved); + return false; + } + psFree(resolved); + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackFinish.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackFinish.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackFinish.c (revision 41840) @@ -0,0 +1,110 @@ +#include "ppStack.h" + +// this function unlinks the temporary files (if desired) +bool ppStackFinish(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psThreadPoolFinalize(); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + bool mdok; // Status of MD lookup + bool tempDelete = psMetadataLookupBool(&mdok, recipe, "TEMP.DELETE"); // Delete temporary files? + + // Delete temporary images + if (tempDelete && options->convolve) { + for (int i = 0; i < options->num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + + // XXX careful about repeatative resolution of nebulous names (though these are probably not neb names) + psString imageResolved = pmConfigConvertFilename(options->convImages->data[i], config, false, false); + psString maskResolved = pmConfigConvertFilename(options->convMasks->data[i], config, false, false); + psString varianceResolved = pmConfigConvertFilename(options->convVariances->data[i], config, false, false); + if (unlink(imageResolved) == -1 || unlink(maskResolved) == -1 || + unlink(varianceResolved) == -1) { + psWarning("Unable to delete temporary files for image %d", i); + } + psFree(imageResolved); + psFree(maskResolved); + psFree(varianceResolved); + } + } + return true; +} + + +psExit ppStackExitCode(psExit exitValue) +{ + if (exitValue != PS_EXIT_SUCCESS) { + return exitValue; + } + + // gcc -Wswitch complains here if err is declared as type psErrorCode + // the collection of ps*ErrorCode values are enums defined separately for + // each module (psphot, pswarp, etc). the lowest type, psErrorCode is only the base set and does + // not include the possible psphot values + + // for now, to get around this, we just use an int for the switch + + // psErrorCode errorCode = psErrorCodeLast(); // Error code + int errorCode = psErrorCodeLast(); // Error code + if (errorCode == PS_ERR_NONE) { + return exitValue; + } + + psErrorStackPrint(stderr, "Unable to perform stack."); + pmFPAfileFreeSetStrict(false); + switch (errorCode) { + case PPSTACK_ERR_UNKNOWN: + case PS_ERR_UNKNOWN: + psLogMsg("ppStack", PS_LOG_WARN, "Unknown error code: %x", errorCode); + exitValue = PS_EXIT_UNKNOWN_ERROR; + break; + case PS_ERR_IO: + case PS_ERR_DB_CLIENT: + case PS_ERR_DB_SERVER: + case PS_ERR_BAD_FITS: + case PS_ERR_OS_CALL_FAILED: + case PM_ERR_SYS: + case PPSTACK_ERR_IO: + psLogMsg("ppStack", PS_LOG_WARN, "I/O error code: %x", errorCode); + exitValue = PS_EXIT_SYS_ERROR; + break; + case PS_ERR_BAD_PARAMETER_VALUE: + case PS_ERR_BAD_PARAMETER_TYPE: + case PS_ERR_BAD_PARAMETER_NULL: + case PS_ERR_BAD_PARAMETER_SIZE: + case PPSTACK_ERR_ARGUMENTS: + case PPSTACK_ERR_CONFIG: + psLogMsg("ppStack", PS_LOG_WARN, "Configuration error code: %x", errorCode); + exitValue = PS_EXIT_CONFIG_ERROR; + break; + case PPSTACK_ERR_PSF: + case PSPHOT_ERR_PSF: + case PM_ERR_STAMPS: + case PM_ERR_SMALL_AREA: + case PPSTACK_ERR_REJECTED: + case PPSTACK_ERR_DATA: + psLogMsg("ppStack", PS_LOG_WARN, "Data error code: %x", errorCode); + exitValue = PS_EXIT_DATA_ERROR; + break; + case PS_ERR_UNEXPECTED_NULL: + case PS_ERR_PROGRAMMING: + case PPSTACK_ERR_NOT_IMPLEMENTED: + case PPSTACK_ERR_PROG: + psLogMsg("ppStack", PS_LOG_WARN, "Programming error code: %x", errorCode); + exitValue = PS_EXIT_PROG_ERROR; + break; + default: + // It's a programming error if we're not dealing with the error correctly + psLogMsg("ppStack", PS_LOG_WARN, "Unrecognised error code: %x", errorCode); + exitValue = PS_EXIT_PROG_ERROR; + break; + } + return exitValue; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackJPEGs.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackJPEGs.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackJPEGs.c (revision 41840) @@ -0,0 +1,36 @@ +#include "ppStack.h" + +bool ppStackJPEGs(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + // Generate binned JPEGs + int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level + int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level + + // Target cells + pmFPAview *view = pmFPAviewAlloc(0); // View to cells of interest + view->chip = view->cell = 0; + pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG1"); + pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG2"); + psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask + psFree(view); + + pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts + if (!pmReadoutRebin(ro1, options->outRO, maskValue, bin1, bin1) || + !pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) { + psError(PPSTACK_ERR_DATA, false, "Unable to bin output."); + psFree(ro1); + psFree(ro2); + return false; + } + psFree(ro1); + psFree(ro2); + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackLoop.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackLoop.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackLoop.c (revision 41840) @@ -0,0 +1,296 @@ +#include "ppStack.h" + +// static functions are defined below +static int stackSummary(const ppStackOptions *options, const char *place); +bool pmConfigDumpRecipe (pmConfig *config, char *filename); + +bool ppStackLoop(pmConfig *config, ppStackOptions *options) +{ + assert(config); + + psTimerStart("PPSTACK_TOTAL"); + psTimerStart("PPSTACK_STEPS"); + + // Setup + psTrace("ppStack", 1, "Setup....\n"); + if (!ppStackSetup(options, config)) { + psError(psErrorCodeLast(), false, "Unable to setup."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 0: Setup: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("setup"); + + // Preparation for stacking + psTrace("ppStack", 1, "Preparation for stacking: merging sources, determining target PSF....\n"); + + if (!ppStackPrepare(options, config)) { + psError(psErrorCodeLast(), false, "Unable to prepare for stacking."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 1: Load Sources and Generate Target PSF: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("prepare"); + if (options->quality) return true; // Can't do anything else + + // Convolve inputs + psTrace("ppStack", 1, "Convolving inputs to target PSF....\n"); + if (!ppStackConvolve(options, config)) { + psError(psErrorCodeLast(), false, "Unable to convolve images."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 2: Generate Convolutions and Save: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("convolve"); + if (options->quality) return true; // Can't do anything else + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + + // Ensure sufficient inputs + { + int numGood = stackSummary(options, "initial combination"); + bool safe = psMetadataLookupBool(NULL, recipe, "SAFE"); // Be safe when combining + if (safe && numGood <= 1) { + options->quality = PPSTACK_ERR_REJECTED; + psErrorStackPrint(stderr, "Insufficient inputs for combination with safety on"); + psErrorClear(); + psWarning("Insufficient inputs for combination with safety on"); + return true; + } + } + + // Define threading elements + ppStackThreadData *stack = ppStackThreadDataSetup(options, config, true); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); + + // Prepare for combination + if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", options->doBackground ? "PPSTACK.OUTPUT.BKGMODEL" : NULL, PPSTACK_FILES_STACK, stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to prepare for combination."); + psFree(stack); + return false; + } + + // Initial combination + psTrace("ppStack", 1, "Initial stack of convolved images....\n"); + if (!ppStackCombineInitial(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to perform initial combination."); + psFree(stack); + return false; + } + ppStackMemDump("initial"); + psLogMsg("ppStack", PS_LOG_INFO, "Stage 3: Make Initial Stack: %f sec", psTimerClear("PPSTACK_STEPS")); + + // Done with stack inputs for now + // XXX is this where we are leaking?? + for (int i = 0; i < options->num; i++) { + pmCellFreeData(options->cells->data[i]); + } + //MEH -- must uncomment back out -- unclear is should be moved after pixel rejection + psFree(stack); + + // Pixel rejection + psTrace("ppStack", 1, "Reject pixels....\n"); + if (!ppStackReject(options, config)) { + psError(psErrorCodeLast(), false, "Unable to reject pixels."); + psFree(stack); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 4: Pixel Rejection: %f sec", psTimerClear("PPSTACK_STEPS")); + + // Check inputs + { + int numGood = stackSummary(options, "final combination"); + if (numGood <= 0) { + options->quality = PPSTACK_ERR_REJECTED; + psErrorStackPrint(stderr, "Insufficient inputs survived rejection stage"); + psErrorClear(); + psWarning("Insufficient inputs survived rejection stage"); + return true; + } + } + + stack = ppStackThreadDataSetup(options, config, true); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // Final combination. This one does NOT need to be normalized since the convolution takes care of that + psTrace("ppStack", 2, "Final stack of convolved images....\n"); + if (options->convolve) { + if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, false, true, false)) { + psError(psErrorCodeLast(), false, "Unable to perform final combination."); + psFree(stack); + return false; + } + } + else { + // Since we haven't convolved, I believe we do need to normalize here. + //MEH -- see below for comment on ppStackCombineFinal and bscaleOffset + if (!ppStackCombineFinal(stack, options->convCovars, options, config, false, true, true, true)) { + psError(psErrorCodeLast(), false, "Unable to perform final combination."); + psFree(stack); + return false; + } + } + + psLogMsg("ppStack", PS_LOG_INFO, "Stage 5: Final Stack: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("final"); + + // Update Header + // Do this before performing photometry so that the cmf header gets all of the information. + if (!ppStackUpdateHeader(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to update header."); + psFree(stack); + return false; + } + + // Generate median background stack here. + if (options->doBackground && !ppStackCombineBackground(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to generate median of background images."); + psFree(stack); + return false; + } + ppStackFileActivation(config,PPSTACK_FILES_BKG ,false); + // Photometry + psTrace("ppStack", 1, "Photometering stacked image....\n"); + if (!ppStackPhotometry(options, config)) { + psError(psErrorCodeLast(), false, "Unable to perform photometry."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("photometry"); + + // Create JPEGS + if (!ppStackJPEGs(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to make jpegs."); + psFree(stack); + return false; + } + // Assemble Stats + if (!ppStackStats(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to assemble statistics."); + psFree(stack); + return false; + } + + // Clean up + psTrace("ppStack", 2, "Cleaning up after combination....\n"); + if (!ppStackCleanupFiles(stack, options, config, PPSTACK_FILES_STACK, PPSTACK_FILES_PHOT, true)) { + psError(psErrorCodeLast(), false, "Unable to clean up."); + psFree(stack); + return false; + } + //MEH -- also must uncomment back out.. + psFree(stack); + psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: Cleanup, WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("cleanup"); + + // Unconvolved stack --- it's cheap to calculate, compared to everything else! + // XXX unconvolved stack is currently using the convolved mask! oops! + if (options->convolve) { + // Start threading + ppStackThreadData *stack = ppStackThreadDataSetup(options, config, false); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // Prepare for combination + if (!ppStackCombinePrepare("PPSTACK.UNCONV", "PPSTACK.UNCONV.EXP", NULL, PPSTACK_FILES_UNCONV, + stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to prepare for combination."); + psFree(stack); + return false; + } + + // generate the unconvolved stack. NOTE: this one must be normalized since the inputs have not been + psTrace("ppStack", 2, "Stack of unconvolved images....\n"); + //MEH -- terrible hack for bscale offset to input warps -- treat like normalization + // -- if BSCALEOFFSET TRUE, 0.5*bscale for offset values, otherwise value 0.0 + // -- bscaleOffset set when files read in and applied in CombineFinal (if arg true, only valid for unconv case) + if (!ppStackCombineFinal(stack, options->origCovars, options, config, false, true, false, true)) { + psError(psErrorCodeLast(), false, "Unable to perform unconvolved combination."); + psFree(stack); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Unconvolved Stack: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("unconv"); + + // Update Header + if (!ppStackUpdateHeader(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to update header."); + psFree(stack); + return false; + } + // Clean up unconvolved stack + psTrace("ppStack", 2, "Cleaning up after unconvolved stack....\n"); + if (!ppStackCleanupFiles(stack, options, config, PPSTACK_FILES_UNCONV, PPSTACK_FILES_NONE, false)) { + psError(psErrorCodeLast(), false, "Unable to clean up."); + psFree(stack); + return false; + } + psFree(stack); + } + psFree(options->cells); options->cells = NULL; + + // Finish up + psTrace("ppStack", 1, "Finishing up....\n"); + if (!ppStackFinish(options, config)) { + psError(psErrorCodeLast(), false, "Unable to finish up."); + return false; + } + ppStackMemDump("finish"); + + return true; +} + +/// Print a summary of the inputs, and return the number of good inputs +static int stackSummary(const ppStackOptions *options, const char *place) +{ + int numGood = 0; // Number of good inputs + psString summary = NULL; // Summary of images + for (int i = 0; i < options->num; i++) { + psString reason = NULL; // Reason for rejecting + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] == 0) { + psStringAppend(&reason, " Good."); + numGood++; + } else { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_CAL) { + psStringAppend(&reason, " Calibration failed."); + } + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_PSF) { + psStringAppend(&reason, " PSF measurement failed."); + } + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_MATCH) { + psStringAppend(&reason, " PSF matching failed."); + } + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_CHI2) { + psStringAppend(&reason, " PSF matching chi^2 deviant."); + } + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_REJECT) { + psStringAppend(&reason, " Rejection exceeded threshold."); + } + } + psStringAppend(&summary, "Image %d: %s\n", i, reason); + psFree(reason); + } + psLogMsg("ppStack", PS_LOG_INFO, "Summary of images for %s:\n%s", place, summary); + psFree(summary); + + return numGood; +} + +// Test function if needed: +bool pmConfigDumpRecipe (pmConfig *config, char *filename) { + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe for ppSim + psAssert (recipe, "oops"); + + psMetadataConfigWrite (recipe, filename, NULL); + + return true; +} + Index: /branches/eam_branches/ppStack.20211015/src/ppStackLoop.h =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackLoop.h (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackLoop.h (revision 41840) @@ -0,0 +1,98 @@ +#ifndef PPSTACK_LOOP_H +#define PPSTACK_LOOP_H + +// Loop over the inputs, doing the combination +bool ppStackLoop( + pmConfig *config, // Configuration + ppStackOptions *options // Options for stacking + ); +// Median only loop. +bool ppStackMedianLoop( + pmConfig *config, // Configuration + ppStackOptions *options // Options for stacking + ); + +// Setup +bool ppStackSetup( + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Preparation for stacking +// +// Merge input source lists, determine target PSF +bool ppStackPrepare( + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Convolve inputs to match target PSF +bool ppStackConvolve( + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Prepare for combination +bool ppStackCombinePrepare( + const char *outName, // Name of output file + const char *expName, // Name of exposure file + const char *bkgName, // Name of background file + ppStackFileList files, // Files of interest + ppStackThreadData *stack, // Stack + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Initial combination +bool ppStackCombineInitial( + ppStackThreadData *stack, // Stack + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Reject pixels +bool ppStackReject( + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Final combination +bool ppStackCombineFinal( + ppStackThreadData *stack, // Stack + psArray *covariances, // Covariances + ppStackOptions *options, // Options + pmConfig *config, // Configuration + bool safe, // Allow safe combination? + bool norm, // Normalise images? + bool grow, // Grow rejection masks? + bool bscaleoffset // Apply bscale offset? + ); + +// Cleanup following combination +bool ppStackCleanupFiles( + ppStackThreadData *stack, // Stack + ppStackOptions *options, // Options + pmConfig *config, // Configuration + ppStackFileList stackFiles, // cleanup these stack files + ppStackFileList photFiles, // cleanup these phot files (PHOT or NOP) + bool closeJPEGs // close the jpeg files? +); + +// Photometry +bool ppStackPhotometry( + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + +// Finish up +bool ppStackFinish( + ppStackOptions *options, // Options + pmConfig *config // Configuration + ); + + +bool ppStackUpdateHeader(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); +bool ppStackJPEGs(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); +bool ppStackStats(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config); + +#endif Index: /branches/eam_branches/ppStack.20211015/src/ppStackMatch.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackMatch.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackMatch.c (revision 41840) @@ -0,0 +1,607 @@ +#include "ppStack.h" + +#define ARRAY_BUFFER 16 // Number to add to array at a time +#define MAG_IGNORE 50 // Ignore magnitudes fainter than this --- they're not real! +#define FAKE_SIZE 1 // Size of fake convolution kernel +#define COVAR_FRAC 0.01 // Truncation fraction for covariance matrix + +// #define TESTING // Enable debugging output +// #define TESTING_REUSE_CONV // Enable debugging for re-using convolved outputs from previous run +// #define TESTING_CZW + +#ifdef TESTING_REUSE_CONV +// Read a FITS image +static bool readImage(psImage **target, // Target for image + const char *name, // Name of FITS file + const pmConfig *config // Configuration + ) +{ + psString resolved = pmConfigConvertFilename(name, config, false, false); // Resolved filename + psFits *fits = psFitsOpen(resolved, "r"); + psFree(resolved); + if (!fits) { + psError(PPSTACK_ERR_IO, false, "Unable to open previously produced image: %s", name); + return false; + } + psImage *image = psFitsReadImage(fits, psRegionSet(0,0,0,0), 0); // Image of interest + if (!image) { + psError(PPSTACK_ERR_IO, false, "Unable to read previously produced image: %s", name); + psFitsClose(fits); + return false; + } + psFitsClose(fits); + + psFree(*target); + *target = image; + + return true; +} +#endif + + +// Add background into the fake image +// Based on ppSubBackground() +static psImage *stackBackgroundModel(pmReadout *ro, // Readout for which to generate background model + const pmConfig *config // Configuration + ) +{ + psAssert(ro && ro->image, "Need readout image"); + psAssert(config, "Need configuration"); + + psImage *image = ro->image; // Image of interest + int numCols = image->numCols, numRows = image->numRows; // Size of image + + psMetadata *ppStackRecipe = psMetadataLookupPtr(NULL, config->recipes, PPSTACK_RECIPE); + psAssert(ppStackRecipe, "Need PPSTACK recipe"); + psMetadata *psphotRecipe = psMetadataLookupPtr(NULL, config->recipes, PSPHOT_RECIPE); + psAssert(psphotRecipe, "Need PSPHOT recipe"); + + psString maskBadStr = psMetadataLookupStr(NULL, ppStackRecipe, "MASK.BAD");// Name of bits to mask for bad + psMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psMetadataAddImageMask(psphotRecipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "user-defined mask", maskBad); + + psImage *binned = psphotModelBackgroundReadoutNoFile(ro, config); // Binned background model + psImageBinning *binning = psMetadataLookupPtr(NULL, ro->analysis, + "PSPHOT.BACKGROUND.BINNING"); // Binning for model + psAssert(binning, "Need binning parameters"); + psImage *unbinned = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Unbinned background model + if (!psImageUnbin(unbinned, binned, binning)) { + psError(PPSTACK_ERR_DATA, false, "Unable to unbin background model"); + psFree(binned); + psFree(unbinned); + return NULL; + } + psFree(binned); + + return unbinned; +} + +// Renormalise a readout's variance map +bool stackRenormaliseReadout(const pmConfig *config, // Configuration + pmReadout *readout // Readout to renormalise + ) +{ +#if 1 + bool mdok; // Status of metadata lookups + + psMetadata *recipe = psMetadataLookupPtr(NULL, config->recipes, PPSTACK_RECIPE); // Recipe for ppStack + psAssert(recipe, "Need PPSTACK recipe"); + + if (!psMetadataLookupBool(&mdok, recipe, "RENORM")) return true; + + int num = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); + if (!mdok) { + psError(PPSTACK_ERR_CONFIG, true, "RENORM.NUM is not set in the recipe"); + return false; + } + float minValid = psMetadataLookupF32(&mdok, recipe, "RENORM.MIN"); + if (!mdok) { + psError(PPSTACK_ERR_CONFIG, true, "RENORM.MIN is not set in the recipe"); + return false; + } + float maxValid = psMetadataLookupF32(&mdok, recipe, "RENORM.MAX"); + if (!mdok) { + psError(PPSTACK_ERR_CONFIG, true, "RENORM.MAX is not set in the recipe"); + return false; + } + + psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask + + psImageCovarianceTransfer(readout->variance, readout->covariance); + return pmReadoutVarianceRenormalise(readout, maskBad, num, minValid, maxValid); +#else + return true; +#endif +} + + + +bool ppStackMatch(pmReadout *readout, const psImage *target, ppStackOptions *options, + int index, const pmConfig *config) +{ + assert(readout); + assert(options); + assert(config); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + float deconvLimit = psMetadataLookupF32(NULL, recipe, "DECONV.LIMIT"); // Limit on deconvolution fraction + + // Look up appropriate values from the ppSub recipe + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + int size = psMetadataLookupS32(NULL, ppsub, "KERNEL.SIZE"); // Kernel half-size + + psString maskValStr = psMetadataLookupStr(NULL, ppsub, "MASK.VAL"); // Name of bits to mask going in + psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Bits to mask going in to pmSubtractionMatch + psString maskPoorStr = psMetadataLookupStr(NULL, recipe, "MASK.POOR"); // Name of bits to mask for poor + psImageMaskType maskPoor = pmConfigMaskGet(maskPoorStr, config); // Bits to mask for poor pixels + psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits to mask for bad + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + bool mdok; // Status of MD lookup + float penalty = psMetadataLookupF32(NULL, ppsub, "PENALTY"); // Penalty for wideness + int threads = psMetadataLookupS32(NULL, config->arguments, "NTHREADS"); // Number of threads + + // Replaced pmReadoutMaskNonfinite with pmReadoutMaskInvalid (tests for already masked pixels) + if (!pmReadoutMaskInvalid(readout, maskVal, maskBad)) { + psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels in readout."); + return false; + } + + // Match the PSF + if (options->convolve) { + pmReadout *conv = pmReadoutAlloc(NULL); // Conv readout, for holding results temporarily + + // MEH -- earlier defn needed with new simple mode modifications to use TESTING_REUSE_CONV... + //const char *typeStr = psMetadataLookupStr(NULL, ppsub, "KERNEL.TYPE"); // Kernel type + //pmSubtractionKernelsType type = pmSubtractionKernelsTypeFromString(typeStr); // Kernel type + +#ifdef TESTING_REUSE_CONV + // This is a hack to use the temporary convolved images and kernel generated previously. + // This makes the 'matching' operation much faster, allowing debugging of the stack process easier. + // It implicitly assumes the output root name is the same between invocations. + + // Read previously produced kernel + if (psMetadataLookupBool(NULL, config->arguments, "PPSTACK.DEBUG.STACK")) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.CONV.KERNEL", index); + psAssert(file, "Require file"); + + pmFPAview *view = pmFPAviewAlloc(0); // View to readout of interest + view->chip = view->cell = view->readout = 0; + psString filename = pmFPAfileNameFromRule(file->filerule, file, view); // Filename of interest + + // Read convolution kernel + psString resolved = pmConfigConvertFilename(filename, config, false, false); // Resolved filename + psFree(filename); + psFits *fits = psFitsOpen(resolved, "r"); // FITS file for subtraction kernel + psFree(resolved); + if (!fits || !pmReadoutReadSubtractionKernels(conv, fits)) { + psError(PPSTACK_ERR_IO, false, "Unable to read previously produced kernel"); + psFitsClose(fits); + return false; + } + psFitsClose(fits); + + if (!readImage(&readout->image, options->convImages->data[index], config) || + !readImage(&readout->mask, options->convMasks->data[index], config) || + !readImage(&readout->variance, options->convVariances->data[index], config)) { + psError(PPSTACK_ERR_IO, false, "Unable to read previously produced image."); + return false; + } + + psRegion *region = psMetadataLookupPtr(NULL, conv->analysis, + PM_SUBTRACTION_ANALYSIS_REGION); // Convolution region + pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, conv->analysis, + PM_SUBTRACTION_ANALYSIS_KERNEL); + + pmSubtractionAnalysis(conv->analysis, NULL, kernels, region, + readout->image->numCols, readout->image->numRows); + + psKernel *kernel = pmSubtractionKernel(kernels, 0.0, 0.0, false); // Convolution kernel + bool oldThreads = psImageCovarianceSetThreads(true); // Old thread setting + psKernel *covar = psImageCovarianceCalculate(kernel, readout->covariance); // Covariance matrix + psImageCovarianceSetThreads(oldThreads); + psFree(readout->covariance); + readout->covariance = covar; + psFree(kernel); + } else { +#endif + // Normal operations here + psAssert(target, "Require target PSF image"); + + int order = psMetadataLookupS32(NULL, ppsub, "SPATIAL.ORDER"); // Spatial polynomial order + float regionSize = psMetadataLookupF32(NULL, ppsub, "REGION.SIZE"); // Size of iso-kernel regs + float spacing = psMetadataLookupF32(NULL, ppsub, "STAMP.SPACING"); // Typical stamp spacing + int footprint = psMetadataLookupS32(NULL, ppsub, "STAMP.FOOTPRINT"); // Stamp half-size + float threshold = psMetadataLookupF32(NULL, ppsub, "STAMP.THRESHOLD"); // Threshold for stmps + int stride = psMetadataLookupS32(NULL, ppsub, "STRIDE"); // Size of convolution patches + int iter = psMetadataLookupS32(NULL, ppsub, "ITER"); // Rejection iterations + float rej = psMetadataLookupF32(NULL, ppsub, "REJ"); // Rejection threshold + float kernelError = psMetadataLookupF32(NULL, ppsub, "KERNEL.ERR"); // Relative systematic error in kernel + float normFrac = psMetadataLookupF32(NULL, ppsub, "NORM.FRAC"); // Fraction of window for normalisn windw + float sysError = psMetadataLookupF32(NULL, ppsub, "SYS.ERR"); // Relative systematic error in images + float skyErr = psMetadataLookupF32(NULL, ppsub, "SKY.ERR"); // Additional error in sky + float covarFrac = psMetadataLookupF32(NULL, ppsub, "COVAR.FRAC"); // Fraction for covariance calculation + + const char *typeStr = psMetadataLookupStr(NULL, ppsub, "KERNEL.TYPE"); // Kernel type + pmSubtractionKernelsType type = pmSubtractionKernelsTypeFromString(typeStr); // Kernel type + psVector *widths = psMetadataLookupPtr(NULL, ppsub, "ISIS.WIDTHS"); // ISIS Gaussian widths + psVector *orders = psMetadataLookupPtr(NULL, ppsub, "ISIS.ORDERS"); // ISIS Polynomial orders + int inner = psMetadataLookupS32(NULL, ppsub, "INNER"); // Inner radius + int ringsOrder = psMetadataLookupS32(NULL, ppsub, "RINGS.ORDER"); // RINGS polynomial order + int binning = psMetadataLookupS32(NULL, ppsub, "SPAM.BINNING"); // Binning for SPAM kernel + float badFrac = psMetadataLookupF32(NULL, ppsub, "BADFRAC"); // Maximum bad fraction + bool optimum = psMetadataLookupBool(&mdok, ppsub, "OPTIMUM"); // Derive optimum parameters? + float optMin = psMetadataLookupF32(&mdok, ppsub, "OPTIMUM.MIN"); // Minimum width for search + float optMax = psMetadataLookupF32(&mdok, ppsub, "OPTIMUM.MAX"); // Maximum width for search + float optStep = psMetadataLookupF32(&mdok, ppsub, "OPTIMUM.STEP"); // Step for search + float optThresh = psMetadataLookupF32(&mdok, ppsub, "OPTIMUM.TOL"); // Tolerance for search + int optOrder = psMetadataLookupS32(&mdok, ppsub, "OPTIMUM.ORDER"); // Order for search + float poorFrac = psMetadataLookupF32(&mdok, ppsub, "POOR.FRACTION"); // Fraction for "poor" + + bool scale = psMetadataLookupBool(NULL, ppsub, "SCALE"); // Scale kernel parameters? + float scaleRef = psMetadataLookupF32(NULL, ppsub, "SCALE.REF"); // Reference for scaling + float scaleMin = psMetadataLookupF32(NULL, ppsub, "SCALE.MIN"); // Minimum for scaling + float scaleMax = psMetadataLookupF32(NULL, ppsub, "SCALE.MAX"); // Maximum for scaling + if (!isfinite(scaleRef) || !isfinite(scaleMin) || !isfinite(scaleMax)) { + psError(PPSTACK_ERR_CONFIG, false, + "Scale parameters (SCALE.REF=%f, SCALE.MIN=%f, SCALE.MAX=%f) not set in PPSUB recipe.", + scaleRef, scaleMin, scaleMax); + return false; + } + + + // These values are specified specifically for stacking + const char *stampsName = psMetadataLookupStr(NULL, config->arguments, "STAMPS");// Stamps filename + + psVector *optWidths = NULL; // Vector with FWHMs for optimum search + if (optimum) { + optWidths = psVectorCreate(optWidths, optMin, optMax, optStep, PS_TYPE_F32); + } + + pmReadout *fake = pmReadoutAlloc(NULL); + fake->image = psImageCopy(NULL, target, PS_TYPE_F32); + fake->mask = psImageCopy(NULL, readout->mask, PS_TYPE_IMAGE_MASK); + +#if 1 + // Add the background into the target image + psImage *bgImage = stackBackgroundModel(readout, config); // Image of background + psBinaryOp(fake->image, fake->image, "+", bgImage); + psFree(bgImage); +#endif + +#ifdef TESTING_CZW + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/fake_%03d.fits", index); + pmStackVisualPlotTestImage(fake->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, fake->image, 0, NULL); + psFitsClose(fits); + } + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/real_%03d.fits", index); + pmStackVisualPlotTestImage(readout->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, readout->image, 0, NULL); + psFitsClose(fits); + } + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/real_var_%03d.fits", index); + pmStackVisualPlotTestImage(readout->variance, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, readout->variance, 0, NULL); + psFitsClose(fits); + } +#endif + + if (threads > 0) { + pmSubtractionThreadsInit(); + } + + // Do the image matching + pmSubtractionKernels *kernel = psMetadataLookupPtr(&mdok, readout->analysis, + PM_SUBTRACTION_ANALYSIS_KERNEL); // Conv kernel + if (kernel) { + if (!pmSubtractionMatchPrecalc(NULL, conv, fake, readout, readout->analysis, + stride, kernelError, covarFrac, maskVal, maskBad, maskPoor, + poorFrac, badFrac)) { + psError(psErrorCodeLast(), false, "Unable to convolve images."); + psFree(fake); + psFree(optWidths); + psFree(conv); + if (threads > 0) { + pmSubtractionThreadsFinalize(); + } + return false; + } + } else { + // we need to register the FWHM values for use downstream + pmSubtractionSetFWHMs(options->targetSeeing, options->inputSeeing->data.F32[index]); + + // Scale the input parameters + psVector *widthsCopy = psVectorCopy(NULL, widths, PS_TYPE_F32); // Copy of kernel widths + + pmSubtractionParamScaleOptions(scale, scaleRef, scaleMin, scaleMax); + + // XXX EAM : the kernel scaling process has changed: the scale is now set + // inside pmSubtractionMatch after the normalization window is measured + + // if (scale && !pmSubtractionParamsScale(&size, &footprint, widthsCopy, scaleRef, scaleMin, scaleMax)) { + // psError(psErrorCodeLast(), false, "Unable to scale kernel parameters"); + // psFree(fake); + // psFree(optWidths); + // psFree(conv); + // psFree(widthsCopy); + // if (threads > 0) { + // pmSubtractionThreadsFinalize(); + // } + // return false; + // } + + if (!pmSubtractionMatch(NULL, conv, fake, readout, footprint, stride, regionSize, spacing, + threshold, options->sources, stampsName, type, size, order, widthsCopy, + orders, inner, ringsOrder, binning, penalty, + optimum, optWidths, optOrder, optThresh, iter, rej, normFrac, + sysError, skyErr, kernelError, covarFrac, maskVal, maskBad, maskPoor, + poorFrac, badFrac, PM_SUBTRACTION_MODE_2)) { + psError(psErrorCodeLast(), false, "Unable to match images."); + psFree(fake); + psFree(optWidths); + psFree(conv); + psFree(widthsCopy); + if (threads > 0) { + pmSubtractionThreadsFinalize(); + } + return false; + } + psFree(widthsCopy); + } + + +#ifdef TESTING_CZW + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/conv_%03d.fits", index); + pmStackVisualPlotTestImage(conv->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, conv->image, 0, NULL); + psFitsClose(fits); + } + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/conv_var_%03d.fits", index); + pmStackVisualPlotTestImage(conv->variance, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, conv->variance, 0, NULL); + psFitsClose(fits); + } + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "/local/tmp/diff_%03d.fits", index); + pmStackVisualPlotTestImage(fake->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psBinaryOp(fake->image, conv->image, "-", fake->image); + psFitsWriteImage(fits, hdu->header, fake->image, 0, NULL); + psFitsClose(fits); + } +#endif + + psFree(fake); + psFree(optWidths); + + if (threads > 0) { + pmSubtractionThreadsFinalize(); + } + + // Replace original images with convolved + psFree(readout->image); + psFree(readout->mask); + psFree(readout->variance); + psFree(readout->covariance); + readout->image = psMemIncrRefCounter(conv->image); + readout->mask = psMemIncrRefCounter(conv->mask); + readout->variance = psMemIncrRefCounter(conv->variance); + readout->covariance = psImageCovarianceTruncate(conv->covariance, COVAR_FRAC); +#ifdef TESTING_REUSE_CONV + } +#endif + + // Extract the regions and solutions used in the image matching + // This stops them from being freed when we iterate back up the FPA + psArray *regions = options->regions->data[index] = psArrayAllocEmpty(ARRAY_BUFFER); // Match regions + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_REGION); + psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_REGION); + regions = psArrayAdd(regions, ARRAY_BUFFER, item->data.V); + } + psFree(iter); + } + psArray *kernels = options->kernels->data[index] = psArrayAllocEmpty(ARRAY_BUFFER); // Match kernels + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_UNKNOWN); + pmSubtractionKernels *kernel = item->data.V; // Kernel used in subtraction + kernels = psArrayAdd(kernels, ARRAY_BUFFER, kernel); + } + psFree(iter); + } + psAssert((regions)->n == (kernels)->n, "Number of match regions and kernels should match"); + + // Record chi^2 + { + double sum = 0.0; // Sum of chi^2 + int num = 0; // Number of measurements of chi^2 + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_UNKNOWN); + pmSubtractionKernels *kernels = item->data.V; // Convolution kernels + // EAM/MEH sum the RMS instead of the mean now + sum += kernels->rms; + num++; + // psLogMsg("ppStack", PS_LOG_INFO, "KERNMATCHDUMP %d %d %g %g %g %g\n", + // index,num,kernels->mean,kernels->rms,sum,psImageCovarianceFactor(readout->covariance)); + } + psFree(iter); + + // Apply the covariance factor to the chi^2 value + double covar_factor = 1.0; + if (type != PM_SUBTRACTION_KERNEL_SIMPLE) { // Except if we're using simple kernels. + covar_factor = psImageCovarianceFactor(readout->covariance); + } + options->matchChi2->data.F32[index] = sum / (covar_factor * num); + + } + + // Kernel normalisation + { + double sum = 0.0; // Sum of chi^2 + int num = 0; // Number of measurements of chi^2 + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_NORM); + psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_F32); + float norm = item->data.F32; // Normalisation + sum += norm; + num++; + } + psFree(iter); + float conv = sum/num; // Mean normalisation from convolution + float stars = powf(10.0, -0.4 * options->norm->data.F32[index]); // Normalisation from stars + float renorm = stars / conv; // Renormalisation to apply + psLogMsg("ppStack", PS_LOG_INFO, "Renormalising image %d by %f (kernel: %f, stars: %f)\n", + index, renorm, conv, stars); + psBinaryOp(readout->image, readout->image, "*", psScalarAlloc(renorm, PS_TYPE_F32)); + psBinaryOp(readout->variance, readout->variance, "*", psScalarAlloc(PS_SQR(renorm), PS_TYPE_F32)); + } + + // Reject image completely if the maximum deconvolution fraction exceeds the limit + float deconv = psMetadataLookupF32(NULL, conv->analysis, + PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Max deconvolution fraction + if (deconv > deconvLimit) { + psWarning("Maximum deconvolution fraction (%f) exceeds limit (%f) --- rejecting image %d\n", + deconv, deconvLimit, index); + psFree(conv); + return NULL; + } + + readout->analysis = psMetadataCopy(readout->analysis, conv->analysis); + + psFree(conv); + } else { + // Match the normalisation + float norm = powf(10.0, -0.4 * options->norm->data.F32[index]); // Normalisation + psBinaryOp(readout->image, readout->image, "*", psScalarAlloc(norm, PS_TYPE_F32)); + psBinaryOp(readout->variance, readout->variance, "*", psScalarAlloc(PS_SQR(norm), PS_TYPE_F32)); + } + + // Ensure the background value is zero + psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for background + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(bg, NULL, readout->image, readout->mask, maskVal | maskBad, rng)) { + psWarning("Can't measure background for image."); + psErrorClear(); + } else { + if (!psMetadataLookupBool(NULL, config->arguments, "PPSTACK.SKIP.BG.SUB")) { + psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)", + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV)); + (void)psBinaryOp(readout->image, readout->image, "-", + psScalarAlloc(psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32)); + } + } + +#ifdef TESTING_CZW + // CZW begin check of weight before renormalize step + // Measure the variance level for the weighting + if (psMetadataLookupBool(NULL, recipe, "WEIGHTS")) { + if (!psImageBackground(bg, NULL, readout->variance, readout->mask, maskVal | maskBad, rng)) { + psError(PPSTACK_ERR_DATA, false, "Can't measure mean variance for image."); + psFree(rng); + psFree(bg); + return false; + } + options->weightings->data.F32[index] = 1.0 / (psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN) * + psImageCovarianceFactor(readout->covariance)); + } else { + options->weightings->data.F32[index] = 1.0; + } + psLogMsg("ppStack", PS_LOG_INFO, "Prenorm Weighting for image %d is %g covar %g median %g\n", + index, options->weightings->data.F32[index],psImageCovarianceFactor(readout->covariance), + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN)); + // CZW end +#endif + if (!stackRenormaliseReadout(config, readout)) { + psFree(rng); + psFree(bg); + return false; + } + + // Measure the variance level for the weighting + if (psMetadataLookupBool(NULL, recipe, "WEIGHTS")) { + if (!psImageBackground(bg, NULL, readout->variance, readout->mask, maskVal | maskBad, rng)) { + psError(PPSTACK_ERR_DATA, false, "Can't measure mean variance for image."); + psFree(rng); + psFree(bg); + return false; + } + options->weightings->data.F32[index] = 1.0 / (psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN) * + psImageCovarianceFactor(readout->covariance)); + } else { + options->weightings->data.F32[index] = 1.0; + } + psLogMsg("ppStack", PS_LOG_INFO, "Weighting for image %d is %g covar %g median %g\n", + index, options->weightings->data.F32[index],psImageCovarianceFactor(readout->covariance), + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN)); + + psFree(rng); + psFree(bg); + +#ifdef TESTING + { + pmHDU *hdu = pmHDUFromCell(readout->parent); + psString name = NULL; + psStringAppend(&name, "convolved_%03d.fits", index); + pmStackVisualPlotTestImage(readout->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, readout->image, 0, NULL); + psFitsClose(fits); + } +#endif + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackMedian.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackMedian.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackMedian.c (revision 41840) @@ -0,0 +1,51 @@ +#include "ppStack.h" + +int main(int argc, char *argv[]) +{ + psLibInit(NULL); + psTimerStart("PPSTACK"); + psTimerStart("PPSTACK_STEPS"); + + pmErrorRegister(); + ppStackErrorRegister(); + psphotErrorRegister(); + + ppStackOptions *options = NULL; // Options for stacking + + pmConfig *config = pmConfigRead(&argc, argv, PPSTACK_RECIPE); // Configuration + if (!config) { + ppStackCleanup(config, options); + } + + ppStackVersionPrint(); + + if (!pmModelClassInit()) { + psError(PPSTACK_ERR_PROG, false, "Unable to initialise model classes."); + ppStackCleanup(config, options); + } + + if (!psphotInit()) { + psError(PPSTACK_ERR_PROG, false, "Error initialising psphot."); + ppStackCleanup(config, options); + } + + if (!ppStackArgumentsSetup(argc, argv, config)) { + ppStackCleanup(config, options); + } + + if (!ppStackCamera(config)) { + ppStackCleanup(config, options); + } + + if (!ppStackArgumentsParse(config)) { + ppStackCleanup(config, options); + } + + options = ppStackOptionsAlloc(); + if (!ppStackMedianLoop(config, options)) { + ppStackCleanup(config, options); + } + + ppStackCleanup(config, options); +} + Index: /branches/eam_branches/ppStack.20211015/src/ppStackMedianLoop.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackMedianLoop.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackMedianLoop.c (revision 41840) @@ -0,0 +1,204 @@ +#include "ppStack.h" + +// static functions are defined below +/* static int stackSummary(const ppStackOptions *options, const char *place); */ + +bool ppStackMedianLoop(pmConfig *config, ppStackOptions *options) +{ + assert(config); + + psTimerStart("PPSTACK_TOTAL"); + psTimerStart("PPSTACK_STEPS"); + + // Setup + psTrace("ppStack", 1, "Setup....\n"); + if (!ppStackSetup(options, config)) { + psError(psErrorCodeLast(), false, "Unable to setup."); + return false; + } + psLogMsg("ppStack", PS_LOG_INFO, "Stage 0: Setup: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("setup"); + + // Duplicate code that is in ppStackConvolve + options->cells = psArrayAlloc(options->num); // Cells for convolved images --- a handle for reading again + pmFPAview *view = ppStackFilesIterateDown(config); + for (int i = 0; i < options->num; i++) { + ppStackFileActivationSingle(config, PPSTACK_FILES_MEDIAN_IN, true, i); + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest + + // pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy + if (!view) { + return false; + } + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); // Input readout + pmCell *inCell = readout->parent; // Input cell */ + options->cells->data[i] = psMemIncrRefCounter(inCell); + + } + psFree(view); + + ppStackThreadData *stack = ppStackThreadDataSetup(options, config, options->convolve); + if (!stack) { + psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); + return false; + } + + // Preparation for stacking + if (!ppStackCombinePrepare("PPSTACK.OUTPUT",NULL,NULL,PPSTACK_FILES_MEDIAN_OUT,stack,options,config)) { + psError(psErrorCodeLast(), false, "Unabel to combine data."); + psFree(stack); + return false; + } + + if (!ppStackCombineMedian(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unabel to combine data."); + psFree(stack); + return false; + } +/* for (int x = 0; x < options->outRO->image->numCols; x++) { */ +/* for (int y = 0; y < options->outRO->image->numRows; y++) { */ +/* options->outRO->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0; */ +/* } */ +/* } */ + psFree(options->outRO->mask); + // Do this before performing photometry so that the cmf header gets all of the information. + options->zpInput = NULL; + options->expTimeInput = NULL; + options->airmassInput = NULL; + if (!ppStackUpdateHeader(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to update header."); + psFree(stack); + return false; + } +/* // Generate median background stack here. */ +/* if (!ppStackCombineBackground(stack, options, config)) { */ +/* psError(psErrorCodeLast(), false, "Unable to generate median of background images."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ +// ppStackFileActivation(config,PPSTACK_FILES_BKG ,false); +/* // Photometry */ +/* psTrace("ppStack", 1, "Photometering stacked image....\n"); */ +/* if (!ppStackPhotometry(options, config)) { */ +/* psError(psErrorCodeLast(), false, "Unable to perform photometry."); */ +/* return false; */ +/* } */ +/* psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS")); */ +/* ppStackMemDump("photometry"); */ + + // Create JPEGS +/* if (!ppStackJPEGs(stack, options, config)) { */ +/* psError(psErrorCodeLast(), false, "Unable to make jpegs."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ +/* // Assemble Stats */ +/* if (!ppStackStats(stack, options, config)) { */ +/* psError(psErrorCodeLast(), false, "Unable to assemble statistics."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ + + // Clean up + + psTrace("ppStack", 2, "Cleaning up after combination....\n"); + if (!ppStackCleanupFiles(stack, options, config, PPSTACK_FILES_MEDIAN_OUT, PPSTACK_FILES_MEDIAN_OUT, false)) { + psError(psErrorCodeLast(), false, "Unable to clean up."); + psFree(stack); + return false; + } + // psFree(stack); + psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: Cleanup, WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS")); + ppStackMemDump("cleanup"); + +/* // Unconvolved stack --- it's cheap to calculate, compared to everything else! */ +/* // XXX unconvolved stack is currently using the convolved mask! oops! */ +/* if (options->convolve) { */ +/* // Start threading */ +/* ppStackThreadData *stack = ppStackThreadDataSetup(options, config, false); */ +/* if (!stack) { */ +/* psError(psErrorCodeLast(), false, "Unable to initialise stack threads."); */ +/* return false; */ +/* } */ + +/* // Prepare for combination */ +/* if (!ppStackCombinePrepare("PPSTACK.UNCONV", "PPSTACK.UNCONV.EXP", NULL, PPSTACK_FILES_UNCONV, */ +/* stack, options, config)) { */ +/* psError(psErrorCodeLast(), false, "Unable to prepare for combination."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ + +/* // generate the unconvolved stack. NOTE: this one must be normalized since the inputs have not been */ +/* psTrace("ppStack", 2, "Stack of unconvolved images....\n"); */ +/* if (!ppStackCombineFinal(stack, options->origCovars, options, config, false, true, false)) { */ +/* psError(psErrorCodeLast(), false, "Unable to perform unconvolved combination."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ +/* psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Unconvolved Stack: %f sec", psTimerClear("PPSTACK_STEPS")); */ +/* ppStackMemDump("unconv"); */ + +/* // Update Header */ +/* if (!ppStackUpdateHeader(stack, options, config)) { */ +/* psError(psErrorCodeLast(), false, "Unable to update header."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ +/* // Clean up unconvolved stack */ +/* psTrace("ppStack", 2, "Cleaning up after unconvolved stack....\n"); */ +/* if (!ppStackCleanupFiles(stack, options, config, PPSTACK_FILES_UNCONV, PPSTACK_FILES_NONE, false)) { */ +/* psError(psErrorCodeLast(), false, "Unable to clean up."); */ +/* psFree(stack); */ +/* return false; */ +/* } */ +/* psFree(stack); */ +/* } */ + psFree(options->cells); options->cells = NULL; + + // Finish up + psTrace("ppStack", 1, "Finishing up....\n"); + if (!ppStackFinish(options, config)) { + psError(psErrorCodeLast(), false, "Unable to finish up."); + return false; + } + ppStackMemDump("finish"); + + return true; +} + +/* /// Print a summary of the inputs, and return the number of good inputs */ +/* static int stackSummary(const ppStackOptions *options, const char *place) */ +/* { */ +/* int numGood = 0; // Number of good inputs */ +/* psString summary = NULL; // Summary of images */ +/* for (int i = 0; i < options->num; i++) { */ +/* psString reason = NULL; // Reason for rejecting */ +/* if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] == 0) { */ +/* psStringAppend(&reason, " Good."); */ +/* numGood++; */ +/* } else { */ +/* if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_CAL) { */ +/* psStringAppend(&reason, " Calibration failed."); */ +/* } */ +/* if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_PSF) { */ +/* psStringAppend(&reason, " PSF measurement failed."); */ +/* } */ +/* if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_MATCH) { */ +/* psStringAppend(&reason, " PSF matching failed."); */ +/* } */ +/* if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_CHI2) { */ +/* psStringAppend(&reason, " PSF matching chi^2 deviant."); */ +/* } */ +/* if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_REJECT) { */ +/* psStringAppend(&reason, " Rejection exceeded threshold."); */ +/* } */ +/* } */ +/* psStringAppend(&summary, "Image %d: %s\n", i, reason); */ +/* psFree(reason); */ +/* } */ +/* psLogMsg("ppStack", PS_LOG_INFO, "Summary of images for %s:\n%s", place, summary); */ +/* psFree(summary); */ + +/* return numGood; */ +/* } */ Index: /branches/eam_branches/ppStack.20211015/src/ppStackOptions.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackOptions.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackOptions.c (revision 41840) @@ -0,0 +1,85 @@ +#include "ppStack.h" + +static void stackOptionsFree(ppStackOptions *options) +{ + psFree(options->stats); + if (options->statsFile) { + fclose(options->statsFile); + } + psFree(options->origImages); + psFree(options->origMasks); + psFree(options->origVariances); + psFree(options->origCovars); + psFree(options->convImages); + psFree(options->convMasks); + psFree(options->convVariances); + psFree(options->psf); + psFree(options->inputSeeing); + psFree(options->exposures); + psFree(options->inputMask); + psFree(options->sourceLists); + psFree(options->bscaleOffset); + psFree(options->norm); + psFree(options->sources); + psFree(options->cells); + psFree(options->kernels); + psFree(options->regions); + psFree(options->matchChi2); + psFree(options->weightings); + psFree(options->convCovars); + psFree(options->outRO); + psFree(options->expRO); + psFree(options->inspect); + psFree(options->rejected); + + return; +} + +ppStackOptions *ppStackOptionsAlloc(void) +{ + ppStackOptions *options = psAlloc(sizeof(ppStackOptions)); // Options, to return + psMemSetDeallocator(options, (psFreeFunc)stackOptionsFree); + + options->convolve = true; + options->matchZPs = true; + options->photometry = false; + options->doBackground = false; + options->stats = NULL; + options->statsFile = NULL; + options->origImages = NULL; + options->origMasks = NULL; + options->origVariances = NULL; + options->origCovars = NULL; + options->convImages = NULL; + options->convMasks = NULL; + options->convVariances = NULL; + options->num = 0; + options->quality = 0; + options->psf = NULL; + options->sumExposure = NAN; + options->zp = NAN; + options->inputSeeing = NULL; + options->exposures = NULL; + options->targetSeeing = NAN; + options->clippedMean = NAN; + options->clippedStdev = NAN; + options->inputMask = NULL; + options->sourceLists = NULL; + options->bscaleOffset = NULL; + options->norm = NULL; + options->sources = NULL; + options->cells = NULL; + options->kernels = NULL; + options->regions = NULL; + options->numCols = 0; + options->numRows = 0; + options->matchChi2 = NULL; + options->weightings = NULL; + options->convCovars = NULL; + options->outRO = NULL; + options->expRO = NULL; + options->inspect = NULL; + options->rejected = NULL; + + return options; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackOptions.h =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackOptions.h (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackOptions.h (revision 41840) @@ -0,0 +1,61 @@ +#ifndef PPSTACK_OPTIONS_H +#define PPSTACK_OPTIONS_H + +/// Options for stacking process +typedef struct { + // Setup + bool convolve; // Convolve images? + bool matchZPs; // Adjust relative fluxes based on transparency analysis? + bool photometry; // Perform photometry? + bool doBackground; // Do background model combination? + psMetadata *stats; // Statistics for output + FILE *statsFile; // File to which to write statistics + psArray *origImages, *origMasks, *origVariances; // Filenames of the original images + psArray *convImages, *convMasks, *convVariances; // Filenames for the temporary convolved images + psArray *origCovars; // Original covariances matrices + int num; // Number of inputs + int quality; // Bad data quality flag + // Prepare + pmPSF *psf; // Target PSF + psVector *inputSeeing; // Input seeing FWHMs + float targetSeeing; // Target seeing FWHM + psVector *exposures; // Exposure times + float sumExposure; // Sum of exposure times + float zp; // Zero point for output + float zpErr; // Zero point error for output + float airmass; // airmass for output image + float airmassSlope; // airmass slope used for zp analysis + psVector *inputMask; // Mask for inputs + psArray *sourceLists; // Individual lists of sources for matching + psVector *norm; // Normalisation for each image + psVector *zpInput; // reported zero point of input exposure + psVector *expTimeInput; // exposure time of input exposure + psVector *airmassInput; // airmass of input exposure + psArray *sources; // Matched sources + float clippedMean; // clipped mean of input fwhm + float clippedStdev; // clipped stdev of input fwhm + psVector *bscaleOffset; // hack offset of bzero using 0.5*bscale if option, 0.0 if not + // Convolve + psArray *cells; // Cells for convolved images --- a handle for reading again + psArray *kernels; // PSF-matching kernels --- required in the stacking + psArray *regions; // PSF-matching regions --- required in the stacking + int numCols, numRows; // Size of image + psVector *matchChi2; // chi^2 for stamps from matching + psVector *weightings; // Combination weightings for images (1/noise^2) + psArray *convCovars; // Convolved covariance matrices + // Combine initial + pmReadout *outRO; // Output readout + pmReadout *expRO; // Exposure readout + psArray *inspect; // Array of arrays of pixels to inspect + // Rejection + psArray *rejected; // Rejected pixels + // Background + pmReadout *bkgRO; // Output background readout + psArray *bkgImages; // Input background images +} ppStackOptions; + +/// Allocator +ppStackOptions *ppStackOptionsAlloc(void); + + +#endif Index: /branches/eam_branches/ppStack.20211015/src/ppStackPSF.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackPSF.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackPSF.c (revision 41840) @@ -0,0 +1,85 @@ +#include "ppStack.h" + +//#define TESTING + +pmPSF *ppStackPSF(const pmConfig *config, int numCols, int numRows, + const psArray *psfs, ppStackOptions *options) +{ + bool mdok = false; + pmPSF *psf = NULL; + + const psVector *inputMask = options->inputMask; + + // Get the recipe values + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + bool autosize = psMetadataLookupBool(&mdok, recipe, "PSF.AUTOSIZE"); // Spatial order for PSF + if (!mdok) { + // older config files which lack PSF.AUTOSIZE used TRUE as the default + autosize = true; + } + + char *psfModel = psMetadataLookupStr(NULL, recipe, "PSF.MODEL"); // Model for PSF + + if (autosize) { + + int psfInstances = psMetadataLookupS32(NULL, recipe, "PSF.INSTANCES"); // Number of instances for PSF + float psfRadius = psMetadataLookupF32(NULL, recipe, "PSF.RADIUS"); // Radius for PSF + int psfOrder = psMetadataLookupS32(NULL, recipe, "PSF.ORDER"); // Spatial order for PSF + + psString maskValStr = psMetadataLookupStr(&mdok, recipe, "MASK.VAL"); // Name of bits to mask going in + if (!mdok || !maskValStr) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find MASK.VAL in recipe"); + return NULL; + } + psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Bits to mask + + for (int i = 0; i < psfs->n; i++) { + if (inputMask->data.U8[i]) { + psFree(psfs->data[i]); + psfs->data[i] = NULL; + } + } + + // Solve for the target PSF + psf = pmPSFEnvelope(numCols, numRows, psfs, psfInstances, psfRadius, psfModel, psfOrder, psfOrder, maskVal); + if (!psf) { + psError(PPSTACK_ERR_PSF, false, "Unable to determine output PSF."); + return NULL; + } + } else { + + // Manually defined target PSF + float psfFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.OUTPUT.FWHM"); // Radius for PSF + if (!mdok) { + psfFWHM = 4.0; + } + + //MEH -- preliminary add option -1 for choosing mean FWHM of inputs. Nsig<0 for scaling +Nsig by mean + if (psfFWHM<0.0){ + float NpsfFWHMsig = psMetadataLookupF32(&mdok, recipe, "PSF.OUTPUT.NFWHMSIG"); // Nsig to include + if (!mdok){ + NpsfFWHMsig = 0.0; + } + float clippedMean = options->clippedMean; + float clippedStdev = options->clippedStdev; + psfFWHM = clippedMean + NpsfFWHMsig*clippedStdev; + if (NpsfFWHMsig<0.0){ + psfFWHM = clippedMean + NpsfFWHMsig * -1.0*clippedStdev/clippedMean; + } + } + + + + float Mxx = M_SQRT2 * psfFWHM / 2.35; + + psf = pmPSFBuildSimple(psfModel, Mxx, Mxx, 0.0, 0.0); + if (!psf) { + psError(PPSTACK_ERR_PSF, false, "Unable to build dummy PSF."); + return NULL; + } + } + + return psf; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackPhotometry.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackPhotometry.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackPhotometry.c (revision 41840) @@ -0,0 +1,101 @@ +#include "ppStack.h" + +bool ppStackPhotometry(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + if (!options->photometry) { + // Nothing to do + return true; + } + + psTimerStart("PPSTACK_PHOT"); + + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivation(config, PPSTACK_FILES_PHOT, true); + pmFPAview *photView = ppStackFilesIterateDown(config); // View to readout + ppStackFileActivation(config, PPSTACK_FILES_STACK, true); + + pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); // File for photometry + pmFPACopy(photFile->fpa, options->outRO->parent->parent->parent); + + if (psMetadataLookupBool(NULL, config->arguments, "-visual")) { + pmVisualSetVisual(true); + } + + psMetadata *psphot = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe + +#if 0 + // Need to ensure aperture residual is not calculated + psMetadataItem *item = psMetadataLookup(recipe, "MEASURE.APTREND"); // Item determining aptrend + if (!item) { + psWarning("Unable to find MEASURE.APTREND in psphot recipe"); + psErrorClear(); + } else { + item->data.B = false; + } +#endif + + // set maskValue and markValue in the psphot recipe + psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask + psImageMaskType markValue = pmConfigMaskGet("MARK.VALUE", config); // Bits to use for marking + psMetadataAddImageMask(psphot, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "Bits to mask", maskValue); + psMetadataAddImageMask(psphot, PS_LIST_TAIL, "MARK.PSPHOT", PS_META_REPLACE, "Bits to use for mark", markValue); + + psArray *inSources = options->sources; + if (!inSources) { + psError(PPSTACK_ERR_PROG, false, "Unable to find input sources"); + psFree(photView); + return false; + } + + // Add the expnum image to the photometry readout's analysis metadata so that it is available to + // psphotSetNFrames + pmReadout *photRO = pmFPAviewThisReadout(photView, photFile->fpa); + if (photRO && options->expRO->mask) { + psMetadataAddImage(photRO->analysis, PS_LIST_TAIL, "EXPNUM", PS_META_REPLACE, "EXPNUM image", options->expRO->mask); + } + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + if (!psphotReadoutKnownSources(config, photView, "PSPHOT.INPUT", inSources)) { + // This is likely a data quality issue + // XXX Split into multiple cases using error codes? + psErrorStackPrint(stderr, "Unable to perform photometry on image"); + psWarning("Unable to perform photometry on image --- suspect bad data quality."); + if (options->quality == 0) { + options->quality = psErrorCodeLast(); + } + psErrorClear(); + psphotFilesActivate(config, false); + } + + if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") || + !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") || + !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) { + psError(PPSTACK_ERR_PROG, false, "Unable to drop PSPHOT internal files."); + return false; + } + + pmFPAfileActivate(config->files, false, "PSPHOT.INPUT"); + + if (options->stats) { + pmReadout *photRO = pmFPAviewThisReadout(photView, photFile->fpa); // Readout with the sources + pmDetections *detections = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.DETECTIONS"); // detections + if (detections && detections->allSources) { + psMetadataAddS32(options->stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", detections->allSources->n); + } else { + psMetadataAddS32(options->stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", 0); + } + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_PHOT", PS_META_REPLACE, "Time to do photometry", psTimerMark("PPSTACK_PHOT")); + } + + psFree(photView); + + psLogMsg("ppStack", PS_LOG_INFO, "Time to do photometry: %f sec", psTimerClear("PPSTACK_PHOT")); + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackPrepare.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackPrepare.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackPrepare.c (revision 41840) @@ -0,0 +1,531 @@ +# include "ppStack.h" + + +// Option to use the old FWHM rejection code. +# define NEW_REJECTION +// #define OLD_REJECTION + + +# define RE_PHOTOMETER 0 + +# define PHOT_SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_SATSTAR | PM_SOURCE_MODE_BLEND | \ + PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_SATURATED | \ + PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_EXT_LIMIT) // Mask to apply to sources + +bool ppStackInputPhotometer(const pmReadout *ro, const psArray *sources, const pmConfig *config) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PS_ASSERT_ARRAY_NON_NULL(sources, false); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + +# if (RE_PHOTOMETER) + bool mdok; // Status of MD lookup + + float zpRadius = psMetadataLookupS32(&mdok, recipe, "PHOT.RADIUS"); // Radius for PHOT measurement + if (!mdok) { + psError(PPSTACK_ERR_CONFIG, true, "Unable to find PHOT.RADIUS in recipe"); + return false; + } + float zpSigma = psMetadataLookupF32(&mdok, recipe, "PHOT.SIGMA"); // Gaussian sigma for photometry + if (!mdok) { + psError(PPSTACK_ERR_CONFIG, true, "Unable to find PHOT.SIGMA in recipe"); + return false; + } + float zpFrac = psMetadataLookupF32(&mdok, recipe, "PHOT.FRAC"); // Fraction of good pixels for photometry + if (!mdok) { + psError(PPSTACK_ERR_CONFIG, true, "Unable to find PHOT.FRAC in recipe"); + return false; + } + + psString maskValStr = psMetadataLookupStr(&mdok, recipe, "MASK.VAL"); // Name of bits to mask going in + if (!mdok || !maskValStr) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find MASK.VAL in recipe"); + return false; + } + psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Bits to mask + + // Measure background + psImage *image = ro->image, *mask = ro->mask; // Image and mask from readout + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics + if (!psImageBackground(stats, NULL, image, mask, maskVal, rng)) { + psError(PPSTACK_ERR_DATA, false, "Unable to measure background for image"); + psFree(stats); + psFree(rng); + return false; + } + float bg = stats->robustMedian; // Background level + psFree(stats); + psFree(rng); +# endif + + // Examine the sources (photometer if RE_PHOTOMETER is set) + int numSources = sources->n; // Number of sources + int numGood = 0; // Number of good sources + float maxMag = -INFINITY; // Maximum magnitude + for (int j = 0; j < numSources; j++) { + pmSource *source = sources->data[j]; // Source of interest + if (source->mode & PHOT_SOURCE_MASK || !isfinite(source->psfMag)) { + source->psfMag = NAN; + continue; + } + +// This code re-measures the magnitude in the assumption that the warp analysis failed It would +// be better to catch and correct such cases. In any case, we should check on the validity of +// this assumption (eg, compare the mean psf-ap offset) and raise an error on that? + +# if (RE_PHOTOMETER) + + int numCols = image->numCols, numRows = image->numRows; // Size of image + float x = source->peak->xf, y = source->peak->yf; // Coordinates of source + int xCentral = x + 0.5, yCentral = y + 0.5; // Central pixel + // Range of integration + int xMin = PS_MAX(0, xCentral - zpRadius), xMax = PS_MIN(numCols - 1, xCentral + zpRadius); + int yMin = PS_MAX(0, yCentral - zpRadius), yMax = PS_MIN(numRows - 1, yCentral + zpRadius); + + // Integrate + double sumImage = 0.0, sumKernel = 0.0; // Sums from integration + int numBadPix = 0; // Number of bad pixels + for (int v = yMin; v <= yMax; v++) { + float dy2 = PS_SQR(y - v); // Distance from centroid + for (int u = xMin; u <= xMax; u++) { + if (mask->data.PS_TYPE_IMAGE_MASK_DATA[v][u] & maskVal) { + numBadPix++; + continue; + } + float dx2 = PS_SQR(x - u); // Distance from centroid + double kernel = exp(-0.5 * (dx2 + dy2) / PS_SQR(zpSigma)); // Kernel value + sumImage += (image->data.F32[v][u] - bg) * kernel; + sumKernel += kernel; + } + } + + if (numBadPix > zpFrac * M_PI * PS_SQR(zpRadius) || !isfinite(sumImage)) { + source->psfMag = NAN; + } else { + source->psfMag = - 2.5 * log10(sumImage / sumKernel * M_PI * PS_SQR(zpRadius)); + if (isfinite(source->psfMag)) { + numGood++; + maxMag = PS_MAX(maxMag, source->psfMag); + } + } +# else + numGood++; + maxMag = PS_MAX(maxMag, source->psfMag); +# endif + } + psLogMsg("ppStack", PS_LOG_INFO, "Photometered %d good sources in image; max mag %f", numGood, maxMag); + + return true; +} + + +// Preparation iteration: Load the sources, and get a target PSF model +bool ppStackPrepare(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + int num = options->num = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); + options->sourceLists = psArrayAlloc(num); // Individual lists of sources for matching + + options->inputMask = psVectorAlloc(num, PS_TYPE_VECTOR_MASK); // Mask for inputs + psVectorInit(options->inputMask, 0); + options->exposures = psVectorAlloc(options->num, PS_TYPE_F32); + + psVectorInit(options->exposures, NAN); + + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true); + pmFPAview *view = ppStackFilesIterateDown(config); + if (!view) { + return false; + } + + //MEH -- bscale offset hack for warps + options->bscaleOffset = psVectorAlloc(options->num, PS_TYPE_F32); + psVectorInit(options->bscaleOffset, 0.0); + bool setbscaleoffset = psMetadataLookupBool(NULL,recipe, "BSCALEOFFSET"); + + psArray *psfs = psArrayAlloc(num); // PSFs for PSF envelope + int numCols = 0, numRows = 0; // Size of image + options->sumExposure = 0.0; + int numWithSources = 0; + for (int i = 0; i < num; i++) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest + pmCell *cell = pmFPAviewThisCell(view, file->fpa); // Cell of interest + + options->exposures->data.F32[i] = psMetadataLookupF32(NULL, cell->concepts, "CELL.EXPOSURE"); + if ((options->exposures->data.F32[i] == 0)|| + (!(isfinite(options->exposures->data.F32[i])))){ + options->exposures->data.F32[i] = psMetadataLookupF32(NULL,recipe,"DEFAULT.EXPTIME"); + } + options->sumExposure += options->exposures->data.F32[i]; + + //MEH -- hdu get redefn again below if conv (lots of this happening..) + if (setbscaleoffset) { + pmHDU *hdu = pmHDUFromCell(cell); + assert(hdu && hdu->header); + float bscale = psMetadataLookupF32(NULL, hdu->header, "BSCALE"); + options->bscaleOffset->data.F32[i] = 0.5*bscale; + } + psLogMsg("ppStack", PS_LOG_INFO, "bscaleOffset: %d %f", i,options->bscaleOffset->data.F32[i]); + + // Get list of PSFs, to determine target PSF + if (options->convolve) { + pmChip *chip = pmFPAviewThisChip(view, file->fpa); // The chip: holds the PSF + pmPSF *psf = psMetadataLookupPtr(NULL, chip->analysis, "PSPHOT.PSF"); // PSF + if (!psf) { + psError(PPSTACK_ERR_PROG, false, "Unable to find PSF."); + psFree(view); + psFree(psfs); + return false; + } + psfs->data[i] = psMemIncrRefCounter(psf); + psMetadataRemoveKey(chip->analysis, "PSPHOT.PSF"); + + pmCell *cell = pmFPAviewThisCell(view, file->fpa); // Cell of interest + pmHDU *hdu = pmHDUFromCell(cell); + assert(hdu && hdu->header); + int naxis1 = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); // Number of columns + int naxis2 = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); // Number of rows + if (naxis1 <= 0 || naxis2 <= 0) { + psError(PPSTACK_ERR_PROG, false, "Unable to determine size of image from PSF."); + psFree(view); + psFree(psfs); + return false; + } + if (numCols == 0 && numRows == 0) { + numCols = naxis1; + numRows = naxis2; + } + } + + + bool redoPhot = psMetadataLookupBool(NULL, recipe, "PHOT"); + + pmDetections *detections = NULL; + if (options->convolve || options->matchZPs || options->photometry || redoPhot) { + pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout with sources + detections = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.DETECTIONS"); // Sources + if (!detections || !detections->allSources || !detections->allSources->n) { + psWarning("No detections found for image %d --- rejecting.", i); + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_CAL; + continue; + } + psAssert (detections->allSources, "missing sources?"); + + options->sourceLists->data[i] = psMemIncrRefCounter(detections->allSources); + numWithSources++; + } + + // Re-do photometry if we don't trust the source lists + if (redoPhot) { + psTrace("ppStack", 2, "Photometering input %d of %d....\n", i, num); + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivationSingle(config, PPSTACK_FILES_CONVOLVE, true, i); + if (options->convolve) { + pmFPAfileActivate(config->files, true, "PPSTACK.CONV.KERNEL"); + } + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File + pmFPAview *photView = ppStackFilesIterateDown(config); + if (!photView) { + psFree(view); + return false; + } + + pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout of interest + + if (!ppStackInputPhotometer(ro, detections->allSources, config)) { + psError(psErrorCodeLast(), false, "Unable to do photometry on input sources"); + psFree(view); + psFree(photView); + return false; + } + + psFree(photView); + if (!ppStackFilesIterateUp(config)) { + psFree(view); + return false; + } + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true); + } + } + if (numWithSources < 2) { + // This can happen if the inputs have been destreaked + psErrorStackPrint(stderr, "Not enough inputs have sources"); + psWarning("No inputs have sources --- suspect bad data quality."); + if (options->quality == 0) { + options->quality = PPSTACK_ERR_DATA; + } + psErrorClear(); + psFree(view); + return false; + } + + bool mdok = false; + bool simpleClip = psMetadataLookupBool(&mdok, recipe, "PSF.INPUT.CLIP.SIMPLE"); + float maxFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.MAX"); // max allowed input fwhm + float clipFWHMnSig = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.CLIP.NSIGMA"); // sigma clipping of inputs + float threshFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.THRESH"); // FWHM size that we refuse to clip below + float asymmetryFWHM = psMetadataLookupF32(&mdok, recipe, "PSF.INPUT.ASYMMETRY"); // max bimodal asymmetry + + + psString log = psStringCopy("Input seeing FWHMs:\n"); // Log message + bool havePSFs = false; // Do we have any PSFs? + options->inputSeeing = psVectorAlloc(num, PS_TYPE_F32); + psVectorInit(options->inputSeeing, NAN); + for (int i = 0; i < num; i++) { + pmPSF *psf = psfs->data[i]; // PSF for image + if (!psf) { + continue; + } + havePSFs = true; + + int xNum = PS_MAX(psf->trendNx, 1), yNum = PS_MAX(psf->trendNy, 1); // Number of realisations + double sumFWHM = 0.0; // FWHM for image + int numFWHM = 0; // Number of FWHM measurements + for (int y = 0; y < yNum; y++) { + float yPos = ((float)y + 0.5) / (float)yNum * numRows; + for (int x = 0; x < xNum; x++) { + float xPos = ((float)x + 0.5) / (float)xNum * numCols; + float fwhm = pmPSFtoFWHM(psf, xPos, yPos); // FWHM for image + if (isfinite(fwhm)) { + sumFWHM += fwhm; + numFWHM++; + } + } + } + if (numFWHM == 0) { + options->inputSeeing->data.F32[i] = NAN; + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_PSF; + psLogMsg("ppStack", PS_LOG_INFO, "Unable to measure PSF FWHM for image %d --- rejected.", i); + } else { + options->inputSeeing->data.F32[i] = sumFWHM / (float)numFWHM; + } +#ifdef OLD_REJECTION + // reject any input images which exceed the specified max FWHM + if (isfinite(maxFWHM) && (options->inputSeeing->data.F32[i] > maxFWHM)) { + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_PSF; + psLogMsg("ppStack", PS_LOG_INFO, "PSF FWHM for image %d is too large (%f vs %f maxFWHM) --- rejected.", i, options->inputSeeing->data.F32[i], maxFWHM); + } + // End old rejection fixed limit. +#endif + psStringAppend(&log, "Input %d: %f\n", i, options->inputSeeing->data.F32[i]); + } + if (havePSFs) { + psLogMsg("ppStack", PS_LOG_INFO, "%s", log); + } + psFree(log); + +#ifdef NEW_REJECTION + // Begin new rejection code. + float limit = INFINITY; // Default to a big value. + if (!isfinite(threshFWHM)) { // If this isn't set, initialize it to zero. No inputs may be clipped if smaller than this + threshFWHM = 0.0; + } + if (!isfinite(maxFWHM)) { // No maxFWHM was set, accept all inputs + maxFWHM = INFINITY; + } + limit = maxFWHM; // Initialize the limit to be the maxFWHM + if (limit < threshFWHM) { + limit = threshFWHM; + } + if (!isfinite(clipFWHMnSig)) { // We cannot do a sigma clip with nSig, so use the maxFWHM to set the limit. + psLogMsg("ppStack",PS_LOG_INFO, + "PSF FWHM distribution: Unable to determine limit based on recipe. Setting to %f. (clipFWHMnSig: %f maxFWHM: %f threshFWHM: %f asymmetryFWHM: %f)", + limit, + clipFWHMnSig,maxFWHM,threshFWHM,asymmetryFWHM); + } + if (simpleClip) { // Do a sigma clip like the old rejection code. + psStats *fwhmStats = psStatsAlloc(PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); + psVectorStats (fwhmStats, options->inputSeeing, NULL, options->inputMask, 0xff); + psLogMsg("ppStack", PS_LOG_INFO, "Input FWHMs : %f +/- %f", fwhmStats->clippedMean, fwhmStats->clippedStdev); + + options->clippedMean = fwhmStats->clippedMean; + options->clippedStdev = fwhmStats->clippedStdev; + limit = options->clippedMean + clipFWHMnSig * options->clippedStdev; + if (limit < threshFWHM) { + limit = threshFWHM; + } + if (limit > maxFWHM) { + limit = maxFWHM; + } + psLogMsg("ppStack",PS_LOG_INFO, + "PSF FWHM distribution: Used simple clip method. Limit set to %f.", + limit); + psFree(fwhmStats); + } + else { // Do the mixture model rejection. + if(!isfinite(asymmetryFWHM)) { // Or not, because the parameters aren't set. + psLogMsg("ppStack",PS_LOG_INFO, + "PSF FWHM distribution: Unable to determine limit based on recipe. Setting to %f. (clipFWHMnSig: %f maxFWHM: %f threshFWHM: %f asymmetryFWHM: %f)", + limit, + clipFWHMnSig,maxFWHM,threshFWHM,asymmetryFWHM); + } + else { + // Do GMM test with two modes to decide where to put the break. + double Punimodal; + int m = 2; + psVector *modes = psVectorAlloc(num,PS_TYPE_F32); + psVector *means = psVectorAlloc(m,PS_TYPE_F32); + psVector *S = psVectorAlloc(m,PS_TYPE_F32); + psVector *pi = psVectorAlloc(m,PS_TYPE_F32); + psImage *P = psImageAlloc(m,num,PS_TYPE_F32); + + // XXX: we should not pass in images to this analysis which have already been rejected by the FWHM limits + if (!psMM1DClass(options->inputSeeing, + options->inputSeeing->n, + modes,means, + S,pi,P, + 2, + &Punimodal)) { + // Handle error here + psFree(modes); + psFree(means); + psFree(S); + psFree(pi); + psFree(P); + return(false); + } + // fprintf(stderr,"means: %g %g\n",means->data.F32[0],means->data.F32[1]); + // fprintf(stderr,"sigma: %g %g \n",S->data.F32[0],S->data.F32[1]); + // fprintf(stderr,"pi: %g %g\n",pi->data.F32[0],pi->data.F32[1]); + + // Use Gaussian mixture model analysis of the FWHM distribution to decide an optional FWHM limit + if ((Punimodal > 0.5)|| // This distribution is best represented by a single mode + (num <= 4)) { // Or we have a small number of inputs, making this statistic poor. + limit = maxFWHM; + } + else { // This is a bimodal distribution + if ((fabs(pi->data.F32[0] - pi->data.F32[1]) < asymmetryFWHM)|| // However, both modes are equally populated + (pi->data.F32[1] > pi->data.F32[0])) { // Or the larger FWHM mode is more populated + limit = means->data.F32[1] + clipFWHMnSig * S->data.F32[1]; + } + else { // The smaller FWHM mode is more populated + limit = means->data.F32[0] + clipFWHMnSig * S->data.F32[0]; + } + } + if (limit > maxFWHM) { limit = maxFWHM; } // We should not be larger than our max + if (limit < threshFWHM) { limit = threshFWHM; } // Nor smaller than our min + psLogMsg("ppStack",PS_LOG_INFO, + "PSF FWHM distribution: limit: %f (%f %f %f) (%f %f %f) %f", + limit,means->data.F32[0],S->data.F32[0],pi->data.F32[0], + means->data.F32[1],S->data.F32[1],pi->data.F32[1], + Punimodal); + psFree(means); + psFree(modes); + psFree(S); + psFree(pi); + psFree(P); + } // End mixture model case + } + // Perform rejection using the limit set + for (int i = 0; i < num; i++) { + if (options->inputSeeing->data.F32[i] > limit) { + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_PSF; + psLogMsg("ppStack", PS_LOG_INFO, + "PSF FWHM for image %d is too large (%f vs %f fwhmlimit) --- rejected", + i, options->inputSeeing->data.F32[i],limit); + } + } + // End new rejection code +#endif + +#ifdef OLD_REJECTION + // We should have the ability to filter the input list based on the seeing: + // * reject above some max value and/or min value + // * reject images out-of-line with the rest of the images + + // measure stats + psStats *fwhmStats = psStatsAlloc(PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); + psVectorStats (fwhmStats, options->inputSeeing, NULL, options->inputMask, 0xff); + psLogMsg("ppStack", PS_LOG_INFO, "Input FWHMs : %f +/- %f", fwhmStats->clippedMean, fwhmStats->clippedStdev); + + options->clippedMean = fwhmStats->clippedMean; + options->clippedStdev = fwhmStats->clippedStdev; + + if (isfinite(clipFWHMnSig)) { + float fwhmLimit = fwhmStats->clippedMean + clipFWHMnSig * fwhmStats->clippedStdev; + fwhmLimit = isfinite(maxFWHM) ? PS_MIN (maxFWHM, fwhmLimit) : fwhmLimit; + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue; + if (options->inputSeeing->data.F32[i] > fwhmLimit) { + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_PSF; + psLogMsg("ppStack", PS_LOG_INFO, "PSF FWHM for image %d is too large (%f vs %f fwhmLimit) --- rejected.", i, options->inputSeeing->data.F32[i], fwhmLimit); + } + } + } + // End old rejection sigma clip +#endif + // Generate target PSF + if (options->convolve) { + options->psf = ppStackPSF(config, numCols, numRows, psfs, options); + psFree(psfs); + if (!options->psf) { +#if 1 + psError(psErrorCodeLast(), false, "Unable to determine output PSF."); +#else + // This will repair the problem reproted in ticket 1427 but we aren't yet sure + // why ppStackPSF is failing so we are going to continue to fault for now + int errorCode = psErrorCodeLast(); + if (errorCode == PPSTACK_ERR_PSF) { + psErrorStackPrint(stderr, "Unable to determine output PSF."); + psWarning("Unable to determine output PSF --- suspect bad data quality."); + if (options->quality == 0) { + options->quality = errorCode; + } + psErrorClear(); + } else { + psError(psErrorCodeLast(), false, "Unable to determine output PSF."); + } +#endif // notyet + psFree(view); + return false; + } + psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "PSF.TARGET", PS_DATA_UNKNOWN, + "Target PSF for stack", options->psf); + options->targetSeeing = pmPSFtoFWHM(options->psf, 0.5 * numCols, 0.5 * numRows); // FWHM for target + + + bool psfTargetAsMax = psMetadataLookupBool(&mdok, recipe, "PSF.TARGET.AS.MAX"); + if (psfTargetAsMax) { // Should we use the largest input as the target? + float psfTargetEpsilon = psMetadataLookupF32(&mdok,recipe,"PSF.TARGET.AS.MAX.EPSILON"); + if (!mdok) { psfTargetEpsilon = 0.0; } + options->targetSeeing = 0.0; + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue; + options->targetSeeing = PS_MAX(options->targetSeeing,options->inputSeeing->data.F32[i]); + } + psLogMsg("ppStack", PS_LOG_INFO, "Using MAX accepted input FWHM as target (max: %f epsilon: %f target %f)\n", + options->targetSeeing,psfTargetEpsilon,options->targetSeeing + psfTargetEpsilon); + options->targetSeeing = options->targetSeeing + psfTargetEpsilon; + } + psLogMsg("ppStack", PS_LOG_INFO, "Target seeing FWHM: %f\n", options->targetSeeing); + + pmChip *outChip = pmFPAfileThisChip(config->files, view, "PPSTACK.TARGET.PSF"); // Output chip + psMetadataAddPtr(outChip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN, + "Target PSF", options->psf); + outChip->data_exists = true; + } + + // Zero point calibration + if (!ppStackSourcesTransparency(options, view, config)) { + psError(PPSTACK_ERR_DATA, false, "Unable to calculate transparency differences"); + psFree(view); + return false; + } + psFree(view); + + if (!ppStackFilesIterateUp(config)) { + return false; + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackReadout.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackReadout.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackReadout.c (revision 41840) @@ -0,0 +1,317 @@ +#include "ppStack.h" + +bool ppStackReadoutInitialThread(psThreadJob *job) +{ + PS_ASSERT_THREAD_JOB_NON_NULL(job, false); + + psArray *args = job->args; // Arguments + ppStackThread *thread = args->data[0]; // Thread + ppStackOptions *options = args->data[1]; // Options + pmConfig *config = args->data[2]; // Configuration + + pmReadout *outRO = options->outRO; // Output readout + psVector *mask = options->inputMask; // Mask for inputs + psVector *weightings = options->weightings; // Weightings (1/noise^2) for each image + psVector *exposures = options->exposures; // Exposure times for each image + psVector *addVariance = options->matchChi2; // Additional variance when rejecting + + job->results = ppStackReadoutInitial(config, outRO, thread->readouts, mask, + weightings, exposures, addVariance); + thread->busy = false; + + return job->results ? true : false; +} + +bool ppStackReadoutFinalThread(psThreadJob *job) +{ + PS_ASSERT_THREAD_JOB_NON_NULL(job, false); + + psArray *args = job->args; // Arguments + ppStackThread *thread = args->data[0]; // Thread + psArray *reject = args->data[1]; // Rejected pixels for each image + ppStackOptions *options = args->data[2]; // Options + pmConfig *config = args->data[3]; // Configuration + bool safety = PS_SCALAR_VALUE(args->data[4], U8); // Safety switch on? + bool normalise = PS_SCALAR_VALUE(args->data[5], U8); // Normalise images? + bool bscaleoffset = PS_SCALAR_VALUE(args->data[6], U8); // Apply bscale offset? + + psVector *mask = options->inputMask; // Mask for inputs + psVector *weightings = options->weightings; // Weightings (1/noise^2) for each image + psVector *exposures = options->exposures; // Exposure times for each image + psVector *addVariance = options->matchChi2; // Additional variance when rejecting + psVector *norm = normalise ? options->norm : NULL; // Normalisations to apply to images + psVector *bscaleApplyOffset = bscaleoffset ? options->bscaleOffset : NULL; // BSCALE offset to apply to images + + bool status = ppStackReadoutFinal(config, options->outRO, options->expRO, thread->readouts, mask, reject, + weightings, exposures, addVariance, safety, norm, bscaleApplyOffset); // Status of operation + + thread->busy = false; + + psAssert(status, "Stacking failed."); + + return status; +} + + +bool ppStackInspect(psThreadJob *job) +{ + PS_ASSERT_THREAD_JOB_NON_NULL(job, false); + + psArray *args = job->args; // Input arguments + psArray *inspects = args->data[0]; // Array of pixel arrays + psArray *rejects = args->data[1]; // Array of pixel arrays + int index = PS_SCALAR_VALUE(args->data[2], S32); // Index of interest + + psArray *inInspects = inspects->data[index]; // Array of interest + psArray *inRejects = rejects->data[index]; // Array of interest + psAssert(inInspects->n == inRejects->n, "Size should be the same"); + psPixels *outInspect = NULL, *outReject = NULL; // Output pixel lists + for (int i = 0; i < inInspects->n; i++) { + psPixels *inInspect = inInspects->data[i]; // Input pixel list + if (inInspect && inInspect->n > 0) { + outInspect = psPixelsConcatenate(outInspect, inInspect); + } + psPixels *inReject = inRejects->data[i]; // Input pixel list + if (inReject && inReject->n > 0) { + outReject = psPixelsConcatenate(outReject, inReject); + } + } + + // If there are no pixels to inspect, then just fake it + if (!outInspect) { + outInspect = psPixelsAllocEmpty(0); + } + if (!outReject) { + outReject = psPixelsAllocEmpty(0); + } + + psFree(inspects->data[index]); + inspects->data[index] = outInspect; + psFree(rejects->data[index]); + rejects->data[index] = outReject; + + return true; +} + + +psArray *ppStackReadoutInitial(const pmConfig *config, pmReadout *outRO, const psArray *readouts, + const psVector *mask, const psVector *weightings, const psVector *exposures, + const psVector *addVariance) +{ + assert(config); + assert(outRO); + assert(readouts); + assert(mask && mask->n == readouts->n && mask->type.type == PS_TYPE_VECTOR_MASK); + assert(weightings && weightings->n == readouts->n && weightings->type.type == PS_TYPE_F32); + assert(addVariance && addVariance->n == readouts->n && addVariance->type.type == PS_TYPE_F32); + static int sectionNum = 0; // Section number; for debugging outputs + + // Get the recipe values + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + bool mdok; // Status of MD lookup + float iter = psMetadataLookupF32(NULL, recipe, "COMBINE.ITER"); // Rejection iterations + float combineRej = psMetadataLookupF32(NULL, recipe, "COMBINE.REJ"); // Combination threshold + float combineSys = psMetadataLookupF32(NULL, recipe, "COMBINE.SYS"); // Combination systematic error + float combineDiscard = psMetadataLookupF32(NULL, recipe, "COMBINE.DISCARD"); // Olympic discard fraction + bool useVariance = psMetadataLookupBool(&mdok, recipe, "VARIANCE"); // Use variance for rejection? + bool safe = psMetadataLookupBool(&mdok, recipe, "SAFE"); // Be safe when combining small numbers of pixels + + int nminpix = psMetadataLookupS32(&mdok, recipe, "NMINPIX"); // Minimum input per pixel to combine with + + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + int kernelSize = psMetadataLookupS32(NULL, ppsub, "KERNEL.SIZE"); // Kernel half-size + + psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.VAL"); // Name of bits to mask for bad + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + psString maskSuspectStr = psMetadataLookupStr(NULL, recipe, "MASK.SUSPECT"); // Name of suspect mask bits + psImageMaskType maskSuspect = pmConfigMaskGet(maskSuspectStr, config); // Suspect bits + + bool status = false; + psImageMaskType maskBlank; + psString maskBlankStr = psMetadataLookupStr(&status, recipe, "MASK.BLANK"); // Name of bits to set for empty pixels + if (maskBlankStr) { + maskBlank = pmConfigMaskGet(maskBlankStr, config); // Bits to mask for bad pixels + } else { + maskBlankStr = psMetadataLookupStr(&status, recipe, "MASK.BAD"); // Old name for MASK.BLANK + if (maskBlankStr) { + maskBlank = pmConfigMaskGet(maskBlankStr, config); // Bits to mask for bad pixels + } else { + maskBlank = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels + } + } + + int num = readouts->n; // Number of inputs + psArray *stack = psArrayAlloc(num); // Array for stacking + + for (int i = 0; i < num; i++) { + pmReadout *ro = readouts->data[i]; + if (!ro || mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + // Bad image + continue; + } + + // Ensure there is a mask, or pmStackCombine will complain + if (!ro->mask) { + ro->mask = psImageAlloc(ro->image->numCols, ro->image->numRows, PS_TYPE_IMAGE_MASK); + psImageInit(ro->mask, 0); + } + + stack->data[i] = pmStackDataAlloc(ro, weightings->data.F32[i], exposures->data.F32[i], + addVariance->data.F32[i]); + } + + if (!pmStackCombine(outRO, NULL, stack, maskBad, maskSuspect, maskBlank, kernelSize, iter, + combineRej, combineSys, combineDiscard, useVariance, safe, nminpix, false)) { + psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts with rejection."); + psFree(stack); + return false; + } + + // Save lists of pixels + psArray *inspect = psArrayAlloc(num); // List of pixels to inspect + psArray *reject = psArrayAlloc(num); // List of pixels rejected + for (int i = 0; i < num; i++) { + pmStackData *data = stack->data[i]; // Data for this image + if (!data) { + continue; + } + pmReadout *readout = data->readout; // Readout of interest + if (!readout) { + continue; + } + inspect->data[i] = psMemIncrRefCounter(data->inspect); + reject->data[i] = psMemIncrRefCounter(data->reject); + } + psFree(stack); + + //MEH change to trace + //psLogMsg("ppStack", PS_LOG_INFO, "initial stack image sectionNum %d", sectionNum); + + sectionNum++; + + psArray *results = psArrayAlloc(2); // Array of results + results->data[0] = inspect; + results->data[1] = reject; + + return results; +} + + +bool ppStackReadoutFinal(const pmConfig *config, pmReadout *outRO, pmReadout *expRO, const psArray *readouts, + const psVector *mask, const psArray *rejected, const psVector *weightings, + const psVector *exposures, const psVector *addVariance, bool safety, + const psVector *norm, const psVector *bscaleApplyOffset) +{ + assert(config); + assert(outRO); + assert(expRO); + assert(readouts); + assert(!rejected || readouts->n == rejected->n); + assert(mask && mask->n == readouts->n && mask->type.type == PS_TYPE_VECTOR_MASK); + assert(weightings && weightings->n == readouts->n && weightings->type.type == PS_TYPE_F32); + + static int sectionNum = 0; + + // Get the recipe values + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + bool mdok; // Status of MD lookup + bool useVariance = psMetadataLookupBool(&mdok, recipe, "VARIANCE"); // Use variance for rejection? + bool safe = psMetadataLookupBool(&mdok, recipe, "SAFE"); // Be safe when combining small numbers of pixels + + int nminpix = psMetadataLookupS32(&mdok, recipe, "NMINPIX"); // Minimum input per pixel to combine with + + psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.VAL"); // Name of bits to mask for bad + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + psString maskSuspectStr = psMetadataLookupStr(NULL, recipe, "MASK.SUSPECT"); // Name of suspect mask bits + psImageMaskType maskSuspect = pmConfigMaskGet(maskSuspectStr, config); // Suspect bits + + bool status = false; + psImageMaskType maskBlank; + psString maskBlankStr = psMetadataLookupStr(&status, recipe, "MASK.BLANK"); // Name of bits to set for empty pixels + if (maskBlankStr) { + maskBlank = pmConfigMaskGet(maskBlankStr, config); // Bits to mask for bad pixels + } else { + maskBlankStr = psMetadataLookupStr(&status, recipe, "MASK.BAD"); // Old name for MASK.BLANK + if (maskBlankStr) { + maskBlank = pmConfigMaskGet(maskBlankStr, config); // Bits to mask for bad pixels + } else { + maskBlank = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels + } + } + + int num = readouts->n; // Number of inputs + psArray *stack = psArrayAlloc(num); // Array for stacking + + // We have rejection from a previous combination: combine without flagging pixels to inspect + safe &= safety; + int iter = 0; + float combineRej = NAN; + float combineSys = NAN; + float combineDiscard = NAN; + + for (int i = 0; i < num; i++) { + pmReadout *ro = readouts->data[i]; + if (mask->data.U8[i]) { + // Image completely rejected + continue; + } + + // Ensure there is a mask, or pmStackCombine will complain + if (!ro->mask) { + ro->mask = psImageAlloc(ro->image->numCols, ro->image->numRows, PS_TYPE_IMAGE_MASK); + psImageInit(ro->mask, 0); + } + + pmStackData *data = pmStackDataAlloc(ro, weightings->data.F32[i], exposures->data.F32[i], + addVariance ? addVariance->data.F32[i] : NAN); + data->reject = rejected ? psMemIncrRefCounter(rejected->data[i]) : NULL; + stack->data[i] = data; + + //MEH -- apply bscale offset before norm + if (bscaleApplyOffset) { + //MEH change to trace + //psLogMsg("ppStack", PS_LOG_INFO, "bscaleApplyOffset: %d %f", i, bscaleApplyOffset->data.F32[i]); + psBinaryOp(ro->image, ro->image, "-", psScalarAlloc(bscaleApplyOffset->data.F32[i], PS_TYPE_F32)); + } + + if (norm) { + float normalise = powf(10.0, -0.4 * norm->data.F32[i]); // Normalisation + psBinaryOp(ro->image, ro->image, "*", psScalarAlloc(normalise, PS_TYPE_F32)); + psBinaryOp(ro->variance, ro->variance, "*", psScalarAlloc(PS_SQR(normalise), PS_TYPE_F32)); + } + } + + if (!pmStackCombine(outRO, expRO, stack, maskBad, maskSuspect, maskBlank, 0, iter, combineRej, + combineSys, combineDiscard, useVariance, safe, nminpix, rejected)) { + psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts."); + psFree(stack); + return false; + } + + pmCell *outCell = outRO->parent; // Output cell + pmChip *outChip = outCell->parent; // Output chip + outRO->data_exists = true; + outCell->data_exists = true; + outChip->data_exists = true; + + pmCell *expCell = expRO->parent; // Exposure cell + pmChip *expChip = expCell->parent; // Exposure chip + expRO->data_exists = true; + expCell->data_exists = true; + expChip->data_exists = true; + + psFree(stack); + + //MEH change to trace + //psLogMsg("ppStack", PS_LOG_INFO, "final stack image sectionNum %d", sectionNum); + + sectionNum++; + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackReject.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackReject.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackReject.c (revision 41840) @@ -0,0 +1,195 @@ +#include "ppStack.h" + +// This is the doomsday switch. +// #define TESTING + +bool ppStackReject(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + +/* if (!options->convolve) { */ +/* // No need to do complicated rejection when we haven't convolved */ +/* return true; */ +/* } */ + + + int num = options->num; // Number of inputs + + // Construct a small convolution kernel to aid in rejection + if (!options->convolve) { + for (int i = 0; i < num; i++) { + psArray *regions = psArrayAllocEmpty(1); + psRegion *region = psRegionAlloc(0,options->numCols - 1, 0, options->numRows - 1); + regions = psArrayAdd(regions,1, region); + + psArray *kernels = psArrayAllocEmpty(1); + psVector *fwhms = psVectorAllocEmpty(1, PS_TYPE_F32); + psVectorAppend(fwhms,5.0); // Should be a parameter + psVector *orders = psVectorAllocEmpty(1, PS_TYPE_S32); + psVectorAppend(orders,0); + pmSubtractionKernels *kernel = pmSubtractionKernelsISIS(15,0,fwhms,orders,0,*region,PM_SUBTRACTION_MODE_2); + kernels = psArrayAdd(kernels, 1, kernel); + + kernel->solution1 = psVectorAlloc(3, PS_TYPE_F64); + psVectorSet(kernel->solution1, 0, 1.0); + psVectorSet(kernel->solution1, 1, 1.0); + psVectorSet(kernel->solution1, 2, 1.0); + kernel->solution2 = psVectorAlloc(3, PS_TYPE_F64); + psVectorSet(kernel->solution2, 0, 1.0); + psVectorSet(kernel->solution2, 1, 1.0); + psVectorSet(kernel->solution2, 2, 1.0); + + options->kernels->data[i] = kernels; + options->regions->data[i] = regions; + } + } + + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + psAssert(ppsub, "We've thrown an error on this before."); + + float threshold = psMetadataLookupF32(NULL, recipe, "THRESHOLD.MASK"); // Threshold for mask deconvolution + float imageRej = psMetadataLookupF32(NULL, recipe, "IMAGE.REJ"); // Maximum fraction of image to reject + // before rejecting entire image + int stride = psMetadataLookupS32(NULL, ppsub, "STRIDE"); // Size of convolution patches + + // Count images rejected out of hand + int numRejected = 0; // Number of inputs rejected completely + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + numRejected++; + } + } + + // Concatenate inspection lists + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + + psThreadJob *job = psThreadJobAlloc("PPSTACK_INSPECT"); // Job to start + psArrayAdd(job->args, 1, options->inspect); + psArrayAdd(job->args, 1, options->rejected); + PS_ARRAY_ADD_SCALAR(job->args, i, PS_TYPE_S32); + if (!psThreadJobAddPending(job)) { + return false; + } + } + if (!psThreadPoolWait(true, true)) { + psError(psErrorCodeLast(), false, "Unable to concatenate inspection lists."); + return false; + } + + + // Reject bad pixels + if (psMetadataLookupS32(NULL, config->arguments, "NTHREADS") > 0) { + pmStackRejectThreadsInit(); + } + for (int i = 0; i < num; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + psTimerStart("PPSTACK_REJECT"); + +#ifdef TESTING + { + psImage *mask = psPixelsToMask(NULL, options->inspect->data[i], + psRegionSet(0, options->numCols - 1, 0, options->numRows - 1), + 0xff); // Mask image + psString name = NULL; // Name of image + psStringAppend(&name, "inspect_%03d.fits", i); + pmStackVisualPlotTestImage(mask, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, mask, 0, NULL); + psFree(mask); + psFitsClose(fits); + } +#endif + +#ifdef TESTING + { + psImage *mask = psPixelsToMask(NULL, options->rejected->data[i], + psRegionSet(0, options->numCols - 1, 0, options->numRows - 1), + 0xff); // Mask image + psString name = NULL; // Name of image + psStringAppend(&name, "pre_reject_%03d.fits", i); + pmStackVisualPlotTestImage(mask, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, mask, 0, NULL); + psFree(mask); + psFitsClose(fits); + } +#endif + + psPixels *reject = pmStackReject(options->inspect->data[i], options->numCols, options->numRows, + threshold, stride, options->regions->data[i], + options->kernels->data[i]); // Rejected pixels + + psFree(options->inspect->data[i]); + options->inspect->data[i] = NULL; + + if (!reject) { + psWarning("Rejection on image %d didn't work --- reject entire image.", i); + numRejected++; + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_REJECT; + } else { + float frac = reject->n / (float)(options->numCols * options->numRows); // Pixel fraction + psLogMsg("ppStack", PS_LOG_INFO, "%ld pixels rejected from image %d (%.1f%%)", + reject->n, i, frac * 100.0); + if (frac > imageRej) { + psWarning("Image %d rejected completely because rejection fraction (%.3f) " + "exceeds limit (%.3f)", i, frac, imageRej); + psFree(reject); + reject = NULL; + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_BAD; + numRejected++; + } else { + // Add to list of pixels already rejected + reject = psPixelsConcatenate(reject, options->rejected->data[i]); + options->rejected->data[i] = psPixelsDuplicates(options->rejected->data[i], reject); + } + } + +#ifdef TESTING + { + psImage *mask = psPixelsToMask(NULL, options->rejected->data[i], + psRegionSet(0, options->numCols - 1, 0, options->numRows - 1), + 0xff); // Mask image + psString name = NULL; // Name of image + psStringAppend(&name, "reject_%03d.fits", i); + pmStackVisualPlotTestImage(mask, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, mask, 0, NULL); + psFree(mask); + psFitsClose(fits); + } +#endif + + if (options->stats) { + psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_REJECT", PS_META_DUPLICATE_OK, + "Time to perform rejection", psTimerMark("PPSTACK_REJECT")); + psMetadataAddS32(options->stats, PS_LIST_TAIL, "REJECT_PIXELS", PS_META_DUPLICATE_OK, + "Number of pixels rejected", reject ? reject->n : 0); + } + + psFree(reject); + psLogMsg("ppStack", PS_LOG_INFO, "Time to perform rejection on image %d: %f sec", i, + psTimerClear("PPSTACK_REJECT")); + } + + psFree(options->inspect); options->inspect = NULL; + + if (options->stats) { + psMetadataAddS32(options->stats, PS_LIST_TAIL, "REJECT_IMAGES", 0, + "Number of images rejected completely", numRejected); + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackSetup.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackSetup.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackSetup.c (revision 41840) @@ -0,0 +1,142 @@ +#include "ppStack.h" + +#define BUFFER 16 // Buffer for name array + +bool ppStackSetup(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + // XXX : switch to this name? options->matchZPs = psMetadataLookupBool(NULL, recipe, "MATCH.ZERO.POINTS"); // Adjust zero points based on tranparency analysis? + options->matchZPs = psMetadataLookupBool(NULL, recipe, "ZP"); // Adjust zero points? + + options->photometry = psMetadataLookupBool(NULL, recipe, "PHOTOMETRY"); // Perform photometry? + + options->convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images? + options->doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); + + if (!psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) { + psWarning("No PSFs provided --- unable to convolve to common PSF."); + options->convolve = false; + } + + int num = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs + options->num = num; + + bool mdok; // Status of MD lookup + const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics + if (statsName && strlen(statsName) > 0) { + psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename + options->statsFile = fopen(resolved, "w"); + if (!options->statsFile) { + psError(PPSTACK_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); + psFree(resolved); + return false; + } + psFree(resolved); + options->stats = psMetadataAlloc(); + } + + // Generate temporary names for convolved images + const char *tempDir = psMetadataLookupStr(NULL, config->arguments, "-temp-dir"); // Directory for temps + if (!tempDir) { + tempDir = psMetadataLookupStr(NULL, config->site, "TEMP.DIR"); + } + if (!tempDir) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find TEMP.DIR in site configuration"); + return false; + } + + psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments, + "OUTPUT")); // Name for temporary files + const char *tempName = psStringFileBasename(outputName); + if (!tempName) { + psError(PPSTACK_ERR_ARGUMENTS, false, "Unable to construct basename for temporary files."); + psFree(outputName); + return false; + } + + const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for images + const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for masks + const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for var maps + if (!tempImage || !tempMask || !tempVariance) { + psError(PPSTACK_ERR_CONFIG, false, + "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.VARIANCE in recipe"); + psFree(outputName); + psFree(tempName); + return false; + } + + options->convImages = psArrayAlloc(num); + options->convMasks = psArrayAlloc(num); + options->convVariances = psArrayAlloc(num); + for (int i = 0; i < num; i++) { + psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images + psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage); + psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask); + psStringAppend(&varianceName, "%s/%s.%d.%s", tempDir, tempName, i, tempVariance); + psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, varianceName); + options->convImages->data[i] = imageName; + options->convMasks->data[i] = maskName; + options->convVariances->data[i] = varianceName; + } + psFree(outputName); + psFree(tempName); + + // Original images + options->origImages = psArrayAlloc(num); + options->origMasks = psArrayAlloc(num); + options->origVariances = psArrayAlloc(num); + options->bkgImages = psArrayAlloc(num); + pmFPAview *view = pmFPAviewAlloc(0); + int nullMasks = 0; + int nullVariances = 0; + for (int i = 0; i < num; i++) { + { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); + options->origImages->data[i] = pmFPAfileName(file, view, config); + } + { + // We want the convolved mask, since that defines the area that has been tested for outliers + if (options->convolve) { + options->origMasks->data[i] = psMemIncrRefCounter(options->convMasks->data[i]); + } + else { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.MASK", i); + options->origMasks->data[i] = pmFPAfileName(file, view, config); + } + if (!(options->origMasks->data[i])) { + nullMasks++; + } + } + { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.VARIANCE", i); + options->origVariances->data[i] = pmFPAfileName(file, view, config); + if (!(options->origVariances->data[i])) { + nullVariances++; + } + } +/* { */ +/* pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.BKGMODEL", i); */ +/* options->bkgImages->data[i] = pmFPAfileName(file, view, config); */ +/* } */ + } + if (nullMasks == num) { + psFree(options->origMasks); + } + if (nullVariances == num) { + psFree(options->origVariances); + } + + psFree(view); + + if (!pmConfigMaskSetBits(NULL, NULL, config)) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to determine mask value."); + return false; + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackSources.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackSources.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackSources.c (revision 41840) @@ -0,0 +1,540 @@ +#include "ppStack.h" + +//#define TESTING // Enable debugging output + +//#define ASTROMETRY // Correct astrometry? + +#ifdef TESTING +// Size of fake image; set by hand because it's trouble to get it from other places +#define FAKE_COLS 4861 +#define FAKE_ROWS 4913 +#endif + +// #define TESTING_CZW + +#ifdef TESTING_CZW +// Dump matches to a file +static void dumpMatches(const char *filename, // File to which to dump + int num, // Number of inputs + psArray *matches, // Star matches + psVector *zp, // Zero points + psVector *trans // Transparencies + ) +{ + FILE *outMatches = fopen(filename, "w"); // Output matches + psVector *mag = psVectorAlloc(num, PS_TYPE_F32); // Magnitudes for each star + psVector *magErr = psVectorAlloc(num, PS_TYPE_F32); // Errors for each star + for (int i = 0; i < matches->n; i++) { + pmSourceMatch *match = matches->data[i]; // Match of interest + psVectorInit(mag, NAN); + psVectorInit(magErr, NAN); + for (int j = 0; j < match->num; j++) { + if (match->mask) { + if (match->mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) { + continue; + } + } + int index = match->image->data.U32[j]; // Image index + mag->data.F32[index] = match->mag->data.F32[j] - zp->data.F32[index]; + if (trans) { + mag->data.F32[index] -= trans->data.F32[index]; + } + magErr->data.F32[index] = match->magErr->data.F32[j]; + } + for (int j = 0; j < num; j++) { + fprintf(outMatches, "%f %f %f %f %f %f\t", + match->mag->data.F32[j], + zp->data.F32[j], + trans ? trans->data.F32[j] : NAN, + magErr->data.F32[j], + match->x->data.F32[j], + match->y->data.F32[j]); + } + fprintf(outMatches, "\n"); + } + psFree(mag); + psFree(magErr); + fclose(outMatches); + return; +} +#endif + + +bool ppStackSourcesTransparency(ppStackOptions *options, const pmFPAview *view, const pmConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(options, false); + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + if (!options->matchZPs && !options->photometry) { + options->norm = psVectorAlloc(options->num, PS_TYPE_F32); + psVectorInit(options->norm, 0.0); + return true; + } + + psArray *sourceLists = options->sourceLists; // Source lists for each input + psVector *inputMask = options->inputMask; // Mask for inputs + + PS_ASSERT_ARRAY_NON_NULL(sourceLists, false); + PS_ASSERT_VECTOR_NON_NULL(inputMask, false); + PS_ASSERT_VECTOR_TYPE(inputMask, PS_TYPE_U8, false); + PS_ASSERT_VECTOR_SIZE(inputMask, sourceLists->n, false); + +#if defined(TESTING) && 1 + { + // Deliberately induce a major transparency difference + psArray *sources = sourceLists->data[0]; // Sources to correct + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (!source) { + continue; + } + source->psfMag += 1.0; // modified only for a test +#ifdef ASTROMETRY + if (source->modelPSF) { + source->modelPSF->params->data.F32[PM_PAR_XPOS] += 1.0; + source->modelPSF->params->data.F32[PM_PAR_YPOS] += 1.0; + } + if (source->peak) { + source->peak->xf += 1.0; + source->peak->yf += 1.0; + } +#endif + } + } +#endif + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + float radius = psMetadataLookupF32(NULL, recipe, "ZP.RADIUS"); // Radius (pixels) for matching sources + int iter1 = psMetadataLookupS32(NULL, recipe, "ZP.ITER.1"); // Maximum iterations for pass 1 + int iter2 = psMetadataLookupS32(NULL, recipe, "ZP.ITER.2"); // Maximum iterations for pass 2 + float tol = psMetadataLookupF32(NULL, recipe, "ZP.TOL"); // Tolerance for zero point iterations + int transIter = psMetadataLookupS32(NULL, recipe, "ZP.TRANS.ITER"); // Iterations for transparency + float transRej = psMetadataLookupF32(NULL, recipe, "ZP.TRANS.REJ");// Rejection threshold for transparency + float transThresh = psMetadataLookupF32(NULL, recipe, "ZP.TRANS.THRESH"); // Threshold for transparency + + float starRej1 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.REJ.1"); // Rejection threshold for stars + float starSys1 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.SYS.1"); // Estimated systematic error + float starRej2 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.REJ.2"); // Rejection threshold for stars + float starSys2 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.SYS.2"); // Estimated systematic error + + float starLimit = psMetadataLookupF32(NULL, recipe, "ZP.STAR.LIMIT"); // Limit on star rejection fraction + + float fracMatch = psMetadataLookupF32(NULL, recipe, "ZP.MATCH"); // Fraction of images to match for star + + bool mdok = false; + float airmassTarget = psMetadataLookupF32(&mdok, recipe, "ZP.AIRMASS.TARGET"); // output airmass value + if (!mdok) { + airmassTarget = 1.0; + } + + psMetadata *airmassZP = psMetadataLookupMetadata(NULL, recipe, "ZP.AIRMASS"); // Airmass terms (slopes) by filter + if (!airmassZP) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find ZP.AIRMASS in recipe."); + return false; + } + psMetadata *zpTargetMenu = psMetadataLookupMetadata(NULL, recipe, "ZP.TARGET"); // Target zero point terms + if (!zpTargetMenu) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find ZP.TARGET in recipe."); + return false; + } + + int num = options->num; // Number of inputs + psAssert(num == sourceLists->n, "Wrong number of source lists: %ld\n", sourceLists->n); + + // vectors to store these inputs so they may be recorded in the output headers + options->zpInput = psVectorAlloc(num, PS_TYPE_F32); + options->expTimeInput = psVectorAlloc(num, PS_TYPE_F32); + options->airmassInput = psVectorAlloc(num, PS_TYPE_F32); + + psVector *zp = psVectorAlloc(num, PS_TYPE_F32); // Relative zero points for each image + psVector *zpExp = psVectorAlloc(num, PS_TYPE_F32); // Measured zero points for each image (maybe) + int zpExpNum = 0; // Number of measured zero points + const char *filter = NULL; // Filter name + float airmassTerm = NAN; // Airmass term + float zpTarget = NAN; // Target zero point + int numGoodImages = 0; // Number of good images + for (int i = 0; i < num; i++) { + psArray *sources = sourceLists->data[i]; // Source list + if (!sources || sources->n == 0) { + psLogMsg("ppStack", PS_LOG_WARN, "Image %d has no sources for transparency measurement.", i); + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_CAL; + zp->data.F32[i] = NAN; + continue; + } + numGoodImages++; + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest + +#if defined(TESTING) && 0 + pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout + pmPSF *psf = psMetadataLookupPtr(NULL, config->arguments, "PSF.TARGET"); // PSF for fake image + pmReadoutFakeFromSources(fake, FAKE_COLS, FAKE_ROWS, sourceLists->data[i], 0, + NULL, NULL, psf, 5, 0, false, true); + psString name = NULL; + psStringAppend(&name, "start_%03d.fits", i); + pmStackVisualPlotTestImage(fake->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, fake->image, 0, NULL); + psFitsClose(fits); + psFree(fake); +#endif + + float exptime = options->exposures->data.F32[i]; // Exposure time + float airmass = psMetadataLookupF32(NULL, file->fpa->concepts, "FPA.AIRMASS"); // Airmass + const char *expFilter = psMetadataLookupStr(NULL, file->fpa->concepts, "FPA.FILTER"); // Filter name + zpExp->data.F32[i] = psMetadataLookupF32(NULL, file->fpa->concepts, "FPA.ZP"); // Exposure zero point + // XXX need to get the zero point error values and propagate to get the FPA.ZP.ERR value + + options->zpInput->data.F32[i] = zpExp->data.F32[i]; // NOTE zpExp may be re-assigned below using relative photometry + options->expTimeInput->data.F32[i] = exptime; + options->airmassInput->data.F32[i] = airmass; + + psLogMsg("ppStack", PS_LOG_INFO, + "Image %d: %.2f sec exposure in %s at airmass %.2f with zero point %.2f", + i, exptime, expFilter, airmass, zpExp->data.F32[i]); + if (!isfinite(exptime) || exptime == 0 || !isfinite(airmass) || airmass == 0 || + !expFilter || strlen(expFilter) == 0) { + psError(PPSTACK_ERR_CONFIG, false, + "Unable to find exposure time (%f), airmass (%f) or filter (%s)", + exptime, airmass, expFilter); + psFree(zp); + return false; + } + if (isfinite(zpExp->data.F32[i])) { + zpExp->data.F32[i] += 2.5 * log10(exptime); + zpExpNum++; + } + + if (!filter) { + filter = expFilter; + airmassTerm = psMetadataLookupF32(&mdok, airmassZP, filter); + if (!mdok || !isfinite(airmassTerm)) { + psError(PPSTACK_ERR_CONFIG, false, + "Unable to find airmass term (ZP.AIRMASS) for filter %s", filter); + psFree(zp); + return false; + } + zpTarget = psMetadataLookupF32(&mdok, zpTargetMenu, filter); + if (!mdok || !isfinite(zpTarget)) { + psError(PPSTACK_ERR_CONFIG, false, + "Unable to find target zero point (ZP.TARGET) for filter %s", filter); + psFree(zp); + return false; + } + } else if (strcmp(filter, expFilter) != 0) { + psWarning("Filters don't match: %s vs %s", filter, expFilter); + } + + // XXX this is wrong, or at least inconsistent with the above: this needs to include + // a value for the nominal system zero point to be consistent with zpExp + zp->data.F32[i] = airmassTerm * airmass + 2.5 * log10(exptime); + } + + if (numGoodImages == 0) { + psLogMsg("ppStack", PS_LOG_WARN, "No images with sources to measure transparency."); + options->quality = PPSTACK_ERR_REJECTED; + psFree(zp); + psFree(zpExp); + return true; + } + if (numGoodImages == 1) { + psArray *sources = NULL; // Sources + for (int i = 0; i < num && !sources; i++) { + if (options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) { + continue; + } + sources = sourceLists->data[i]; + } + options->quality = PPSTACK_ERR_REJECTED; + options->sources = psMemIncrRefCounter(sources); + options->norm = psVectorAlloc(num, PS_TYPE_F32); + psVectorInit(options->norm, 1.0); + options->zp = NAN; + psLogMsg("ppStack", PS_LOG_WARN, "Single image with sources --- cannot match transparency."); + psFree(zp); + psFree(zpExp); + return true; + } + + if (zpExpNum == numGoodImages) { + psLogMsg ("ppStack", PS_LOG_INFO, "all zero points are finite; using reported zero points listed above"); + for (int i = 0; i < num; i++) { + zp->data.F32[i] = zpExp->data.F32[i]; + } + } else { + psLogMsg ("ppStack", PS_LOG_INFO, "missing some zero points; using guess values:"); + for (int i = 0; i < num; i++) { + psLogMsg("ppStack", PS_LOG_INFO, "Image %d: %.2f sec exposure with zero point %.2f", i, options->exposures->data.F32[i], zp->data.F32[i]); + } + } + + psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches + if (!matches) { + psError(PPSTACK_ERR_DATA, false, "Unable to match sources"); + psFree(zp); + return false; + } + options->sources = psMemIncrRefCounter(matches); + psArray *matchedSources = psArrayAlloc(0); + +#ifdef TESTING_CZW + dumpMatches("source_match.dat", num, matches, zp, NULL); +#endif + + if (options->matchZPs) { + + psVector *trans = pmSourceMatchRelphot(matches, matchedSources, zp, tol, iter1, starRej1, starSys1, + iter2, starRej2, starSys2, starLimit, + transIter, transRej, transThresh); // Transparencies per image + if (!trans) { + psError(PPSTACK_ERR_DATA, false, "Unable to measure transparencies"); + return false; + } + for (int i = 0; i < trans->n; i++) { + if (!isfinite(trans->data.F32[i])) { + inputMask->data.U8[i] |= PPSTACK_MASK_CAL; + } + } + + // EAM : the discussion here was not quite right (or at least sloppy). Here is a replacement explanation: + + // For any star, the observed instrumental magnitude on an image and the apparent magnitude are related by: + // M_app = m_inst + zp + c1 * airmass + 2.5log(t) - transparency + // NOTE the sign of 'transparency' this must agree with the definition in pmSourceMatch.c. see, eg, line 457 where + // transparency = m_inst + zp + c1 * airmass + 2.5log(t) - M_app + + // we want to adjust the input images to be in a consistent flux system so that the + // final stack can be generated with a specific target zero point. Any adjustment to + // the flux scale of the image must be made in coordination with the resulting + // zeropoint, exposure time, and airmass such that the above relationship yields the + // same apparent magnitude for a given star: + + // m_inst_i : instrumental mags on input image (in) + // m_inst_o : instrumental mags on re-normalized image (out) + + // m_inst_o + zp_o + c1 * airmass_o + 2.5log(t_o) - trans_o = m_inst_i + zp_i + c1 * airmass_i + 2.5log(t_i) - trans_i + + // m_inst_o = m_inst_i + (zp_i - zp_o) + c1 * (airmass_i - airmass_o) + 2.5log(t_i) - 2.5log(t_o) - trans_i + trans_o + + // zp_i, airmass_i, t_i, trans_i : reported or measured for input image + + // zp_o = zpTarget (from recipe) + // airmass_o = airmassTarget (from recipe) + // t_o = sumExpTime [sum of input exposure times: once images are scale to this time, they can be avereaged] + // trans_o = 0.0 [obviously!] + + // we have 2 cases: (a) all reported ZPs are good or (b) some are bad: + // (a) FPA.ZP = zp_i + c1 * airmass_i + // --> zp[i] = zp_i + c1 * airmass_i + 2.5log(exptime_i) + // (b) zp[i] = c1 * airmass_i + 2.5log(exptime_i) + // NOTE: in case (b), the current code is equating the TARGET zp with the NOMINAL zp, which is wrong. + + // m_inst_o - m_inst_i = zp[i] - zpTarget - c1 * airmassTarget - 2.5log(sumExpTime) - trans_i +#ifdef TESTING_CZW + dumpMatches("source_match2.dat", num, matches, zp, trans); +#endif + if (options->matchZPs) { + options->norm = psVectorAlloc(num, PS_TYPE_F32); + for (int i = 0; i < num; i++) { + if (!isfinite(trans->data.F32[i])) { + continue; + } + psArray *sources = sourceLists->data[i]; // Sources of interest + float magCorr = zp->data.F32[i] - trans->data.F32[i] - 2.5*log10(options->sumExposure) - airmassTerm * airmassTarget; + if (zpExpNum == numGoodImages) { // case (a) + // Using measured zero points, so attempt to set target zero point + // XXX see NOTE above regarding case (b) : this is wrong. the code should load a nominal zero point and supply it above + // + magCorr -= zpTarget; + } + options->norm->data.F32[i] = magCorr; + psLogMsg("ppStack", PS_LOG_INFO, + "Applying scale correction to image %d: %f mag (%f) (with inputMask value %d)\n", + i, magCorr, trans->data.F32[i],inputMask->data.U8[i]); + + for (int j = 0; j < sources->n; j++) { + pmSource *source = sources->data[j]; // Source of interest + source->psfMag += magCorr; + source->apMag += magCorr; + + if (!source) { + continue; + } + // XXX need to apply to apMag as well + psTrace("ppStack",5,"Source corrections: %d %d %f %f %f %f %f %f %f %f\n",i,j, + source->peak->xf,source->peak->yf, + source->psfMag,source->psfMagErr, + airmassTerm,airmassTarget,2.5 * log10(options->sumExposure), + source->psfMag + magCorr); + } + } + } + matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches + + if (zpExpNum == numGoodImages) { + // Producing image with target zero point + options->zp = zpTarget; + options->airmass = airmassTarget; + options->airmassSlope = airmassTerm; + } else { + options->zp = NAN; + } + + + psFree(trans); + +#ifdef TESTING_CZW + // Double check: all transparencies should be zero + { + psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches + psArray *matchedSources = psArrayAlloc(matches->n); + if (!matches) { + psError(PPSTACK_ERR_DATA, false, "Unable to match sources"); + psFree(zp); + return false; + } + psVector *trans = pmSourceMatchRelphot(matches, matchedSources, zp, tol, iter1, starRej1, starSys1, + iter2, starRej2, starSys2, starLimit, + transIter, transRej, transThresh); // Transparencies + for (int i = 0; i < num; i++) { + fprintf(stderr, "Transparency of image %d: %f\n", i, trans->data.F32[i]); + } + + psFree(trans); + psFree(matches); + psFree(matchedSources); + } +#endif + } + + + psFree(zpExp); + +#ifdef ASTROMETRY + // CZW: This is off by default. + // Position offsets + { + psArray *offsets = pmSourceMatchRelastro(matches, num, tol, iter1, starRej1, + iter2, starRej2, starLimit); // Shifts for each image + if (!offsets) { + psError(PPSTACK_ERR_DATA, false, "Unable to measure offsets"); + return false; + } + for (int i = 0; i < num; i++) { + if (options->inputMask->data.U8[i]) { + continue; + } + psArray *sources = sourceLists->data[i]; // Sources of interest + psVector *offset = offsets->data[i]; // Offsets for image + float dx = offset->data.F32[0], dy = offset->data.F32[1]; // Offsets to apply + if (!isfinite(dx) || !isfinite(dy)) { + continue; + } + psLogMsg("ppStack", PS_LOG_INFO, "Applying astrometric correction to image %d: %f,%f\n", + i, dx, dy); + for (int j = 0; j < sources->n; j++) { + pmSource *source = sources->data[j]; // Source of interest + if (!source) { + continue; + } + if (source->modelPSF) { + source->modelPSF->params->data.F32[PM_PAR_XPOS] -= dx; + source->modelPSF->params->data.F32[PM_PAR_YPOS] -= dy; + } + if (source->peak) { + source->peak->xf -= dx; + source->peak->yf -= dy; + } + } + } + psFree(offsets); + } +#endif + +#if (defined TESTING && defined ASTROMETRY) + // CZW: This is off by default. + // Double check: all offsets should be zero + { + psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches + if (!matches) { + psError(PPSTACK_ERR_DATA, false, "Unable to match sources"); + psFree(zp); + return false; + } + psArray *offsets = pmSourceMatchRelastro(matches, num, tol, iter1, starRej1, + iter2, starRej2, starLimit); // Shifts for each image + for (int i = 0; i < num; i++) { + psVector *offset = offsets->data[i]; // Offsets for image + fprintf(stderr, "Offset of image %d: %f,%f\n", i, offset->data.F32[0], offset->data.F32[1]); + } + psFree(offsets); + psFree(matches); + } +#endif + + + if (options->photometry) { + // Save best matches for future photometry + psArray *sourcesBest = options->sources = psArrayAllocEmpty(matches->n); + // XXX something of a hack: require at least 2 detections or the nominated fraction of the max possible + //int minMatches = PS_MAX(2, fracMatch * num);// Minimum number of matches required + //MEH - if many poor images, will hit oddity with num fraction. use numGoodImages instead + int minMatches = PS_MAX(2, fracMatch * numGoodImages);// Minimum number of matches required + + int good0 = 0; + for (int i = 0; i < num; i++) { + if (inputMask->data.U8[i]) { + continue; + } + good0 = i; + break; + } + + for (int i = 0; i < matches->n; i++) { + pmSourceMatch *match = matches->data[i]; // Match of interest + if (match->num < minMatches) { + continue; + } + + // We need to grab a single instance of this source: just take the first available + // MEH - if first available is a rejected image, then problem. so take first non-rejected instance above + // int image = match->image->data.S32[0]; // Index of image + // int index = match->index->data.S32[0]; // Index of source within image + int image = match->image->data.S32[good0]; // Index of image + int index = match->index->data.S32[good0]; // Index of source within image + psArray *sources = sourceLists->data[image]; // Sources for image + pmSource *source = sources->data[index]; // Source of interest + + // CZW: Accept the first source for all properties, except for the magnitudes. + // We have average magnitudes from relphot, so we should use those + pmSource *photsource = matchedSources->data[i]; + if ((photsource)&&(isfinite(photsource->psfMag))) { + // CZW: photsource is the average relphot mag, which is equal to + // (primed are averaged values for this discussion) + // Therefore, the correct magnitude for psfMag' is: + // psfMag' = (photsource - zpIN_j + magCorr) + // apMag - psfMag should be retained, so: + // apMag - psfMag = apMag' - psfMag' + // apMag' = apMag + psfMag' - psfMag + source->apMag = source->apMag + (photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image] - source->psfMag); + source->psfMag = photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image]; + + psTrace("ppStack", 5, "Best Source %d/%ld @ %f %f %f %f %f %d",i,sourcesBest->n,source->peak->xf,source->peak->yf, + source->psfMag, + photsource->psfMag + options->norm->data.F32[image] - zp->data.F32[image] , + source->psfMagErr,-1); + psArrayAdd(sourcesBest, sourcesBest->n, source); + } + } + psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis", sourcesBest->n); + + } + psFree(zp); + psFree(matches); + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackStats.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackStats.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackStats.c (revision 41840) @@ -0,0 +1,27 @@ +#include "ppStack.h" + +bool ppStackStats(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) +{ + psAssert(stack, "Require stack"); + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + // Statistics on output + if (options->stats) { + psTrace("ppStack", 1, "Gathering statistics on stacked image....\n"); + psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + pmFPAview *view = pmFPAviewAlloc(0); // View to readout + view->chip = view->cell = view->readout = 0; + + ppStatsFPA(options->stats, options->outRO->parent->parent->parent, view, maskBad, config); + + psFree(view); + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackTarget.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackTarget.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackTarget.c (revision 41840) @@ -0,0 +1,236 @@ +#include "ppStack.h" + +#define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_SATURATED | \ + PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_EXT_LIMIT) // Mask to apply to input sources + +// Get coordinates from a source +static void coordsFromSource(float *x, float *y, float *mag, const pmSource *source) +{ + assert(x && y); + assert(source); + + if (source->modelPSF) { + *x = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + *y = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + } else { + *x = source->peak->xf; + *y = source->peak->yf; + } + if (mag) { + *mag = source->psfMag; + } + return; +} + +// Filter a list of sources to exclude sources with near neighbours +static psArray *stackSourcesFilter(psArray *sources, // Source list to filter + int exclusion, // Exclusion zone, pixels + float minMagDiff // Minimum magnitude difference + ) +{ + psAssert(sources && sources->n > 0, "Require array of sources"); + if (exclusion <= 0) { + return psMemIncrRefCounter(sources); + } + exclusion = 2; + + int num = sources->n; // Number of sources + psVector *x = psVectorAlloc(num, PS_TYPE_F32), *y = psVectorAlloc(num, PS_TYPE_F32); // Coordinates + psVector *mag = psVectorAlloc(num, PS_TYPE_F32); // Magnitudes + int numGood = 0; // Number of good sources + for (int i = 0; i < num; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (!source) { + continue; + } + coordsFromSource(&x->data.F32[numGood], &y->data.F32[numGood], &mag->data.F32[numGood], source); + numGood++; + } + x->n = y->n = mag->n = numGood; + + psTree *tree = psTreePlant(2, 2, PS_TREE_EUCLIDEAN, x, y); // kd tree + + psArray *filtered = psArrayAllocEmpty(numGood); // Filtered list of sources + psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of source + int numFiltered = 0; // Number of filtered sources + for (int i = 0; i < num; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (!source) { + continue; + } + float xSource, ySource; // Coordinates of source + float Smag; // magnitude of source + coordsFromSource(&xSource, &ySource, &Smag, source); + + coords->data.F64[0] = xSource; + coords->data.F64[1] = ySource; + + psVector *indices = psTreeAllWithin(tree, coords, exclusion); // Number within exclusion zone + psTrace("ppStack", 5, "Filtering: Source at %.0lf %.0lf with mag %f has %ld sources in exclusion zone", + coords->data.F64[0], coords->data.F64[1], Smag, indices->n); + if (indices->n == 1) { + // Only itself inside the exclusion zone + filtered = psArrayAdd(filtered, filtered->n, source); + } else { + float inMag = mag->data.F32[i]; // Input magnitude + bool filter = false; // Filter this source? + for (int j = 0; j < indices->n; j++) { + long index = indices->data.S64[j]; // Index of matching source + float compareMag = mag->data.F32[index]; // Magnitude of matching source + if (fabsf(inMag - compareMag) < minMagDiff) { + filter = true; + break; + } + } + if (!filter) { + filtered = psArrayAdd(filtered, filtered->n, source); + } else { + numFiltered++; + } + } + } + psFree(coords); + psFree(tree); + psFree(x); + psFree(y); + psFree(mag); + + psLogMsg("ppStack", PS_LOG_INFO, "Filtered out %d of %d sources", numFiltered, numGood); + + return filtered; +} + + +psImage *ppStackTarget(ppStackOptions *options, pmConfig *config) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + psAssert(options->psf, "Require target PSF"); + psAssert(options->sourceLists, "Require source lists"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + psAssert(recipe, "We've thrown an error on this before."); + + psString maskValStr = psMetadataLookupStr(NULL, recipe, "MASK.VAL"); // Name of input bits to mask + psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Input bits to mask + float targetFrac = psMetadataLookupF32(NULL, recipe, "TARGET.FRAC"); // Target min flux fraction of noise + + int num = options->num; // Number of inputs + int numCols = 0, numRows = 0; // Size of image + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + float minVariance = INFINITY; // Minimum variance + for (int i = 0; i < num; i++) { + if (options->inputMask->data.U8[i]) { + continue; + } + psTrace("ppStack", 2, "Characterising image %d....\n", i); + pmFPAfileActivate(config->files, false, NULL); + ppStackFileActivationSingle(config, PPSTACK_FILES_TARGET, true, i); + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT.VARIANCE", i); // File to read + pmFPAview *view = ppStackFilesIterateDown(config); + if (!view) { + psFree(rng); + return NULL; + } + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); // Input readout + psFree(view); + + if (numCols == 0 && numRows == 0) { + numCols = readout->variance->numCols; + numRows = readout->variance->numRows; + } else if (numCols != readout->variance->numCols || + numRows != readout->variance->numRows) { + psError(PPSTACK_ERR_ARGUMENTS, true, "Sizes of input images don't match: %dx%d vs %dx%d", + readout->variance->numCols, readout->variance->numRows, numCols, numRows); + psFree(rng); + return NULL; + } + + psImage *variance = readout->variance, *mask = readout->mask; // Dereference images + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (!isfinite(variance->data.F32[y][x])) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskVal; + } + } + } + + psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics for background + float mean = NAN; // Measured mean variance + if (!psImageBackground(bg, NULL, variance, mask, maskVal, rng)) { + psErrorClear(); + // Retry using all the available pixels + bg->nSubsample = variance->numCols * variance->numRows + 1; + if (!psImageStats(bg, variance, mask, maskVal)) { + psLogMsg("ppStack", PS_LOG_DETAIL, + "Couldn't measure mean variance for image %d; retrying.", i); + psErrorClear(); + // Retry with desperate statistic + bg->options = PS_STAT_SAMPLE_MEAN; + if (!psImageStats(bg, variance, mask, maskVal)) { + psWarning("Unable to measure mean variance for image %d --- rejecting.", i); + psErrorStackPrint(stderr, "Unable to measure mean variance for image %d", i); + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_PSF; + goto DONE; + } else { + // Desperate retry + mean = bg->sampleMean; + } + } else { + // Retry with all available pixels + mean = bg->robustMedian; + } + } else { + // First attempt + mean = bg->robustMedian; + } + + float norm = powf(10.0, -0.4 * options->norm->data.F32[i]); // Normalisation from stars + float meanVariance = mean * PS_SQR(norm); // Mean variance in normalised image + + if (meanVariance < minVariance) { + minVariance = meanVariance; + } + + DONE: + psFree(bg); + if (!ppStackFilesIterateUp(config)) { + psFree(rng); + return NULL; + } + + ppStackMemDump("target"); + } + psFree(rng); + + float minFlux = targetFrac * sqrtf(minVariance); // Minimum flux for target image + + int footprint = psMetadataLookupS32(NULL, ppsub, "STAMP.FOOTPRINT"); // Stamp half-size + int size = psMetadataLookupS32(NULL, ppsub, "KERNEL.SIZE"); // Kernel half-size + float minMagDiff = psMetadataLookupF32(NULL, recipe, "TARGET.MINMAG"); // Minimum magnitude difference + + // For the sake of stamps, remove nearby sources + psArray *stampSources = stackSourcesFilter(options->sources, footprint, minMagDiff); // Filtered list + + bool oldThreads = pmReadoutFakeThreads(true); // Old threading state + pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout with target PSF + if (!pmReadoutFakeFromSources(fake, numCols, numRows, stampSources, SOURCE_MASK, NULL, NULL, options->psf, + minFlux, footprint + size, false, true)) { + psError(PPSTACK_ERR_DATA, false, "Unable to generate fake image with target PSF."); + psFree(fake); + return NULL; + } + pmReadoutFakeThreads(oldThreads); + + psFree(stampSources); + + psImage *target = psMemIncrRefCounter(fake->image); + psFree(fake); + + return target; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackThread.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackThread.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackThread.c (revision 41840) @@ -0,0 +1,286 @@ +#include "ppStack.h" + +#define THREAD_WAIT 10000 // Microseconds to wait if thread is not available + +static void stackThreadFree(ppStackThread *thread) +{ + psFree(thread->readouts); + return; +} + +ppStackThread *ppStackThreadAlloc(psArray *readouts) +{ + ppStackThread *thread = psAlloc(sizeof(ppStackThread)); + psMemSetDeallocator(thread, (psFreeFunc)stackThreadFree); + + thread->readouts = psMemIncrRefCounter(readouts); + thread->read = false; + thread->busy = false; + thread->firstScan = 0; + thread->lastScan = 0; + return thread; +} + +static void stackThreadDataFree(ppStackThreadData *stack) +{ + psFree(stack->threads); + for (int i = 0; i < stack->imageFits->n; i++) { + if (stack->imageFits->data[i]) { + psFitsClose(stack->imageFits->data[i]); + } + if (stack->maskFits->data[i]) { + psFitsClose(stack->maskFits->data[i]); + } + if (stack->varianceFits->data[i]) { + psFitsClose(stack->varianceFits->data[i]); + } + stack->imageFits->data[i] = stack->maskFits->data[i] = stack->varianceFits->data[i] = NULL; + } + psFree(stack->imageFits); + psFree(stack->maskFits); + psFree(stack->varianceFits); + psFree(stack->bkgFits); + return; +} + +ppStackThreadData *ppStackThreadDataSetup(const ppStackOptions *options, const pmConfig *config, bool conv) +{ + psAssert(options, "Require options"); + psAssert(config, "Require configuration"); + + const psArray *cells = options->cells; // Array of input cells + const psArray *imageNames = conv ? options->convImages : options->origImages; // Names of images to read + const psArray *maskNames = conv ? options->convMasks : options->origMasks; // Names of masks to read + const psArray *varianceNames = conv ? options->convVariances : options->origVariances; // Variance names + const psArray *covariances = conv ? options->convCovars : options->origCovars; // Covariance matrices + + PS_ASSERT_ARRAY_NON_NULL(cells, NULL); + if (imageNames) { + PS_ASSERT_ARRAYS_SIZE_EQUAL(cells, imageNames, NULL); + } + if (maskNames) { + PS_ASSERT_ARRAYS_SIZE_EQUAL(cells, maskNames, NULL); + } + if (varianceNames) { + PS_ASSERT_ARRAYS_SIZE_EQUAL(cells, varianceNames, NULL); + } + if (covariances) { + PS_ASSERT_ARRAYS_SIZE_EQUAL(cells, covariances, NULL); + } + + ppStackThreadData *stack = psAlloc(sizeof(ppStackThreadData)); // Thread data, to return + psMemSetDeallocator(stack, (psFreeFunc)stackThreadDataFree); + + stack->lastScan = 0; + + int numInputs = cells->n; // Number of inputs + stack->imageFits = psArrayAlloc(numInputs); + stack->maskFits = psArrayAlloc(numInputs); + stack->varianceFits = psArrayAlloc(numInputs); + stack->bkgFits = psArrayAlloc(numInputs); + for (int i = 0; i < numInputs; i++) { + if (!cells->data[i]) { + // Bad image + continue; + } + +// Open an image +#define IMAGE_OPEN(NAMES, FITS, INDEX) \ + if (NAMES) { \ + psString resolved = pmConfigConvertFilename((NAMES)->data[INDEX], config, false, false); \ + (FITS)->data[INDEX] = psFitsOpen(resolved, "r"); \ + if (!(FITS)->data[INDEX]) { \ + psError(PPSTACK_ERR_IO, false, "Unable to open file %s", (char*)(NAMES)->data[INDEX]); \ + psFree(resolved); \ + return NULL; \ + } \ + psFree(resolved); \ + } + + IMAGE_OPEN(imageNames, stack->imageFits, i); + if (maskNames) { + IMAGE_OPEN(maskNames, stack->maskFits, i); + } + if (varianceNames) { + IMAGE_OPEN(varianceNames, stack->varianceFits, i); + } + } + + int numThreads = psMetadataLookupS32(NULL, config->arguments, "NTHREADS"); // Number of threads + + // Generate readouts for each input file in each file group + psArray *threads = psArrayAlloc(numThreads + 1); // A stack for each thread + for (int i = 0; i < threads->n; i++) { + psArray *readouts = psArrayAlloc(numInputs); // Input readouts + for (int j = 0; j < numInputs; j++) { + pmCell *cell = cells->data[j]; // Cell with data + if (!cell) { + continue; + } + pmReadout *ro = pmReadoutAlloc(cell); // Readout for thread + if (covariances) { + ro->covariance = psMemIncrRefCounter(covariances->data[j]); + } + readouts->data[j] = ro; + } + threads->data[i] = ppStackThreadAlloc(readouts); + psFree(readouts); // Drop reference + } + stack->threads = threads; + + return stack; +} + + +ppStackThread *ppStackThreadRead(bool *status, ppStackThreadData *stack, pmConfig *config, int numChunk, + int overlap) +{ + assert(status); + PS_ASSERT_PTR_NON_NULL(stack, NULL); + PS_ASSERT_PTR_NON_NULL(config, NULL); + + *status = true; + + psArray *threads = stack->threads; // Threads for reading + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe + if (!recipe) { + psError(PPSTACK_ERR_CONFIG, false, "Unable to find recipe %s", PPSTACK_RECIPE); + return NULL; + } + int rows = psMetadataLookupS32(NULL, recipe, "ROWS"); // Number of rows to read per chunk + if (rows <= 0) { + psError(PPSTACK_ERR_CONFIG, false, "ROWS is not set in the recipe."); + return NULL; + } + + // Select an available group + while (true) { + // check for any groups which can read data + for (int j = 0; j < threads->n; j++) { + ppStackThread *thread = threads->data[j]; + if (thread->read) { + continue; + } + + thread->firstScan = stack->lastScan; // Overlap is taken care of in pmReadoutReadChunk + thread->lastScan = stack->lastScan + rows; + stack->lastScan = thread->lastScan; + + psArray *readouts = thread->readouts; + + psTimerStart ("ppStackReadChunk"); + + psTrace("ppStack", 2, "Reading data for chunk %d into group %d....\n", numChunk, j); + for (int i = 0; i < readouts->n; i++) { + pmReadout *ro = readouts->data[i]; // Input readout + if (!ro) { + continue; + } + + // override the recorded last scan + ro->thisImageScan = thread->firstScan; + ro->thisVarianceScan = thread->firstScan; + ro->thisMaskScan = thread->firstScan; + ro->lastImageScan = thread->lastScan; + ro->lastMaskScan = thread->lastScan; + ro->lastVarianceScan = thread->lastScan; + ro->forceScan = true; + + psFits *imageFits = stack->imageFits->data[i]; // FITS file for image + psFits *maskFits = stack->maskFits->data[i]; // FITS file for mask + psFits *varianceFits = stack->varianceFits->data[i]; // FITS file for variance + + int zMax = 0; + bool keepReading = false; + + if (imageFits && pmReadoutMore(ro, imageFits, 0, &zMax, rows, config)) { + keepReading = true; + if (!pmReadoutReadChunk(ro, imageFits, 0, NULL, rows, overlap, config)) { + psError(PPSTACK_ERR_IO, false, + "Unable to read chunk %d for file PPSTACK.INPUT %d", + numChunk, i); + *status = false; + return NULL; + } + } + + if (maskFits && pmReadoutMoreMask(ro, maskFits, 0, &zMax, rows, config)) { + keepReading = true; + if (!pmReadoutReadChunkMask(ro, maskFits, 0, NULL, rows, overlap, config)) { + psError(PPSTACK_ERR_IO, false, + "Unable to read chunk %d for file PPSTACK.INPUT.MASK %d", + numChunk, i); + *status = false; + return NULL; + } + } + + if (varianceFits && pmReadoutMoreVariance(ro, varianceFits, 0, &zMax, rows, config)) { + keepReading = true; + if (!pmReadoutReadChunkVariance(ro, varianceFits, 0, NULL, rows, overlap, config)) { + psError(PPSTACK_ERR_IO, false, + "Unable to read chunk %d for file PPSTACK.INPUT.VARIANCE %d", + numChunk, i); + *status = false; + return NULL; + } + } + if (!keepReading) { + return NULL; + } + } + + thread->read = thread->busy = true; + return thread; + } + + // Check for threads that are ready to read + bool wait = true; + for (int j = 0; j < threads->n; j++) { + ppStackThread *thread = threads->data[j]; + if (thread->busy) { + continue; + } + thread->read = false; + wait = false; + } + if (wait) { + // No threads currently available + usleep(THREAD_WAIT); + } + } + return NULL; +} + + +void ppStackSetThreads(void) +{ + static bool threaded = false; // Are we running threaded? + if (threaded) { + psAbort("Already running threaded."); + } + threaded = true; + + { + psThreadTask *task = psThreadTaskAlloc("PPSTACK_INITIAL_COMBINE", 3); + task->function = &ppStackReadoutInitialThread; + psThreadTaskAdd(task); + psFree(task); + } + + { + psThreadTask *task = psThreadTaskAlloc("PPSTACK_INSPECT", 3); + task->function = &ppStackInspect; + psThreadTaskAdd(task); + psFree(task); + } + + { + psThreadTask *task = psThreadTaskAlloc("PPSTACK_FINAL_COMBINE", 7); + task->function = &ppStackReadoutFinalThread; + psThreadTaskAdd(task); + psFree(task); + } + + return; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackThread.h =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackThread.h (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackThread.h (revision 41840) @@ -0,0 +1,49 @@ +#ifndef PPSTACK_THREAD_H +#define PPSTACK_THREAD_H + +// Thread for stacking chunks +// +// Each input file contributes a readout, into which is read a chunk from that file +typedef struct { + psArray *readouts; // Input readouts to read and stack + bool read; // Has the scan been read? + bool busy; // Is the scan being processed? + int firstScan; // First row of the chunk to be read for this group + int lastScan; // Last row of the chunk to be read for this group +} ppStackThread; + +// Allocator +ppStackThread *ppStackThreadAlloc( + psArray *readouts // Inputs readouts to read and stack + ); + +// Data for threads +typedef struct { + psArray *threads; // Threads doing stacking + int lastScan; // Last row that's been read + psArray *imageFits; // FITS file pointers for images + psArray *maskFits; // FITS file pointers for masks + psArray *varianceFits; // FITS file pointers for variances + psArray *bkgFits; // FITS file pointers for background models +} ppStackThreadData; + +// Set up thread data +ppStackThreadData *ppStackThreadDataSetup( + const ppStackOptions *options, // Options + const pmConfig *config, // Configuration + bool conv // Use convolved products? + ); + +// Read chunk into the first available file thread +ppStackThread *ppStackThreadRead(bool *status, // Status of read + ppStackThreadData *stack, // Stacks available for reading + pmConfig *config, // Configuration + int numChunk, // Chunk number (only for interest) + int overlap // Overlap between subsequent scans + ); + +// Initialise the threads +void ppStackSetThreads(void); + + +#endif Index: /branches/eam_branches/ppStack.20211015/src/ppStackUpdateHeader.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackUpdateHeader.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackUpdateHeader.c (revision 41840) @@ -0,0 +1,150 @@ +#include "ppStack.h" + +#define WCS_TOLERANCE 0.001 // Tolerance for WCS + +bool ppStackUpdateHeader(ppStackThreadData *stack, ppStackOptions *options, pmConfig *config) { + + pmReadout *outRO = options->outRO; // Output readout + pmReadout *expRO = options->expRO; + + // Propagate WCS + bool wcsDone = false; // Have we done the WCS? + for (int i = 0; i < options->num && !wcsDone; i++) { + if ((options->inputMask)&&(options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i])) { + continue; + } + + ppStackThread *thread = stack->threads->data[0]; // Representative stack + pmReadout *inRO = thread->readouts->data[i]; // Template readout + if (inRO && !wcsDone) { + // Copy astrometry over + wcsDone = true; + pmHDU *inHDU = pmHDUFromCell(inRO->parent); // Template HDU + pmHDU *outHDU = pmHDUFromCell(outRO->parent); // Output HDU + pmChip *outChip = outRO->parent->parent; // Output chip + pmFPA *outFPA = outChip->parent; // Output FPA + if (!outHDU || !inHDU) { + psWarning("Unable to find HDU at FPA level to copy astrometry."); + } else { + if (!pmAstromReadWCS(outFPA, outChip, inHDU->header, 1.0)) { + psErrorClear(); + psWarning("Unable to read WCS astrometry from input FPA."); + wcsDone = false; + } else { + if (!outHDU->header) { + outHDU->header = psMetadataAlloc(); + } + if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_TOLERANCE)) { + psErrorClear(); + psWarning("Unable to write WCS astrometry to output FPA."); + wcsDone = false; + } + } + } + } + } + + // Put version information into the header + pmHDU *hdu = pmHDUFromCell(outRO->parent); + if (!hdu) { + psError(PPSTACK_ERR_PROG, false, "Unable to find HDU for output."); + return false; + } + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppStackVersionHeader(hdu->header); + + // other interesting header info + psMetadataAddS32(hdu->header, PS_LIST_TAIL, "NINPUTS", PS_META_REPLACE, "Number of input images", options->num); + + psString stacktype = psMetadataLookupStr(NULL, config->arguments, "STACK_TYPE"); // NIGHTLY, DEEP, BEST_IQ + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "STK_TYPE", PS_META_REPLACE, "type of stack", stacktype); + + psString stackID = psMetadataLookupStr(NULL, config->arguments, "-stack_id"); // stack ID (eg, 123222) + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "STK_ID", PS_META_REPLACE, "type of stack", stackID); + + psString skycellID = psMetadataLookupStr(NULL, config->arguments, "-skycell_id"); // skycell ID (eg, skycell.101.00) + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "SKYCELL", PS_META_REPLACE, "type of stack", skycellID); + + psString tessID = psMetadataLookupStr(NULL, config->arguments, "-tess_id"); // tessellation ID (eg, RINGS.V0) + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "TESS_ID", PS_META_REPLACE, "type of stack", tessID); + + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "AIRM_SLP", PS_META_REPLACE, "airmass slope", options->airmassSlope); + + + // write the following fields to the output headers: + // INP_NNNN : input file names + // SCL_NNNN : scale factor for each input file + // ZPT_NNNN : zero point of input file + // EXT_NNNN : exptime of input file + // AIR_NNNN : airmass of input file + + // use separate loops so they are blocked together in the headers (more readable) + + // save the input filenames + for (int i = 0; i < options->num; i++) { + char field[64]; + snprintf (field, 64, "INP_%04d", i); + + psString basename = psStringFileBasename(options->origImages->data[i]); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, field, PS_META_REPLACE, "input image name", basename); + psFree(basename); + } + + // save the input normalizations + for (int i = 0; i < options->num; i++) { + char field[64]; + + float value = options->norm ? pow(10.0, -0.4*options->norm->data.F32[i]) : NAN; + if ((options->inputMask)&&(options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i])) { + value = 0.0; // If this is NAN, then it's recorded in the header as a character string instead of a float. + } + snprintf (field, 64, "SCL_%04d", i); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, field, PS_META_REPLACE, "input image scale factor", value); + } + // save the input exptimes + for (int i = 0; i < options->num; i++) { + char field[64]; + + float value = options->zpInput ? options->zpInput->data.F32[i] : NAN; + snprintf (field, 64, "ZPT_%04d", i); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, field, PS_META_REPLACE, "input image zero point", value); + } + // save the input normalizations + for (int i = 0; i < options->num; i++) { + char field[64]; + + float value = options->expTimeInput ? options->expTimeInput->data.F32[i] : NAN; + snprintf (field, 64, "EXP_%04d", i); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, field, PS_META_REPLACE, "input image exptime", value); + } + // save the input normalizations + for (int i = 0; i < options->num; i++) { + char field[64]; + + float value = options->airmassInput ? options->airmassInput->data.F32[i] : NAN; + snprintf (field, 64, "AIR_%04d", i); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, field, PS_META_REPLACE, "input image airmass", value); + } + + // Copy information into expRO, because it should be there too. + if ((expRO)&&(expRO->parent)) { + pmHDU *expROhdu = pmHDUFromCell(expRO->parent); + if (!expROhdu->header) { + expROhdu->header = psMetadataAlloc(); + } + + expRO->parent->parent->parent->hdu->header = psMetadataCopy(expRO->parent->parent->parent->hdu->header, + outRO->parent->parent->parent->hdu->header); + + expRO->parent->concepts = psMetadataCopy(expRO->parent->concepts, + outRO->parent->concepts); + expRO->parent->parent->concepts = psMetadataCopy(expRO->parent->parent->concepts, + outRO->parent->concepts); + expRO->parent->parent->parent->concepts = psMetadataCopy(expRO->parent->parent->parent->concepts, + outRO->parent->parent->parent->concepts); + } + + return true; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackVersion.c =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackVersion.c (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackVersion.c (revision 41840) @@ -0,0 +1,107 @@ +#include "ppStack.h" +#include "ppStackVersionDefinitions.h" + +#ifndef PPSTACK_VERSION +#error "PPSTACK_VERSION is not set" +#endif +#ifndef PPSTACK_BRANCH +#error "PPSTACK_BRANCH is not set" +#endif +#ifndef PPSTACK_SOURCE +#error "PPSTACK_SOURCE is not set" +#endif + +psString ppStackVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PPSTACK_BRANCH, PPSTACK_VERSION); + return value; +} + +psString ppStackSource(void) +{ + return psStringCopy(PPSTACK_SOURCE); +} + +psString ppStackVersionLong(void) +{ + psString version = ppStackVersion(); // Version, to return + psString source = ppStackSource(); // Source + + psStringPrepend(&version, "ppStack "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + + return version; +}; + + +bool ppStackVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "ppStack at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + psModulesVersionHeader(header); + psphotVersionHeader(header); + ppStatsVersionHeader(header); + + psString version = ppStackVersion(); // Software version + psString source = ppStackSource(); // Software source + + psMetadataAddStr(header, PS_LIST_TAIL, "STACK_V", PS_META_REPLACE, NULL, PPSTACK_VERSION); + + psStringPrepend(&version, "ppStack version: "); + psStringPrepend(&source, "ppStack source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + + return true; +} + +void ppStackVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("ppStack", PS_LOG_INFO, "ppStack at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString psphot = psphotVersionLong(); // psphot version + psString ppStats = ppStatsVersionLong(); // psastro version + psString ppStack = ppStackVersionLong(); // ppStack version + + psLogMsg("ppStack", PS_LOG_INFO, "%s", pslib); + psLogMsg("ppStack", PS_LOG_INFO, "%s", psmodules); + psLogMsg("ppStack", PS_LOG_INFO, "%s", psphot); + psLogMsg("ppStack", PS_LOG_INFO, "%s", ppStats); + psLogMsg("ppStack", PS_LOG_INFO, "%s", ppStack); + + psFree(pslib); + psFree(psmodules); + psFree(psphot); + psFree(ppStats); + psFree(ppStack); + + return; +} Index: /branches/eam_branches/ppStack.20211015/src/ppStackVersionDefinitions.h.in =================================================================== --- /branches/eam_branches/ppStack.20211015/src/ppStackVersionDefinitions.h.in (revision 41840) +++ /branches/eam_branches/ppStack.20211015/src/ppStackVersionDefinitions.h.in (revision 41840) @@ -0,0 +1,8 @@ +#ifndef PPSTACK_VERSION_DEFINITIONS_H +#define PPSTACK_VERSION_DEFINITIONS_H + +#define PPSTACK_VERSION @PPSTACK_VERSION@ // SVN version +#define PPSTACK_BRANCH @PPSTACK_BRANCH@ // SVN branch +#define PPSTACK_SOURCE @PPSTACK_SOURCE@ // SVN source + +#endif