Index: /branches/eam_branches/ppImage.20240412/Doxyfile.in =================================================================== --- /branches/eam_branches/ppImage.20240412/Doxyfile.in (revision 42655) +++ /branches/eam_branches/ppImage.20240412/Doxyfile.in (revision 42655) @@ -0,0 +1,1310 @@ +# Doxyfile 1.5.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file that +# follow. The default is UTF-8 which is also the encoding used for all text before +# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into +# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of +# possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = ppImage + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = ipp-2.7.dev + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, +# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, +# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, +# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is +# documented as struct with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code where the coding convention is that all structs are +# typedef'ed and only the typedef is referenced never the struct's name. + +TYPEDEF_HIDES_STRUCT = NO + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be extracted +# and appear in the documentation as a namespace called 'anonymous_namespace{file}', +# where file will be replaced with the base name of the file that contains the anonymous +# namespace. By default anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text " + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ./src + +# This tag can be used to specify the character encoding of the source files that +# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default +# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. +# See http://www.gnu.org/software/libiconv for the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = *.h + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the output. +# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, +# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH +# then you must also enable this option. If you don't then doxygen will produce +# a warning and turn it on anyway + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to +# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to +# specify the directory where the mscgen tool resides. If left empty the tool is assumed to +# be found in the default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the number +# of direct children of the root node in a graph is already larger than +# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO Index: /branches/eam_branches/ppImage.20240412/Makefile.am =================================================================== --- /branches/eam_branches/ppImage.20240412/Makefile.am (revision 42655) +++ /branches/eam_branches/ppImage.20240412/Makefile.am (revision 42655) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *~ core core.* Index: /branches/eam_branches/ppImage.20240412/autogen.sh =================================================================== --- /branches/eam_branches/ppImage.20240412/autogen.sh (revision 42655) +++ /branches/eam_branches/ppImage.20240412/autogen.sh (revision 42655) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppImage +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOLIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/eam_branches/ppImage.20240412/configure.ac =================================================================== --- /branches/eam_branches/ppImage.20240412/configure.ac (revision 42655) +++ /branches/eam_branches/ppImage.20240412/configure.ac (revision 42655) @@ -0,0 +1,45 @@ +AC_PREREQ(2.61) + +AC_INIT([ppImage], [1.1.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDLDFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +dnl ------------------------------------------------------------ + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) +PKG_CHECK_MODULES([PSPHOT], [psphot >= 0.8.0]) +PKG_CHECK_MODULES([PSASTRO], [psastro >= 0.8.0]) +PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) + +IPP_STDOPTS +IPP_STDCFLAGS + +IPP_VERSION + +AC_SUBST([PPIMAGE_CFLAGS]) +AC_SUBST([PPIMAGE_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile + Doxyfile +]) +AC_OUTPUT Index: /branches/eam_branches/ppImage.20240412/notes.txt =================================================================== --- /branches/eam_branches/ppImage.20240412/notes.txt (revision 42655) +++ /branches/eam_branches/ppImage.20240412/notes.txt (revision 42655) @@ -0,0 +1,15 @@ + +20090611 : adjusting things to allow the readnoise to be an image map for each cell + + * pmReadoutSetVaraince : when variance is created, we are suppling a single CELL.READNOISE value -- change this to an optional image (carried through concepts?) + + I need to: + + * define a format for the readnoise map (image map in SPLIT/MEF format?) + * load the readnoise map + + * supply the map for each cell to the pmReadoutSetVariance (pointer to ppImageMap?) + pmReadoutSetVariance -> add psImageMap to API (also pmReadoutGenerateVariance) + + * add recipe information to define the source of the READNOISE info. + Index: /branches/eam_branches/ppImage.20240412/src/Makefile.am =================================================================== --- /branches/eam_branches/ppImage.20240412/src/Makefile.am (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/Makefile.am (revision 42655) @@ -0,0 +1,75 @@ + +bin_PROGRAMS = ppImage + +noinst_HEADERS = ppImage.h ppImageErrorCodes.h + +# Force recompilation of ppImageVersion.c, since it gets the version information +ppImageVersion.c: ppImageVersionDefinitions.h +ppImageVersionDefinitions.h: ppImageVersionDefinitions.h.in FORCE + pslib-setsvnversion.pl PPIMAGE ppImageVersionDefinitions.h.in ppImageVersionDefinitions.h +FORCE: ; + +BUILT_SOURCES = ppImageErrorCodes.h ppImageErrorCodes.c ppImageVersionDefinitions.h + +ppImage_CFLAGS = $(PPIMAGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) +ppImage_LDFLAGS = $(PPIMAGE_LIBS) $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) +ppImage_SOURCES = \ + ppImage.c \ + ppImageArguments.c \ + ppImageParseCamera.c \ + ppImageLoop.c \ + ppImageCleanup.c \ + ppImageOptions.c \ + ppImageDetrendReadout.c \ + ppImageDetrendRecord.c \ + ppImageDetrendNonLinear.c \ + ppImageDetrendNewNonLinear.c \ + ppImageDetrendFringe.c \ + ppImageDetrendFree.c \ + ppImageDetrendPattern.c \ + ppImageErrorCodes.c \ + ppImageRebinReadout.c \ + ppImageMosaic.c \ + ppImageMosaicBackground.c \ + ppImageMaskStats.c \ + ppImagePhotom.c \ + ppImageAstrom.c \ + ppImageAddstar.c \ + ppImageStatsOutput.c \ + ppImagePixelStats.c \ + ppImageMetadataStats.c \ + ppImageReplaceBackground.c \ + ppImageMeasureCrosstalk.c \ + ppImageCorrectCrosstalk.c \ + ppImageDefineFile.c \ + ppImageSetMaskBits.c \ + ppImageSetThreads.c \ + ppImageBurntoolMask.c \ + ppImageParityFlip.c \ + ppImageCheckCTE.c \ + ppImageCheckNoise.c \ + ppImageFileCheck.c \ + ppImageVersion.c \ + ppImageMemory.c \ + ppImageAddNoise.c \ + ppImageSquashNANs.c \ + ppImageRandomGaussian.c \ + ppImageAuxiliaryMask.c \ + ppImageBurntoolApply.c + +CLEANFILES = *~ ppImageErrorCodes.h ppImageErrorCodes.c + +EXTRA_DIST = ppImageErrorCodes.dat ppImageErrorCodes.c.in ppImageErrorCodes.h.in + + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` + +ppImageErrorCodes.h : ppImageErrorCodes.dat ppImageErrorCodes.h.in + $(ERRORCODES) --data=ppImageErrorCodes.dat --outdir=. ppImageErrorCodes.h + +ppImageErrorCodes.c : ppImageErrorCodes.dat ppImageErrorCodes.c.in ppImageErrorCodes.h + $(ERRORCODES) --data=ppImageErrorCodes.dat --outdir=. ppImageErrorCodes.c Index: /branches/eam_branches/ppImage.20240412/src/burnparams.h =================================================================== --- /branches/eam_branches/ppImage.20240412/src/burnparams.h (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/burnparams.h (revision 42655) @@ -0,0 +1,59 @@ +/* A global buffer into which routines can accumulate detections */ +#ifndef _INCLUDED_burnparams_ +#define _INCLUDED_burnparams_ + +/* '#define EXTERN' in just one file, burntool.c, to declare variables */ +#ifndef EXTERN +#define EXTERN extern +#endif + +#define MAXBURN 10000 +EXTERN OBJBOX boxbuf[MAXBURN]; + +EXTERN int *median_buf; /* Generic buffer for integer medians */ +EXTERN int nmedian_buf; + +EXTERN MTYPE *mbuf; /* Buffer for object mask */ +EXTERN int nmbuf; + +EXTERN MTYPE *msbuf; /* Buffer for vetoed star mask */ +EXTERN int nmsbuf; + +EXTERN DTYPE *imbuf; /* Copy of cell data w/o BZERO */ +EXTERN int nimbuf; + +EXTERN int VERBOSE; /* Verbosity level */ + +EXTERN int BZERO; +EXTERN int USHORT_BIAS; /* Bias level restored to ushort stamps */ + +EXTERN int MAX_READ_NOISE; /* Maximum believable read noise (ADU) */ +EXTERN double MIN_EADU; /* Minimum believable e/ADU */ +EXTERN int SAT4SURE; /* Ignore pixels above for noise estimate */ +EXTERN double MIN_BLAST_PASS; /* Allow blasted cells if they have BIG satfrac */ +EXTERN double MAX_BLAST_PASS; /* But not if it's all wiped out! */ + +EXTERN int BURN_THRESH ; /* Threshold for onset of burning */ +EXTERN int TRAIL_THRESH ; /* Trailing might go this low */ +EXTERN int MAX_THRESH ; /* Possibly trailing stars? */ +EXTERN int STAR_THRESH ; /* Threshold for star above sky */ +EXTERN double STAR_FRAC ; /* Fraction to follow star profile */ +EXTERN int PSF_THRESH ; /* Threshold for a star to be a PSF */ + +//EXTERN double XMASK_GROW ; /* Growth of burned boxes in x dir */ +//EXTERN double YMASK_GROW ; /* Growth of burn/star in y dir */ + +EXTERN double BMASK_GROW ; /* Growth of burned boxes in size */ +EXTERN double RMASK_GROW ; /* Growth of burn/star in diameter */ + +EXTERN int MIN_PSF_SIZE ; /* Min box size for stamp selection */ +EXTERN int PSF_CTR_TOL ; /* Choose max or box ctr for stamp */ +EXTERN int CONCAT_FITS; /* Write concat FITS for PSF? (else 3D) */ +EXTERN int MAX_PSF_PER_CELL ; /* Max number of PSF stars accepted per cell */ + +EXTERN double NEGLIGIBLE_TRAIL; /* Don't sweat less than this * sigma */ +EXTERN int EXPIRE_TRAIL_TIME; /* Expire trails after this interval */ + +EXTERN int PERSIST_RETAIN; /* Retain bad-slope persistence fits */ + +#endif /* _INCLUDED_burnparams_ */ Index: /branches/eam_branches/ppImage.20240412/src/burntool.h =================================================================== --- /branches/eam_branches/ppImage.20240412/src/burntool.h (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/burntool.h (revision 42655) @@ -0,0 +1,230 @@ +#ifndef _INCLUDED_burntool_ +#define _INCLUDED_burntool_ + +/* Burn correction routines: + ------------------------- + * 1. Identify burned spots from saturation threshold + ? 2. Identify burns and trails morphologically? + ? 2.1 Look for flat-topped stars + ? 2.2 Look for asymmetric, smooth, declining trails? + ? 2.3 Use y overscan? + ? 2.4 Bin in x? + * 3. Mask stars and other cruft? + * 4. Fit up and down row functions + * 5. Read burn time list + * 6. Write updated burn time list + * 8. Undo subtraction + 7. Create and save burn correction FITS table +*/ +/* Algorithm: + ---------- + 1. Find new burned spots + 2. Create new burn entries: x1,x2,y,time=now + 3. For each burn: + if t = now detrail up + else deburn down + if fit negligible delete burn + else save burn fits + */ +/* ToDo: + ----- + * clean up fit start points, esp for persistence + * change burn= and persist= to trailin= and trailout= + * trim edges of fits + * implement MASK_SAT + > test for garbage fits (positive slope, etc) + * test nearly saturated stars for burn. + * undo subtraction (almost perfect -- occasional pixels and baddies) + * star gather into 3D FITS + * test on donuts + * stamp extraction center on box, not max + * Add back nominal bias; output as ushort + * man page... ongoing... + * Scale uses peak/sum? + * Change psf coords to center; save in header not pixels + x better trail function? A tuned ln(x+A) is better, but too hard. + * why various restorations imperfect? edge of cell? + * cell mask argument +... why the honker star not ID'ed as a burn? various other gotchas... + * age away old persistence streaks; merge persistence streaks + * PSF info on median psf + * clean up sky/rms calculation with 5*quartile clipping + * change VERBOSE to a bitmask for access to various portions of the code + * add access to a lot of the tuning variables... + * update man page + tuning, tuning, tuning... + add correction FITS table to MEF +*/ + +#define MAX(a,b) (((a) > (b)) ? (a) : (b)) +#define MIN(a,b) (((a) < (b)) ? (a) : (b)) +#define NINT(x) (x<0?(int)((x)-0.5):(int)((x)+0.5)) +#define ABS(a) (((a) > 0) ? (a) : -(a)) + +// #define STATIC static /* To make function declarations static */ +#define STATIC /* To make function declarations global */ + +#define MAXCELL 64 /* Max cells in an OTA */ +#define MAXSIZE 2048 /* Maximum vertical cell size */ + +#define STAR_RADIUS 4 /* Radius over which a star ctr must be max */ +#define SKY_MARG 3 /* Horiz offset from burn area for sky */ +#define FIT_EDGE 5 /* How far beyond saturation to start fit? */ +#define Y_SCALE 0.01 /* Scale factor for y in fits */ + +typedef signed short int IMTYPE;/* Data type of image */ + +typedef int DTYPE; /* Data type of data copy */ +#define NODATA 0 /* Marker for *No Data */ + +/* Mask codes: keep in order of severity for psf_select()! */ +typedef int MTYPE; /* Data type of mask */ +#define MASK_NONE 0 /* Unmasked pixel (MUST be 0) */ +#define MASK_STAR_HALO 1 /* Extended halo box from star */ +#define MASK_SAT_HALO 2 /* Extended halo box from grow_mask() */ +#define MASK_CTR 3 /* Center mask from star_detect() */ +#define MASK_SAT 4 /* Center mask of saturated pixels */ + +/* Enumeration of function choices */ +#define FUNC_NONE 0 /* No associated function */ +#define BURN_PWR 1 /* Power law */ +#define BURN_EXP 2 /* Exponential */ +#define BURN_BLASTED 3 /* Blasted top to bottom: flag only for IPP */ +#define BURN_POSSLOPE 4 /* Positive slope fit (bad) but significant */ +#define PSF_STAR 9 /* Unfitted: good psf star */ + +/* Fit error codes */ +#define FIT_ERROR 1 /* linearfit failed */ +#define FIT_TOP_ERROR 2 /* Saturation extends to top: no points */ +#define FIT_SLOPE_ERROR 3 /* Unreasonable fit */ +#define FIT_ALL_GONE 4 /* No column survived as significant */ +#define FIT_EXPIRED 9 /* Don't carry any more as persistent */ + +/* Fit parameters */ +#define FIT_MIN_SLOPE -10.0 /* minimum slope which is a credible fit */ +#define FIT_MAX_SLOPE 0.0 /* maximum slope which is a credible fit */ + +/* Verbosity bits */ +#define VISTAMARKER "/tmp/markem.pro" /* Write Vista pro to mark stars?*/ +#define VERB_NORM 0x0001 /* Normal, verbose output */ +#define VERB_DETECT 0x0002 /* Dump detection process */ +#define VERB_PSFSEL 0x0004 /* Dump out PSF selection process */ +#define VERB_FIT 0x0008 /* Dump fit progress */ +#define VERB_FITPROF 0x0010 /* Dump fit profiles */ +#define VERB_VISTA 0x0020 /* Write vista markers as /tmp/markem.pro */ +#define VERB_BOXGROW 0x0040 /* Dump box growth diagnostics */ +#define VERB_MASK 0x0080 /* Write mask in place of corrected image */ + +/* Description of a potentially burned trail */ +typedef struct obj_box { + int cell; /* what cell is this one in? */ + int time; /* PON time when it was created */ + int sx; /* left corner */ + int sy; /* bottom corner */ + int ex; /* right corner */ + int ey; /* top corner */ + int cx; /* center x (position of max) */ + int cy; /* center y (position of max) */ + int max; /* max data value (above sky) */ + int y0m; /* min y value at sx */ + int y0p; /* max y value at sx */ + int y1m; /* min y value at ex */ + int y1p; /* max y value at ex */ + int x0m; /* min x value at sy */ + int x0p; /* max x value at sy */ + int x1m; /* min x value at ey */ + int x1p; /* max x value at ey */ + int sat; /* saturated (i.e. bigger than BURNTHRESH)? */ + int func; /* what are we going to do about it? */ + int diff; /* median diff of up minus down (sum for stars) */ + int up; /* does it trail up or down? (stamp sx for stars) */ + int y0; /* y origin for the fit (stamp sy for stars) */ + int burned; /* do we think it's left a burn trail? */ + int fiterr; /* error fitting the trail? */ + int sxfit; /* starting column for fits */ + int exfit; /* ending column for fits */ + int eyfit; /* y-coord ending column for fits */ + int nfit; /* how many columns were corrected? */ + IMTYPE *stamp; /* postage stamp of this object */ + int *xfit; /* x of each value of the start of correction */ + int *yfit; /* y value of the start of correction */ + double slope; /* slope of fit (stamp sum/max) */ + double *zero; /* zero of fit for each of the columns */ +} OBJBOX; + +/* Info for an entire cell */ +typedef struct cell_info { + int cell; /* Cell number */ + int bias; /* Bias level */ + int sky; /* Sky level */ + int rms; /* RMS in the sky */ + int time; /* PON time of this cell */ + double satfrac; /* Fraction of SAT4SURE saturated pixels */ + int nburn; /* Number of trails left by sat stars */ + OBJBOX *burn; /* Stars which we think left a trail */ + int npersist; /* Number of old persistence streaks */ + OBJBOX *persist; /* Persistent streaks */ + int nstar; /* Number of stars */ + OBJBOX *star; /* Stars, harmless we think */ +} CELL; + +/* Prototypes */ +STATIC int mem_init(int nx, int ny, int NX, int NY); +STATIC int burn_fix(int nx, int ny, int stride, int NY, IMTYPE *buf, + CELL *cell, int cellnum); +STATIC int burn_test(int nx, int ny, int NX, DTYPE *data, int rms, + MTYPE *mask, OBJBOX *box); +STATIC int burn_restore(int nx, int ny, int NX, IMTYPE *buf, CELL *cell); +STATIC int burn_apply(int nx, int ny, int NX, IMTYPE *buf, CELL *cell); +STATIC int persist_read(CELL *cell, const char *infile, int apply, int oldfile); +STATIC int persist_write(CELL *cell, const char *outfile, int oldfile); +STATIC int persist_fix(int nx, int ny, int stride, IMTYPE *buf, CELL *cell); +STATIC int persist_merge(CELL *cell); + +//fh_result persist_fits_read(CELL *cell, const char *filename, int apply); +//fh_result persist_fits_write(CELL *cell, HeaderUnit phu); +//fh_result persist_fits_remove_tables(HeaderUnit phu_in, const char *fileout); + +STATIC int star_detect(int nx, int ny, int NX, int NY, DTYPE *data, + MTYPE *mask, MTYPE *veto, CELL *cell, int cellnum); +STATIC int burn_check(int nx, int ny, int stride, int NY, DTYPE *buf, + MTYPE *mask, CELL *cell); + +STATIC int cell_stats(int nx, int ny, int NX, int NY, DTYPE *data, CELL *cell); +STATIC int burn_blab(CELL *cell); +STATIC int persist_blab(CELL *cell); +STATIC int vista_marker(CELL *cell, char *fname); + +STATIC int fit_trail(int nx, int ny, int NX, DTYPE *data, MTYPE *mask, + OBJBOX *box, int up, int sky, int rms, int fitfunc); +STATIC int sub_fit(int nx, int ny, int NX, IMTYPE *buf, OBJBOX *box, int sign); + +STATIC int psf_select(int nx, int ny, int NX, MTYPE *mask, DTYPE *buf, + int nbox, OBJBOX *box, int size, int sky); +STATIC int psf_write(int nx, int ny, CELL *OTA, int otanum, const char *psffile); +STATIC int psf_write_stats(int nx, int ny, CELL *OTA, int otanum, const char *statfile, int psfavg); +STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias, + double *fwhm, double *q); + +STATIC int grow_box(int nx, int ny, int NX, DTYPE *data, int thr, OBJBOX *box); +STATIC int grow_mask(int nx, int ny, int NX, DTYPE *mask, double xfac, + double yfac, int maskval, int nbox, OBJBOX *box); +STATIC int local_max(int i, int j, int r, int nx, int ny, int NX, DTYPE *data); + +STATIC int wlinearfit(int npt, double *x, double *y, + double *w, double *a, double *b); +STATIC int linearrms(int npt, double *x, double *y, double a, double b, double *rms); +STATIC int int_median(int n, int *key); +STATIC double double_median(int n, double *key); + +STATIC int qsort_int(int n, int *key); +STATIC double qsort_dbl(int n, double *key); + +STATIC void syntax(const char *prog); + +int write_2dfits(int nx, int ny, int sx, int sy, IMTYPE *data, int fd); +int write_3dhdr(int nx, int ny, int nz, int ncmt, char *cmt[], int fd); +int write_2ddata(int nx, int ny, int *ntot, IMTYPE *data, int fd); +int write_3dend(int *ntot, int fd); + +#endif /* _INCLUDED_burntool_ */ Index: /branches/eam_branches/ppImage.20240412/src/ppFocus.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppFocus.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppFocus.c (revision 42655) @@ -0,0 +1,74 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +int main(int argc, char **argv) { + + psLibInit(NULL); + + psTimerStart(TIMER_TOTAL); + + // Parse the configuration and arguments + // Open the input image(s) + // Determine camera, format from header if not already defined + // Construct camera in preparation for reading + pmConfig *config = ppFocusArguments(argc, argv); + if (config == NULL) { + psErrorStackPrint(stderr, " "); + exit(1); + } + + // we search the argument data for the named fileset (argname) + psArray *infiles = psMetadataLookupPtr(NULL, config->arguments, "INPUT"); + if (!infiles) { + psTrace("pmFPAfile", 5, "Failed to find INPUT in argument list"); + exit(1); + } + + // allocate vectors for analysis + psVector *focus = psVectorAllocEmpty(infiles->n, PS_TYPE_F32); + psVector *fwhm = psVectorAllocEmpty(infiles->n, PS_TYPE_F32); + + for (int i = 0; i < infiles->n; i++) { + + // define recipe options + // define the active I/O files + ppImageOptions *options = ppFocusParseCamera(config, i); + if (options == NULL) { + psErrorStackPrint(stderr, " "); + exit(1); + } + + // Image Arithmetic Loop + // XXX ppFocus REQUIRES photom: for it to be true? + // + if (!ppImageLoop(config, options)) { + psErrorStackPrint(stderr, " "); + exit(1); + } + + // determine FWHM at reference location in image + // (also removes PPIMAGE.INPUT from config->files) + ppFocusGetFWHM (config, focus, fwhm); + + ppFocusDropCamera (config); + psFree (options); + } + + ppFocusFitFWHM (config, focus, fwhm); + + psLogMsg ("ppFocus", 3, "complete ppFocus run: %f sec\n", psTimerMark (TIMER_TOTAL)); + + // Cleaning up + psFree (focus); + psFree (fwhm); + ppImageCleanup(config, NULL); + return EXIT_SUCCESS; +} + +// ppFocus is a lot like ppImage, but with a few important differences: +// - the input list is a set of independent images (not multiple files for a single image) +// - each pass to ppImageLoop performs the analysis on a different pmFPAfile +// - after each ppImageLoop, grap the input pmFPAfile and extract the FWHM stats Index: /branches/eam_branches/ppImage.20240412/src/ppFocusArguments.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppFocusArguments.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppFocusArguments.c (revision 42655) @@ -0,0 +1,72 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +static void usage (void) { + fprintf (stderr, "USAGE: ppFocus [-file focus.*.fits] [-list INPUT.txt] OUTPUT\n"); + exit (2); +} + +pmConfig *ppFocusArguments(int argc, char **argv) { + + int N; + bool status; + + if (argc == 1) usage (); + + // load the site-wide configuration information + pmConfig *config = pmConfigRead(&argc, argv, RECIPE_NAME); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + // save the following additional recipe values based on command-line options + // these options override the PPIMAGE recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, RECIPE_NAME); + + // save these recipe options until we have loaded the options + // psMetadata *options = psMetadataAlloc (); + // psMetadataAddPtr (config->arguments, PS_LIST_TAIL, "PPIMAGE.OPTIONS", PS_DATA_METADATA, "", options); + + // the following options override the PPIMAGE recipe options + + // recipe option: -usemask : override MASK setting in phase2.recipe + if ((N = psArgumentGet(argc, argv, "-usemask"))) { + psArgumentRemove (N, &argc, argv); + psMetadataAddBool (options, PS_LIST_TAIL, "MASK", PS_META_REPLACE, "", true); + psArgumentRemove (N, &argc, argv); + } + + // XXX add other PPIMAGE recipe options here + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { usage ();} + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + pmConfigFileSetsMD (config->arguments, &argc, argv, "BIAS", "-bias", "-biaslist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "DARK", "-dark", "-darklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "FLAT", "-flat", "-flatlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist"); + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", + argv[N]); + psArgumentRemove (N, &argc, argv); + } + + if (argc != 2) usage (); + + // Add the input and output images (which remain on the command-line) to the arguments list + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", + argv[1]); + + return config; +} Index: /branches/eam_branches/ppImage.20240412/src/ppFocusFitFWHM.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppFocusFitFWHM.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppFocusFitFWHM.c (revision 42655) @@ -0,0 +1,30 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +bool ppFocusFitFWHM (pmConfig *config, psVector *focus, psVector *fwhm) { + + float minFocus; + + psPolynomial1D *poly = psPolynomial1DAlloc (PS_POLYNOMIAL_ORD, 2); + + if (!psVectorFitPolynomial1D (poly, NULL, 0, fwhm, NULL, focus)) { + psError (PS_ERR_UNKNOWN, false, "failed to fit focus/fwhm trend; invalid image data?"); + return false; + } + + if (poly->coeff[2] <= 0.0) { + psLogMsg ("ppFocus", 3, "poor focus fit: zero or negative curvature\n"); + psLogMsg ("ppFocus", 3, "fit coeffs: %f %f %f\n", + poly->coeff[0], poly->coeff[1], poly->coeff[2]); + } + + minFocus = -0.5 * poly->coeff[1] / poly->coeff[2]; + psLogMsg ("ppFocus", 3, "best fit focus: %f\n", minFocus); + psLogMsg ("ppFocus", 3, "fwhm @ min: %f\n", psPolynomial1DEval (poly, minFocus)); + + psFree (poly); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppFocusGetFWHM.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppFocusGetFWHM.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppFocusGetFWHM.c (revision 42655) @@ -0,0 +1,79 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +bool ppFocusGetFWHM (pmConfig *config, psVector *focus, psVector *fwhm) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + psMetadata *header; + float FOCUS, FWHM, FWHM_X, FWHM_Y, FWHMsum; + int FWHMnum; + + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.OUTPUT"); + if (!status) { + psErrorStackPrint(stderr, "Can't find input data!\n"); + exit(EXIT_FAILURE); + } + + pmFPAview *view = pmFPAviewAlloc (0); + + // - find readouts with measured PSFs + // - measure the average central FWHM for each PSF + FWHMsum = 0.0; + FWHMnum = 0; + + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { + psLogMsg ("ppImageLoop", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (!chip->process || !chip->file_exists) { continue; } + + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("ppImageLoop", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!cell->process || !cell->file_exists) { continue; } + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + if (!readout->data_exists) { continue; } + + // get average FWHM + // psphotReadout writes the FWHM values into the PSPHOT.HEADER table + // the source of this value depends on the psphot options. + // - if breakPoint is set to PEAKS, the value will not be defined + // - if breakPoint is set to MOMENTS, the PSFSTAR moments are used + // - in all other cases, the psf model is used + header = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.HEADER"); + if (header == NULL) { + psError(PS_ERR_IO, false, "Missing header in ppFocus"); + continue; + } + + FWHM_X = psMetadataLookupF32 (&status, header, "FWHM_X"); + FWHM_Y = psMetadataLookupF32 (&status, header, "FWHM_Y"); + + FWHMsum += 0.5*(FWHM_X + FWHM_Y); + FWHMnum ++; + + psLogMsg ("ppFocus", 4, "focus pt: %f,%f, fwhm sum: %f, fwhm num: %d\n", FWHM_X, FWHM_Y, FWHMsum, FWHMnum); + } + } + } + + FWHM = FWHMsum / FWHMnum; + + FOCUS = psMetadataLookupF32 (&status, input->fpa->concepts, "FPA.FOCUS"); + + fwhm->data.F32[fwhm->n] = FWHM; + focus->data.F32[focus->n] = FOCUS; + + psVectorExtend (fwhm, 10, 1); + psVectorExtend (focus, 10, 1); + + psLogMsg ("ppFocus", 4, "focus: %f, fwhm: %f (%d)\n", FOCUS, FWHM, FWHMnum); + + psFree (view); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppFocusParseCamera.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppFocusParseCamera.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppFocusParseCamera.c (revision 42655) @@ -0,0 +1,133 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +// XXX clean up error checks: return NULL, not psAbort +ppImageOptions *ppFocusParseCamera (pmConfig *config, int entry) { + + bool status = false; + + // the first input image defines the camera, and all recipes and options that follow + // select only the first file from the INPUT array + pmFPAfile *input = pmFPAfileDefineSingleFromArgs (&status, config, "PPIMAGE.INPUT", "INPUT", entry); + if (!status) { + psError(PS_ERR_IO, false, "Failed to build FPA from PPIMAGE.INPUT"); + return NULL; + } + + // parse the options from the metadata format to the ppImageOptions structure + ppImageOptions *options = ppImageOptionsParse (config); + + // the following are defined from the argument list, if given, + // otherwise they revert to the config information or detrend database if specified + // not all input or output images are used in a given recipe + if (options->doBias) { + if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.BIAS", "BIAS", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_BIAS)) { + psError (PS_ERR_IO, false, "Can't find a bias image source"); + return NULL; + } + } + if (options->doDark) { + if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.DARK", "DARK", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_DARK)) { + psError (PS_ERR_IO, false, "Can't find a dark image source"); + return NULL; + } + } + if (options->doMask) { + if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.MASK", "MASK", PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) { + psError (PS_ERR_IO, false, "Can't find a mask image source"); + return NULL; + } + } + if (options->doFlat) { + if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.FLAT", "FLAT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_FLAT)) { + psError (PS_ERR_IO, false, "Can't find a shutter image source"); + return NULL; + } + } + + // the following files are output targets + pmFPAfile *output = pmFPAfileDefineOutput (config, input->fpa, "PPIMAGE.OUTPUT"); + pmFPAfile *byChip = pmFPAfileDefineNewCamera (config, "PPIMAGE.OUTPUT.CHIP"); + pmFPAfile *byFPA1 = pmFPAfileDefineNewCamera (config, "PPIMAGE.OUTPUT.FPA1"); + pmFPAfile *byFPA2 = pmFPAfileDefineNewCamera (config, "PPIMAGE.OUTPUT.FPA2"); + + // save any of these files? + output->save = options->BaseFITS; + byChip->save = options->ChipFITS; + byFPA1->save = options->FPA1FITS; + byFPA2->save = options->FPA2FITS; + + // output is used as a carrier: input to byChip + output->freeLevel = PM_FPA_LEVEL_CHIP; + + // define the binned target files (which may just be carriers for some camera configurations) + pmFPAfile *bin1 = pmFPAfileDefineFromFPA (config, byChip->fpa, options->xBin1, options->yBin1, "PPIMAGE.BIN1"); + pmFPAfile *bin2 = pmFPAfileDefineFromFPA (config, byChip->fpa, options->xBin2, options->yBin2, "PPIMAGE.BIN2"); + + // bin1 and bin2 are used as carriers: input for byFPA1, byFPA2 + bin1->freeLevel = PM_FPA_LEVEL_FPA; + bin2->freeLevel = PM_FPA_LEVEL_FPA; + + pmFPAfile *jpg1 = pmFPAfileDefineOutput (config, byFPA1->fpa, "PPIMAGE.JPEG1"); + pmFPAfile *jpg2 = pmFPAfileDefineOutput (config, byFPA2->fpa, "PPIMAGE.JPEG2"); + + // XXX we could potentially not define these pmFPAfiles if no output is requested... + bin1->save = options->Bin1FITS; + bin2->save = options->Bin2FITS; + jpg1->save = options->Bin1JPEG; + jpg2->save = options->Bin2JPEG; + + // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg) + char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS"); + psArray *chips = psStringSplitArray (chipLine, ",", false); + if (chips->n > 0) { + pmFPASelectChip (input->fpa, -1, true); // deselect all chips + for (int i = 0; i < chips->n; i++) { + int chipNum = atoi(chips->data[i]); + if (! pmFPASelectChip(input->fpa, chipNum, false)) { + psError(PS_ERR_IO, false, "Chip number %d doesn't exist in camera.\n", chipNum); + return false; + } + } + } + psFree (chips); + + return (options); +} + +// remove from config all pmFPAfiles which could have been added (ignore missing entries) +void ppFocusDropCamera (pmConfig *config) { + + psMetadataRemoveKey (config->files, "PPIMAGE.INPUT"); + + psMetadataRemoveKey (config->files, "PPIMAGE.BIAS"); + psMetadataRemoveKey (config->files, "PPIMAGE.DARK"); + psMetadataRemoveKey (config->files, "PPIMAGE.MASK"); + psMetadataRemoveKey (config->files, "PPIMAGE.FLAT"); + + psMetadataRemoveKey (config->files, "PPIMAGE.OUTPUT"); + psMetadataRemoveKey (config->files, "PPIMAGE.OUTPUT.CHIP"); + psMetadataRemoveKey (config->files, "PPIMAGE.OUTPUT.FPA1"); + psMetadataRemoveKey (config->files, "PPIMAGE.OUTPUT.FPA2"); + + psMetadataRemoveKey (config->files, "PPIMAGE.BIN1"); + psMetadataRemoveKey (config->files, "PPIMAGE.BIN2"); + + psMetadataRemoveKey (config->files, "PPIMAGE.JPEG1"); + psMetadataRemoveKey (config->files, "PPIMAGE.JPEG2"); + + psMetadataRemoveKey (config->files, "PSPHOT.INPUT"); + psMetadataRemoveKey (config->files, "PSPHOT.OUTPUT"); + psMetadataRemoveKey (config->files, "PSPHOT.RESID"); + psMetadataRemoveKey (config->files, "PSPHOT.BACKMDL"); + psMetadataRemoveKey (config->files, "PSPHOT.BACKGND"); + psMetadataRemoveKey (config->files, "PSPHOT.BACKSUB"); + psMetadataRemoveKey (config->files, "PSPHOT.PSF.LOAD"); + psMetadataRemoveKey (config->files, "PSPHOT.PSF.SAVE"); + + return; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImage.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImage.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImage.c (revision 42655) @@ -0,0 +1,48 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +int main(int argc, char **argv) { + + psLibInit(NULL); + + psTimerStart(TIMER_TOTAL); + + // Parse the configuration and arguments + // Open the input image(s) + // Determine camera, format from header if not already defined + // Construct camera in preparation for reading + pmConfig *config = ppImageArguments(argc, argv); + if (config == NULL) { + psErrorStackPrint(stderr, "Unable to parse command-line arguments."); + ppImageCleanup(config, NULL); + exit(PS_EXIT_CONFIG_ERROR); + } + + ppImageVersionPrint(); + + // define recipe options + // define the active I/O files + ppImageOptions *options = ppImageParseCamera(config); + if (options == NULL) { + psErrorStackPrint(stderr, "Unable to parse camera."); + ppImageCleanup(config, options); + exit(PS_EXIT_CONFIG_ERROR); + } + + // Image Arithmetic Loop + if (!ppImageLoop(config, options)) { + psErrorStackPrint(stderr, "Unable to loop over input"); + ppImageCleanup(config, options); + exit(PS_EXIT_SYS_ERROR); + } + + psLogMsg("ppImage", PS_LOG_INFO, "Complete ppImage run: %f sec\n", psTimerMark(TIMER_TOTAL)); + + // Cleaning up + ppImageCleanup(config, options); + + return PS_EXIT_SUCCESS; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImage.h =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImage.h (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImage.h (revision 42655) @@ -0,0 +1,339 @@ +#ifndef PP_IMAGE_H +#define PP_IMAGE_H + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include // for strcasecmp +#include // for unlink +#include "pslib.h" +#include "psmodules.h" +#include "psphot.h" +#include "psastro.h" +#include "ppStats.h" +#include "ppImageErrorCodes.h" + +#define RECIPE_NAME "PPIMAGE" // Name of the recipe to use +#define TIMER_TOTAL "PPIMAGE.TOTAL" // Name of timer for total time +#define TIMER_DETREND "PPIMAGE.DETREND" // Name of timer for detrend time +#define TIMER_PHOT "PPIMAGE.PHOT" // Name of timer for photometry time + +// Options for ppImage processing +typedef struct { + // actions which ppImage should perform + bool doMaskBuild; // Build internal mask + bool doVarianceBuild; // Build internal variance map + bool doApplyBurntool; // apply burntool correction + bool doApplyPixelZero; // put to zero pixels underneath mask + bool doMaskBurntool; // mask potential burntool trails + bool doMaskSat; // mask saturated pixels + bool doMaskLow; // mask low pixels + bool doMask; // Mask bad pixels + bool doAuxMask; // apply auxillary mask + bool doNonLin; // Non-linearity correction + bool doNewNonLin; // Non-linearity correction + bool doOverscan; // Overscan subtraction + bool doNoiseMap; // Bias subtraction + bool doBias; // Bias subtraction + bool doDark; // Dark subtraction + bool doRemnance; // Remnance masking + bool doShutter; // Shutter correction + bool doFlat; // Flat-field normalisation + bool doPatternRow; // Row pattern correction + bool doPatternCell; // Cell pattern correction + bool doPatternContinuity; // Cell continuity correction + bool doBackgroundContinuity; // Do mosaic continuity correction + bool doPatternDeadCells; // match dead cell background patterns + bool doFringe; // Fringe subtraction + bool doPhotom; // Source identification and photometry + bool doBG; // Background subtraction + bool doAstromChip; // per-chip Astrometry + bool doAstromMosaic; // full-mosaic Astrometry + bool doStats; // call ppStats on the image + bool checkCTE; // measure pixel-based variance + bool checkNoise; // measure cell-level variance + bool squashNANs; // measure cell-level variance + bool applyParity; // Apply Cell parities + bool doMaskStats; // Calculate mask statistics + + bool doCrosstalkMeasure; // measure crosstalk signal + bool doCrosstalkCorrect; // apply crosstalk correction + bool addNoise; // Add noise to degrade MD image to 3pi + + bool hasVideo; // Determine if this OTA has a video cell + bool useVideoDark; // Should we use a video dark if we can? + bool useVideoMask; // Should we use a video mask if we can? + + // output files requested + bool BaseFITS; + bool BaseMaskFITS; + bool BaseVarianceFITS; + + bool ChipFITS; + bool ChipMaskFITS; + bool ChipVarianceFITS; + + bool FPA1FITS; + bool FPA2FITS; + bool Bin1FITS; + bool Bin1JPEG; + bool Bin2FITS; + bool Bin2JPEG; + + // make values for abstract concepts of masking + psImageMaskType maskValue; // apply this bit-mask to choose masked bits + psImageMaskType markValue; // apply this bit-mask to choose masked bits + psImageMaskType satMask; // Mask value to give saturated pixels + psImageMaskType lowMask; // Mask value to give bad pixels + psImageMaskType flatMask; // Mask value to give bad flat pixels + psImageMaskType darkMask; // Mask value to give bad dark pixels + psImageMaskType blankMask; // Mask value to give blank pixels + psImageMaskType burntoolMask; // Suspect pixels that fall where a burntool trail is expected. + // non-linear correction parameters + psDataType nonLinearType; + psMetadataItem *nonLinearData; + void *nonLinearSource; + + // options for the analysis + pmOverscanOptions *overscan; // Overscan options + int burntoolTrails; + // binning parameters + int xBin1; // x-binning, scale 1 + int yBin1; // y-binning, scale 1 + int xBin2; // x-binning, scale 2 + int yBin2; // y-binning, scale 2 + + // parameters used by the fringe analysis + float fringeRej; // Fringe rejection limit + int fringeIter; // Fringe iterations + float fringeKeep; // Fringe keep fraction + + // Pattern correction + int patternRowOrder; // Polynomial order + int patternRowIter; // Clipping iterations + float patternRowRej; // Clipping threshold + float patternRowThresh; // Ignore threshold + psStatsOptions patternRowMean; // Statistic for mean + psStatsOptions patternRowStdev; // Statistic for stdev + psStatsOptions patternCellBG; // statistic for background + psStatsOptions patternCellMean; // Statistic for mean + + int patternContinuityEdgeWidth; // Size of box to use for edge matching. + + int remnanceSize; // Size for remnance detection + float remnanceThresh; // Threshold for remnance detection + + char *normClass; // class to use for per-class normalization + + psU16 maskstat_static; + psU16 maskstat_dynamic; + psU16 maskstat_magic; + psU16 maskstat_advisory; + + psString auxVideoMask; // auxillary video mask file + +} ppImageOptions; + +// Cells to be used in the detrend +typedef struct { + pmCell *input; // The input cell, to be operated upon + pmCell *mask; // The bad pixel mask + pmCell *bias; // The bias correction + pmCell *dark; // The dark correction + pmCell *flat; // The flat-field correction +} ppImageDetrend; + +ppImageOptions *ppImageOptionsAlloc(void); + +// Determine the processing options +ppImageOptions *ppImageOptionsParse(pmConfig *config); + +// Get the configuration +pmConfig *ppImageArguments(int argc, char **argv); + +// Determine what type of camera, and initialise +ppImageOptions *ppImageParseCamera(pmConfig *config); +bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options); +bool ppImageMaskSetInMetadata(psImageMaskType *outMaskValue, // Value of MASK.VALUE, returned + psImageMaskType *outMarkValue, // Value of MARK.VALUE, returned + psMetadata *source // Source of mask bits + ); + +// apply the cell flips to the input data before analysis +bool ppImageParityFlip (pmConfig *config, const ppImageOptions *options, const pmFPAview *view, bool native); + +// Loop over the input +bool ppImageLoop(pmConfig *config, ppImageOptions *options); + +// free memory, check for leaks +void ppImageCleanup (pmConfig *config, ppImageOptions *options); + +// perform the detrend analysis on the current readout +bool ppImageDetrendReadout (pmConfig *config, ppImageOptions *options, pmFPAview *view); + +bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options); + +bool ppImageDetrendNewNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config); + +bool ppImageDetrendNonLinear(pmReadout *input, pmFPAview *linearity, pmConfig *config); +bool ppImageDetrendNonLinearLookup(pmReadout *input, psMetadataItem *dataItem); +bool ppImageDetrendNonLinearPolynomial(pmReadout *input, psMetadataItem *dataItem); + +bool ppImageDetrendCell(ppImageDetrend *detrend, ppImageOptions *options, pmConfig *config); +bool ppImageDetrendBias(pmReadout *inputReadout, pmReadout *bias, pmReadout *dark, ppImageOptions *options); +pmReadout* ppImageDetrendSelectFirst(pmCell *cell, char *name, bool doThis); + +bool ppImageDetrendFree(pmConfig *config, pmFPAview *view); +bool ppImageFringeFree(pmConfig *config, pmFPAview *view); + +bool ppImageCheckCTE(pmConfig *config, ppImageOptions *options, pmFPAview *view); + +bool ppImageCheckNoise(pmConfig *config, ppImageOptions *options, pmFPAview *view); + +bool ppImageSquashNANs(pmConfig *config, ppImageOptions *options, pmFPAview *view); + +bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout); +bool ppImageBurntoolMaskFromTable(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout); + +bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout); + +bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options); +bool ppImageDetrendPatternApplyCell (pmConfig *config, pmFPA *fpa, pmChip *chip, pmCell *cell, pmFPAview *view, ppImageOptions *options); + +// Do background continuity step +bool ppImageMosaicBackground(pmConfig *config, const ppImageOptions *options); + +// Record which detrend file was used for the detrending +bool ppImageDetrendRecord( + pmCell *cell, // Cell of interest + const pmConfig *config, // Configuration + const ppImageOptions *options, // Processing options + const pmFPAview *view // View to cell + ); + +bool ppImageRebinChip (pmConfig *config, pmFPAview *view, ppImageOptions *options, char *outName); + +bool ppImagePhotom(psMetadata *stats, pmConfig *config, pmFPAview *view); +bool ppImageAstrom(pmConfig *config, psMetadata *stats); +bool ppImageAddstar(pmConfig *config); + +// Subtract background from the chip-mosaicked image +bool ppImageSubtractBackground( + pmConfig *config, // Configuration + const pmFPAview *view, // View to chip of interest + const ppImageOptions *options // Processing options + ); + +bool ppImageMosaicChip (pmConfig *config, const ppImageOptions *options, const pmFPAview *view, + const char *outFile, const char *inFile); +bool ppImageMosaicFPA (pmConfig *config, const ppImageOptions *options, + const char *outFile, const char *inFile); + +bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options); + +void ppImageFileCheck (pmConfig *config); + +// functions used by ppFocus +pmConfig *ppFocusArguments(int argc, char **argv); +ppImageOptions *ppFocusParseCamera (pmConfig *config, int entry); +bool ppFocusGetFWHM (pmConfig *config, psVector *focus, psVector *fwhm); +bool ppFocusFitFWHM (pmConfig *config, psVector *focus, psVector *fwhm); + +void ppFocusDropCamera (pmConfig *config); + +bool ppImageDefineFile (pmConfig *config, pmFPA *input, char *filerule, char *argname, pmFPAfileType fileType, pmDetrendType detrendType); + +// write stats to output file +bool ppImageStatsOutput(pmConfig *config, // Configuration + psMetadata *stats, // Statistics output + const ppImageOptions *options // Options + ); + + +// measure the crosstalk signal +bool ppImageMeasureCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view); + +// correct the crosstalk signal +bool ppImageCorrectCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view); + +// Measure fringes +bool ppImageDetrendFringeMeasure(pmReadout *readout, // Readout to measure + pmCell *fringe, // Fringe cell (each readout is a different component) + const bool isResidual, + const ppImageOptions *options // Options + ); + +// Solve the fringe system +bool ppImageDetrendFringeSolve(pmChip *scienceChip, // Chip with science + const pmChip *refChip, // Chip with reference fringes + const bool isResidual, + const ppImageOptions *options // Options + ); + +// Generate fringe frame +bool ppImageDetrendFringeGenerate(pmCell *science, // Science cell + pmCell *fringes, // Fringe cell, one readout per fringe component + const ppImageOptions *options // Options + ); + + +bool ppImageDetrendFringeApply (pmConfig *config, // config + pmChip *chip, // science chip + const pmFPAview *inputView, // current view + const ppImageOptions *options // options + ); + +/// Return short version information +psString ppImageVersion(void); + +/// Return software source +psString ppImageSource(void); + +/// Return long version information +psString ppImageVersionLong(void); + +/// Populate the header with version information for all dependencies +bool ppImageVersionHeader(psMetadata *metadata ///< Header to populate + ); + +/// Print version information +void ppImageVersionPrint(void); + + +// calculate stats, including MD5 +bool ppImagePixelStats(pmConfig *config,// Configuration + psMetadata *stats, // Statistics output + const ppImageOptions *options, // Options + const pmFPAview *inputView // View to data + ); + +// Calculate Mask statistics +bool ppImageMaskStats(pmConfig *config, pmFPAview *view, psMetadata *stats); + +// calculate stats from headers and concepts +bool ppImageMetadataStats(pmConfig *config, // Configuration + psMetadata *stats, // Statistics output + const ppImageOptions *options // Options + ); + +void ppImageFileCheck(pmConfig *config); + +/// Dump memory summary to text file +void ppImageMemoryDump(const char *description); + + +//Functions needed to degrade MD exposures to 3pi exposures + +bool ppImageAddNoise(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmFPA *fpa) ; +double ppImageRandomGaussian (const psRandom *rnd, double mean, double sigma); +double ppImageRandomGaussianNorm (const psRandom *rnd); +void ppImageRandomGaussianFree(void); + +bool ppImageAuxiliaryMask(pmConfig *config, const pmFPAview *view, const ppImageOptions *options, psMetadata *stats); + +bool ppImageSetThreads (void); + +#endif Index: /branches/eam_branches/ppImage.20240412/src/ppImageAddNoise.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageAddNoise.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageAddNoise.c (revision 42655) @@ -0,0 +1,103 @@ +# include "ppImage.h" + +/* This function degrades MD exposures to 3Pi exposures, by adding appropriate noise + and scaling the values in the image - Daniel Farrow. + + EAM : Currently, this function is totally hard-wired to use values for GPC1 MD fields. + The sigmas should be included in a recipe, and probably calculated based on a target + exptime. + + */ + +bool ppImageAddNoise(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmFPA *fpa) +{ + + + // this step is optional. + if (!options->addNoise) { + return true; + } + + // psTimerStart("add.noise"); + + // grizy variances to add to turn MD exposure -> 3pi, calculated from the DRM + // static float add_sigmas[] = {NAN, 13.39, 23.79, 75.25, 110.19, 128.84}; + + // Target Exposure times for 3pi in grizy + static float expTimes3Pi[] = {NAN, 43.0, 40.0, 45.0, 30.0, 30.0}; + float expTime = psMetadataLookupF32(NULL, fpa->concepts, "FPA.EXPOSURE"); // Exposure time for image + + // Something to choose the band, g,r,i,z,y = 0,1,2,3,4 respectively + char *filter = psMetadataLookupStr (NULL, fpa->concepts, "FPA.FILTERID"); + + int band = 0; + if (!strcmp(filter, "g")) { + band = 1; + } + if (!strcmp(filter, "r")) { + band = 2; + } + if (!strcmp(filter, "i")) { + band = 3; + } + if (!strcmp(filter, "z")) { + band = 4; + } + if (!strcmp(filter, "y")) { + band = 5; + } + if (!band) { + psError(PS_ERR_UNKNOWN, true, "ppImageAddNoise doesn't recognise the filter %s, aborting", filter); + return false; + } + + pmReadout *inReadout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + // find the currently selected readout + psImage *image = inReadout->image; + psImage *variance = inReadout->variance; + + // Warning, just in case this is left on accidently! + psWarning("addNoise is set; adding noise to the images."); + + // Add in appropriate variance, and scale the image + float rho = expTime/expTimes3Pi[band]; + float rho2 = PS_SQR(rho); + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN); + stats->nSubsample = 10000; + + psImageBackground(stats, NULL, inReadout->image, inReadout->mask, 0xffff, rng); + double MSKY_MN = stats->sampleMedian; + + // set GAIN and RDNOISE to nominal values: + double GAIN = 1.0; // electrons / DN + double RDNOISE = 6.0; // electrons (== DN) + + double add_sigma = sqrt((MSKY_MN/GAIN)*(rho - 1.0) + PS_SQR(RDNOISE)*(rho2 - 1.0)); + + fprintf (stderr, "mean sky: %f, scaling by %f, adding %f before re-scaling\n", MSKY_MN, rho, add_sigma); + + for (int iy = 0; iy < image->numRows; iy++){ + for (int ix = 0; ix < image->numCols; ix++){ + image->data.F32[iy][ix] += ppImageRandomGaussian(rng, 0.0, add_sigma); + image->data.F32[iy][ix] /= rho; + variance->data.F32[iy][ix] += PS_SQR(add_sigma); + variance->data.F32[iy][ix] /= rho2; + } + } + + // Update the metadata about exposure time + psMetadataAddF32(inReadout->parent->concepts, PS_LIST_TAIL, "CELL.EXPOSURE", PS_META_REPLACE, "the modified exposure time", expTimes3Pi[band]); + psMetadataAddF32(inReadout->parent->concepts, PS_LIST_TAIL, "FPA.EXPOSURE", PS_META_REPLACE, "the modified exposure time", expTimes3Pi[band]); + psMetadataAddF32(fpa->concepts, PS_LIST_TAIL, "EXPTIME", PS_META_REPLACE, "the modified exposure time", expTimes3Pi[band]); + ppImageRandomGaussianFree(); + psFree(rng); + psFree(stats); + + // psLogMsg ("ppImage", 5, "add noise: %f sec\n", psTimerMark ("add.noise")); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageAddstar.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageAddstar.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageAddstar.c (revision 42655) @@ -0,0 +1,120 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +bool setFilename (pmFPAfile *file, pmFPAview *view); +bool addstarFile (char *addstarCommand, char *filename); + +bool ppImageAddstar (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + // select recipe options supplied on command line + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, RECIPE_NAME); + + // find a pmFPAfile PSASTRO.OUTPUT + pmFPAfile *file = psMetadataLookupPtr (&status, config->files, "PSASTRO.OUTPUT"); + if (!status) { + psError (PS_ERR_IO, true, "no PSASTRO.OUTPUT file defined"); + return false; + } + + // find the addstar command: %s is replaced with name of output file + char *addstarCommand = psMetadataLookupStr (&status, recipe, "ADDSTAR.COMMAND"); + + pmFPAview *view = pmFPAviewAlloc (0); + if (file->fileLevel == PM_FPA_LEVEL_CHIP) { + // call addstar on this file + setFilename (file, view); + addstarFile (addstarCommand, file->filename); + return true; + } + + while ((chip = pmFPAviewNextChip (view, file->fpa, 1)) != NULL) { + psLogMsg ("ppImageAddstar", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (!chip->process || !chip->file_exists) { continue; } + + if (file->fileLevel == PM_FPA_LEVEL_CHIP) { + // call addstar on this file + setFilename (file, view); + addstarFile (addstarCommand, file->filename); + continue; + } + + while ((cell = pmFPAviewNextCell (view, file->fpa, 1)) != NULL) { + psLogMsg ("ppImageAddstar", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! cell->process || ! cell->file_exists) { continue; } + + if (file->fileLevel == PM_FPA_LEVEL_CELL) { + // call addstar on this file + setFilename (file, view); + addstarFile (addstarCommand, file->filename); + continue; + } + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, file->fpa, 1)) != NULL) { + if (! readout->data_exists) { continue; } + + if (file->fileLevel == PM_FPA_LEVEL_READOUT) { + // call addstar on this file + setFilename (file, view); + addstarFile (addstarCommand, file->filename); + continue; + } else { + psError (PS_ERR_IO, true, "inconsistent fileLevel for %s: %d\n", file->name, file->fileLevel); + return false; + } + } + } + } + return true; +} + +bool setFilename (pmFPAfile *file, pmFPAview *view) { + + // determine the file name + // free a name allocated earlier + psFree (file->filename); + file->filename = pmFPAfileNameFromRule (file->filerule, file, view); + if (file->filename == NULL) { + psError(PS_ERR_IO, true, "Filename is NULL"); + return false; + } + + // indirect filenames are not allowed for output + if (!strcasecmp (file->filename, "@FILES")) { + psError(PS_ERR_IO, true, "indirect filenames are not allowed for output : %s\n", file->name); + return false; + } + if (!strcasecmp (file->filename, "@DETDB")) { + psError(PS_ERR_IO, true, "detrend db filenames are not allowed for output : %s\n", file->name); + return false; + } + return true; +} + +bool addstarFile (char *addstarCommand, char *filename) { + + bool status; + char *addstarLine = psStringCopy(addstarCommand); + psStringSubstitute(&addstarCommand, filename, "%s"); + + // catch addstar stderr/stdout and do what? + psIOBuffer *buffer = psIOBufferAlloc (512); + psPipe *pipe = psPipeOpen (addstarLine); + status = psIOBufferReadEmpty (buffer, 100, pipe->fd_stdout); + if (!status) { + psError (PS_ERR_IO, false, "detselect is not responding"); + } + psFree (addstarLine); + psFree (buffer); + psPipeClose (pipe); + psFree (pipe); + return status; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageArguments.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageArguments.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageArguments.c (revision 42655) @@ -0,0 +1,164 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +static void usage (void) { + fprintf(stderr, "USAGE: ppImage [-file INPUT.fits] [-list INPUT.txt] OUTPUT\n\n"); + fprintf(stderr, "Optional arguments:\n"); + fprintf(stderr, "\t-stats STATS.mdc: Output statistics into STATS.mdc\n"); + fprintf(stderr, "\t-isfringe: The input image contains fringe data.\n"); + fprintf(stderr, "\t-isdark: The input image contains dark data.\n"); + fprintf(stderr, "\t-usemask MASKVAL: Use this mask value (override recipe).\n"); + fprintf(stderr, "\t-chip CHIPNUM: Only process this chip number.\n"); + fprintf(stderr, "\t-norm VALUE: Divide through by this value when done.\n"); + fprintf(stderr, "\t-normlist file.mdc: normalizations by class_id.\n"); + fprintf(stderr, "\n"); + fprintf(stderr, "Input options (single file / file list):\n"); + fprintf(stderr, "\t-noisemap/-noisemaplist: Noise Map image.\n"); + fprintf(stderr, "\t-bias/-biaslist: Bias image.\n"); + fprintf(stderr, "\t-dark/-darklist: Dark image.\n"); + fprintf(stderr, "\t-shutter/-shutterlist: Shutter image.\n"); + fprintf(stderr, "\t-flat/-flatlist: Flat image.\n"); + fprintf(stderr, "\t-mask/-masklist: Mask image.\n"); + fprintf(stderr, "\t-fringe/-fringelist: Fringe image and data.\n"); + fprintf(stderr, "\t-linearity/-linearlist: linearity correction file.\n"); + fprintf(stderr, "\t-newnonlin/-newnonlinlist: non-linearity correction file (v 2023.01).\n"); + fprintf(stderr, "\n"); + exit (2); +} + +pmConfig *ppImageArguments(int argc, char **argv) +{ + int argnum; // Argument number of interest + + if (argc == 1) { + usage(); + } + + if (psArgumentGet (argc, argv, "-version")) { + psString version; + version = ppImageVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = ppStatsVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = psphotVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = psastroVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = psModulesVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = psLibVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + exit (0); + } + + // load the site-wide configuration information + pmConfig *config = pmConfigRead(&argc, argv, RECIPE_NAME); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + // generic arguments (version -- ignored in this case, dumpconfig) + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // thread arguments + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + // save the following additional recipe values based on command-line options + // these options override the PPIMAGE recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, RECIPE_NAME); + + // save these recipe options until we have loaded the options + // psMetadata *options = psMetadataAlloc (); + // psMetadataAddPtr (config->arguments, PS_LIST_TAIL, "PPIMAGE.OPTIONS", PS_DATA_METADATA, "", options); + + if ((argnum = psArgumentGet(argc, argv, "-stats"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "STATS", PS_META_REPLACE, + "Filename for summary statistics", argv[argnum]); + psArgumentRemove(argnum, &argc, argv); + } + + if ((argnum = psArgumentGet(argc, argv, "-isfringe"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "INPUT_IS_FRINGE", PS_META_REPLACE, + "Input is fringe image", true); + } + if ((argnum = psArgumentGet(argc, argv, "-isdark"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "INPUT_IS_DARK", PS_META_REPLACE, + "Input is dark image", true); + } + + if ((argnum = psArgumentGet(argc, argv, "-visual"))) { + psArgumentRemove(argnum, &argc, argv); + pmVisualSetVisual(true); + } + + // the following options override the PPIMAGE recipe options + + // recipe option: -usemask : override MASK setting in recipe + if ((argnum = psArgumentGet(argc, argv, "-usemask"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddBool(options, PS_LIST_TAIL, "MASK", PS_META_REPLACE, "", true); + psArgumentRemove(argnum, &argc, argv); + } + + // XXX add other PPIMAGE recipe options here + + // the input file is a required argument; if not found, we will exit + pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + pmConfigFileSetsMD (config->arguments, &argc, argv, "NOISEMAP", "-noisemap", "-noisemaplist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "BIAS", "-bias", "-biaslist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "DARK", "-dark", "-darklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "SHUTTER", "-shutter", "-shutterlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "FLAT", "-flat", "-flatlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "FRINGE", "-fringe", "-fringelist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "LINEARITY", "-linearity", "-linearlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "NEWNONLIN", "-newnonlin", "-newnonlinlist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.DEAD.CELLS", "-pattern-dead-cells", "-not-defined"); + + if ((argnum = psArgumentGet(argc, argv, "-burntool"))) { + psArgumentRemove(argnum, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "BURNTOOL.TABLE", PS_META_REPLACE, "", argv[argnum]); + psArgumentRemove(argnum, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((argnum = psArgumentGet (argc, argv, "-chip"))) { + psArgumentRemove (argnum, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[argnum]); + psArgumentRemove (argnum, &argc, argv); + } + + // Optional normalization factor + if ((argnum = psArgumentGet(argc, argv, "-norm"))) { + psArgumentRemove(argnum, &argc, argv); + float norm = atof(argv[argnum]); + psMetadataAddF32(config->arguments, PS_LIST_TAIL, "NORMALIZATION", 0, + "Normalisation to apply", norm); + psArgumentRemove(argnum, &argc, argv); + } + + // Optional per-class normalization table + if ((argnum = psArgumentGet(argc, argv, "-normlist"))) { + psArgumentRemove(argnum, &argc, argv); + + unsigned int nFail = 0; + psMetadata *normlist = psMetadataConfigRead (NULL, &nFail, argv[argnum], false); + // XXX allow this file to be in nebulous? + + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "NORMALIZATION.TABLE", 0, "Normalization to apply", normlist); + psFree (normlist); + psArgumentRemove(argnum, &argc, argv); + } + + if (argc != 2) usage (); + + // Add the input and output images (which remain on the command-line) to the arguments list + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]); + + return config; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageAstrom.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageAstrom.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageAstrom.c (revision 42655) @@ -0,0 +1,66 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +// this function is mostly equivalent to the top-level of psastro, with some +// modifications since the data has already been loaded. +bool ppImageAstrom (pmConfig *config, psMetadata *stats) { + + bool status; + + // select recipe options supplied on command line + // XXX move these options to the "PSASTRO" recipe? + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSASTRO_RECIPE); + + // find or define a pmFPAfile PSPHOT.INPUT + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSASTRO.INPUT"); + PS_ASSERT (input, false); + + ppImageMemoryDump("astrom"); + + // convert the output sources created by psphot into astrometry objects + if (!psastroConvertFPA (config, input->fpa, recipe)) { + psError (PSASTRO_ERR_UNKNOWN, false, "error reading input data\n"); + return false; + } + + if (!psastroAnalysis(config, stats)) { + psError (PSASTRO_ERR_UNKNOWN, false, "failure in psastro analysis\n"); + return false; + } + + // deactivate the psastro files, reactive when needed + pmFPAfileActivate (config->files, false, NULL); + pmFPAfileActivate (config->files, true, "PSASTRO.OUTPUT"); + + // loop over all chips and perform IO needed + pmChip *chip = NULL; + pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest + while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { + psLogMsg ("ppImageLoop", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + + ppImageMemoryDump("astrom"); + + // Output and Close at Chip level + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psFree(view); + return false; + } + } + + // Output and Close FPA + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psFree(view); + return false; + } + + // deactivate the PSASTRO files, re-active all else + // XXX do we need a way to activate / deactivate other groups? + pmFPAfileActivate (config->files, true, NULL); + pmFPAfileActivate (config->files, false, "PSASTRO.OUTPUT"); + + psFree(view); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageAuxiliaryMask.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageAuxiliaryMask.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageAuxiliaryMask.c (revision 42655) @@ -0,0 +1,215 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +psImage *readAuxiliaryMask(pmConfig *config, psString fileName) +{ + psString realName = pmConfigConvertFilename(fileName, config, false, false); + if (!realName) { + psError (psErrorCodeLast(), false, "unable to resolve %s", fileName); + return NULL; + } + + psFits *fits = psFitsOpen(realName, "r"); + if (!fits) { + psError (psErrorCodeLast(), false, "psFitsOpen failed for %s", realName); + psFree(realName); + return NULL; + } + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psFree(fits); + psError (psErrorCodeLast(), false, "psFitsReadHeader failed for %s", realName); + psFree(realName); + return NULL; + } + psRegion region = {0, 0, 0, 0}; + psImage *image = psFitsReadImage(fits, region, 0); + psFree(fits); + psFree(header); + if (!image) { + psError (psErrorCodeLast(), false, "psFitsReadImage failed for %s", realName); + psFree(realName); + return NULL; + } + if (image->type.type != PS_TYPE_IMAGE_MASK) { + psWarning("auxiliary mask image %s has unexpected type %d\n", realName, image->type.type); + return false; + } + psFree(realName); + + return image; +} + +bool recordFileInHeader(pmChip *chip, psString tag, psString desc, psString filename) +{ + pmHDU *hdu = pmHDUGetHighest(chip->parent, chip, NULL); + + // strip off the directories and nebulous bits + char *base = filename; + for (char *new = base; (new = strpbrk(base, "/:")); base = new + 1); + + psMetadataAddStr(hdu->header, PS_LIST_TAIL, tag, PS_META_DUPLICATE_OK, desc, base); + return true; +} + +// In this function, we augment the mask with the more conservative auxiliary mask +bool ppImageAuxiliaryMask(pmConfig *config, const pmFPAview *view, const ppImageOptions *options, psMetadata *stats) +{ + psAssert(config, "Need configuration"); + psAssert(view, "Need view to chip"); + psAssert(options, "Need options"); + + if (!options->doAuxMask) { + psLogMsg ("ppImage", PS_LOG_DETAIL, "Auxiliary mask not enabled."); + return true; + } + + // psTimerStart("aux.mask"); + + bool status; + // Our target chip + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.CHIP"); // File to correct + if (!status) { + psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined"); + return false; + } + + pmFPAfile *auxmask = psMetadataLookupPtr(&status, config->files, "PPIMAGE.AUXMASK"); + if (!status) { + psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined"); + return false; + } + + // Go find the readout + // code to find the readouts was adapted from ppImageSubtractBackground + // XXX: shouldn't most of these psWarnings be psAsserts? + + // Since we are working on a chip-mosaicked image, there should only be a single cell and readout + pmChip *chip = pmFPAviewThisChip(view, input->fpa); // Chip of interest + if (!chip) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find chip"); + return false; + } + if (chip->cells->n == 0) { + psWarning("Chip has no cells"); + return true; + } + if (chip->cells->n > 1) { + psWarning("Chip has %ld cells; only the first will be processed", chip->cells->n); + } + pmCell *cell = chip->cells->data[0]; // Cell of interest + if (!cell || !cell->process || !cell->file_exists) { + // Nothing to process + return true; + } + if (cell->readouts->n == 0) { + psWarning("Cell has no readouts"); + return true; + } + if (cell->readouts->n > 1) { + psWarning("Cell has %ld readouts; only the first will be processed", cell->readouts->n); + } + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + if (!ro || !ro->data_exists) { + // Nothing to process + return true; + } + + psImage *mask = ro->mask; + + if (mask->type.type != PS_TYPE_IMAGE_MASK) { + psWarning("mask image has unexpected type %d\n", mask->type.type); + return false; + } + + pmChip *auxMaskChip = pmFPAviewThisChip(view, auxmask->fpa); + if (!auxMaskChip) { + psWarning("failed to find pmChip for auxiliary mask"); + return false; + } + pmCell *auxMaskCell = auxMaskChip->cells->data[0]; + if (!auxMaskCell) { + psWarning("failed to find pmCell for auxiliary mask"); + return false; + } + pmReadout *auxMaskReadout = auxMaskCell->readouts->data[0]; + if (!auxMaskReadout) { + psWarning("failed to find pmReadout for auxiliary mask"); + return false; + } + psImage *auxMaskImage = auxMaskReadout->mask; + if (!auxMaskImage) { + psWarning("failed to find psImage for auxiliary mask"); + return false; + } + if (auxMaskImage->type.type != PS_TYPE_IMAGE_MASK) { + psWarning("auxiliary mask image has unexpected type %d\n", auxMaskImage->type.type); + return false; + } + + // if the cell has video and the recipe value is set or in the video mask + if (options->hasVideo && options->auxVideoMask && strcmp(options->auxVideoMask, "NULL")) { + psImage *videoMask = readAuxiliaryMask(config, options->auxVideoMask); + if (!videoMask) { + psError(PS_ERR_UNKNOWN, false, "failed to read auxiliary video mask file"); + return false; + } + psLogMsg ("ppImage", PS_LOG_INFO, "Auxiliary video mask file: %s", options->auxVideoMask); + pmConfigRunFilenameAddRead(config, "PPIMAGE.AUXVIDEOMASK", options->auxVideoMask); + recordFileInHeader(chip, "DETREND.AUXVIDEOMASK", "auxiliary video mask", options->auxVideoMask); + + // compute auxMask |= videoMask + if (!psBinaryOp(auxMaskImage, auxMaskImage, "|", videoMask)) { + psError(PS_ERR_UNKNOWN, false, "combination of auxiliary mask and auxiliary video mask failed"); + return false; + } + psFree(videoMask); + } + + if ((mask->numRows != auxMaskImage->numRows) || (mask->numCols != auxMaskImage->numCols) || + (mask->row0 != auxMaskImage->row0) || (mask->col0 != auxMaskImage->col0)) { + psError(PS_ERR_IO, false, "structure of auxiliary mask does not match this chip"); + return false; + } + + psImageMaskType maskDetector = pmConfigMaskGet("DETECTOR", config); + + psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, "PPIMAGE"); + psAssert(recipe, "Need recipe!"); + psImageMaskType staticMaskVal = psMetadataLookupImageMask(&status, recipe, "MASKSTAT.STATIC"); + psAssert(staticMaskVal, "Need staticMaskVal!"); + + // XXX: now that we are using an auxilary mask that matches the format of ps masks, we could just + // do psBinaryOp(mask, mask, "|", auxMaskImage) but run an explicit loop so that we can measure + // and log the number of additional pixels masked + int numCols = mask->numCols, numRows = mask->numRows; // Size of image + unsigned long numMasked = 0; + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (auxMaskImage->data.U16[y][x] != 0) { + if ((mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & staticMaskVal) == 0) { + // pixel was not previously included in the static mask so mask it + ++numMasked; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskDetector; + } + // I don't need to do this. + // image->data.F32[y][x] = 0.0; + // The background subtraction code runs after us and will do it there + } + } + } + + float maskedFrac = (float) numMasked / (numRows * numCols); + psLogMsg ("ppImage", PS_LOG_INFO, "Auxiliary masks masked %ld aditional pixels. Masked fraction: %f", numMasked, maskedFrac); + if (stats) { + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", PS_META_REPLACE, + "Fraction of pixels masked by auxiliary masks", maskedFrac); + } + + // psLogMsg ("ppImage", 5, "auxiliary mask: %f sec\n", psTimerMark ("aux.mask")); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageBurntoolApply.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageBurntoolApply.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageBurntoolApply.c (revision 42655) @@ -0,0 +1,669 @@ +/* + * ppImageBurntoolApply.c + * Includes code ported from gpcsw/gpcsrc to read burntool table files and apply the fits + */ + +#include "ppImage.h" + +// Now include the files copied from gpcsw/gpcsrc + +// these get redefined +#undef MAX +#undef MIN + +#include "burntool.h" +#define EXTERN /* define EXTERN to declare variables in params.h */ +#include "burnparams.h" + + +static void * getBurntoolDataForCell(pmConfig *config, ppImageOptions *options, int burntool_cell); + +bool ppImageBurntoolApply(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *readout) +{ + // convert from pmFPA's cell number to burntool's notion + int burntool_cell = (view->cell % 8) * 8 + (view->cell - (view->cell % 8)) / 8; + + psString cellName = psMetadataLookupStr(NULL, readout->parent->concepts, "CELL.NAME"); + (void) cellName; + + CELL *burntoolData = getBurntoolDataForCell(config, options, burntool_cell); + if (!burntoolData) { + psError(PS_ERR_UNKNOWN, false, "Unable to find burntool data for cell"); + return false; + } + + psImage *image = readout->image; + +#ifdef DUMP_CELL_IMAGES + psString dumpPath = NULL; + psStringAppend(&dumpPath, "cellimages/%s.before.fits", cellName); + psphotSaveImage(NULL, image, dumpPath); + psFree(dumpPath); +#endif + + // The following code is adapted from burntool's functions burn_apply() which is in burn_fix.c and + // sub_fit() which was located in trailfit.c + + int ny = image->numRows; + + psF32 satVal = psMetadataLookupF32(NULL, readout->parent->concepts, "CELL.SATURATION"); + + /* The new burntool tables and method coming into play in 2022 has a slightly different format (an extra column). Need to differentiate */ + psS16 BURNTOOL_STATE_GOOD = psMetadataLookupS16(NULL, config->camera, "BURNTOOL.STATE.GOOD"); + + for (int p = 0; p < burntoolData->npersist; p++) { + OBJBOX *box = &(burntoolData->persist[p]); + + if(box->fiterr) continue; + if(box->func != BURN_PWR && box->func != BURN_EXP) { + continue; + } + + int i, j, k; + int y0, y1, ys, dy; + + y0 = box->y0; + if (BURNTOOL_STATE_GOOD >= 15) { + y1 = box->eyfit; + } else { + y1 = (box->up) ? ny-1 : 0; + } + dy = (box->up) ? +1 : -1; + if (BURNTOOL_STATE_GOOD >= 15) { + ys = (box->up) ? box->sy : box->y0; + } else { + ys = (box->up) ? box->y1m : box->y1p; + } + + /* Calculate all needed y values */ + double ybuf[MAXSIZE]; + for(j=ys; dy*j<=dy*y1; j+=dy) { + if(box->func == BURN_PWR) { + if(dy*(j-y0) <= 0) { + ybuf[j] = exp(box->slope*log(Y_SCALE*1.0)); + } else { + ybuf[j] = exp(box->slope*log(Y_SCALE*dy*(j-y0))); + } + } else if(box->func == BURN_EXP) { + // ybuf[j] = exp(box->slope*Y_SCALE*MAX(0,dy*(j-y0))); + ybuf[j] = exp(box->slope*Y_SCALE*dy*(j-y0)); + } + } + + + /* Subtract the fits */ + // printf("%3d %3d %3d %1d %1d %8.5f\n", + // box->cx, box->cy, box->y0, box->func, box->up, box->slope); + + for(k=0; knfit; k++) { + i = box->xfit[k]; + for(j=ys; dy*j<=dy*y1; j+=dy) { + psF32 inputPixel = image->data.F32[j][i]; + if (false && inputPixel >= satVal) { + // don't modify saturated pixels + continue; + } + +#ifndef MATCH_BURNTOOL + psF32 delta = box->zero[k] * ybuf[j]; + image->data.F32[j][i] -= delta; +#else + // do math in 16 bit signed int space in order to match how the burntool program + // behaves with regard to underflows. burntool allows underflows so that the burntool + // application is reversible. + // Verified that the image matches the burntool implementation with this code + // which shows that we are reading and processing the table correctly. + int delta = box->zero[k] * ybuf[j] + 0.5; + psS16 pixel; + // adjust for BZERO + if (inputPixel >= 65535) { + pixel = 32767; + } else if (inputPixel <= 0) { + pixel = -32768; + } else { + pixel = inputPixel - 32768; + } + pixel -= delta; + // replace the pixel with correction applied + image->data.F32[j][i] = (psF32) pixel + 32768; +#endif + } + } + } +#ifdef DUMP_CELL_IMAGES + dumpPath = NULL; + psStringAppend(&dumpPath, "cellimages/%s.after.fits", cellName); + psphotSaveImage(NULL, image, dumpPath); + psFree(dumpPath); +#endif + + return true; +} + + +bool burntoolFileRead = false; +static CELL OTA[MAXCELL]; // cell structure for entire OTA +static void * getBurntoolDataForCell(pmConfig *config, ppImageOptions *options, int burntool_cell) +{ + if (!burntoolFileRead) { + psString burntoolTablePath = psMetadataLookupStr(NULL, config->arguments, "BURNTOOL.TABLE"); + if (!burntoolTablePath) { + psError(PS_ERR_UNKNOWN, false, "failed to lookup BURNTOOL.TABLE in arguments"); + return NULL; + } + psString burntoolFileName = pmConfigConvertFilename(burntoolTablePath, config, false, false); + if (!burntoolFileName) { + psError(PS_ERR_UNKNOWN, false, "failed to convert filename for %s", burntoolTablePath); + return NULL; + } + + /* The new burntool tables and method coming into play in 2022 has a slightly different format (an extra column). Need to differentiate */ + psS16 BURNTOOL_STATE_GOOD = psMetadataLookupS16(NULL, config->camera, "BURNTOOL.STATE.GOOD"); + int oldfile = 1; + if (BURNTOOL_STATE_GOOD >= 15) oldfile = 0; + + if (persist_read(OTA, burntoolFileName, 1,oldfile)) { + psError(PS_ERR_UNKNOWN, "true", "failed to read burntool file"); + return NULL; + } + psFree(burntoolFileName); + burntoolFileRead = true; + } + + return (void *) &OTA[burntool_cell]; +} +#define PPIMAGE_BURNTOOL_DEBUG 0 + +// adapted from ppImageBurntoolMask +bool ppImageBurntoolMaskFromTable(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *mask) +{ + bool status = true; + int burntool_cell; + + /* Redirects and Memory juggling. */ + view->readout = 0; + psImage *image = mask->mask; + + /* Set the maskValue from the recipes. */ + psImageMaskType maskValue = options->burntoolMask; + + /* The new burntool tables and method coming into play in 2022 has a slightly different format (an extra column). Need to differentiate */ + psS16 BURNTOOL_STATE_GOOD = psMetadataLookupS16(NULL, config->camera, "BURNTOOL.STATE.GOOD"); + + burntool_cell = (view->cell % 8) * 8 + (view->cell - (view->cell % 8)) / 8; + CELL *burntoolData = getBurntoolDataForCell(config, options, burntool_cell); + if (!burntoolData) { + psError(PS_ERR_UNKNOWN, false, "Unable to find burntool data for cell"); + return false; + } + +#if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Status: %d %d\n",burntoolData->npersist,maskValue); +#endif + + psLogMsg("ppImageBurntoolMask", 4, "Cell mapping: %d %d %d\n",view->cell,burntool_cell,-1); + for (int row = 0; row < burntoolData->npersist; row++) { + OBJBOX *box = &(burntoolData->persist[row]); + if (((options->burntoolTrails & 0x01)&& (box->func == 4))|| + (((options->burntoolTrails & 0x02)&& (box->up == 1))|| + ((options->burntoolTrails & 0x04)&& (box->up == 0)))) { + /* If the fit fails, burntool reports zero here. This + signifies that it expected to see a trail (else why + fit) but did not find it when it attempted to + correct. */ +#if PPIMAGE_BURNTOOL_DEBUG + psLogMsg ("ppImageBurntoolMask", 4, "Masking! %d (%d %d %d) %d %d", + burntool_cell, + ((options->burntoolTrails & 0x0001)&& (box->nfit == 0)), + ((options->burntoolTrails & 0x02)&& (box->up == 1)), + ((options->burntoolTrails & 0x04)&& (box->up == 0)), + options->burntoolTrails, + maskValue + ); +#endif + + + /* do separate masking strategy for old (14 and lower) and new burntool tables*/ + if (BURNTOOL_STATE_GOOD >= 15) { + printf("DET: New burntool style %d\n",BURNTOOL_STATE_GOOD); + for (int i = box->sxfit; i<= box->exfit; i++) { + if (box->up == 0) { + for (int j = box->eyfit; j <= box->y0; j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } else { + for (int j = box->sy; j < box->eyfit ; j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + } + + } else { + printf("DET: Old burntool style %d\n",BURNTOOL_STATE_GOOD); + for (int i = box->sxfit; i<= box->exfit; i++) { + if (box->up == 0) { + for (int j = 0; j <= box->y1p; j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } else { + for (int j = box->y1m; j < image->numRows ; j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + } + } + + + } + } + + return(status); +} + +/* From persistio.c - read and write persistence info */ +/****************************************************************/ +/* persist_read(): Read all the persistence trails from a file */ +STATIC int persist_read(CELL *cell, const char *infile, int apply, int oldfile) +{ + int i, k, nbox=0; + char line[1024]; + FILE *fp; + +/* Initialize the counts */ + for(k=0; k 0) { + boxbuf[nbox].zero = (double *)calloc(boxbuf[nbox].nfit, sizeof(double)); + boxbuf[nbox].xfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int)); + boxbuf[nbox].yfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int)); + if(boxbuf[nbox].zero == NULL || + boxbuf[nbox].xfit == NULL || + boxbuf[nbox].yfit == NULL) { + fprintf(stderr, "\rerror: failed to alloc boxbuf\n"); + exit(-673); + } + + for(i=0; i= MAXCELL) { + fprintf(stderr, "\rerror: illegal cell %d at %d from '%s'\n", + k, nbox, line); + fclose(fp); + return(-1); + } + cell[k].npersist += 1; + + if(++nbox >= MAXBURN) { + fprintf(stderr, "\rerror: persist_read overflowed MAXBURN\n"); + fclose(fp); + return(-1); + } + } + +/* Allocate some space for them */ + for(k=0; k 0) { + if( (cell[k].persist = (OBJBOX *)calloc(i, sizeof(OBJBOX))) == NULL) { + fprintf(stderr, "\rerror: failed to alloc cell persist buffer\n"); + exit(-674); + } + cell[k].npersist = 0; + } + } +/* Copy the results to the cells */ + for(i=0; inpersist; + box = cell->persist; + +/* Assess the clustering of all the streaks */ + for(i=0; i= MAXSIZE-1) continue; + for(i=box[k].sxfit; i<=box[k].exfit; i++) xusage[i] += 1; + } + +/* Identify clusters */ + for(xs=0; xs xe || box[k].exfit < xs) continue; + boxid[n] = k; + yctr[n] = (box[k].y0m+box[k].y1m+box[k].y0p+box[k].y0p+2) / 4; + n++; + } + +/* Case 1: different y start => sever */ + for(kp=0; kp 0) zk = box[k].zero[box[k].nfit/2]; + for(jp=kp+1; jp 0) zj = box[j].zero[box[j].nfit/2]; + if(ABS(yctr[jp]-yctr[kp]) > DIFFERENT_STREAK) { +/* Trim back the feebler streak */ + if(zk > zj) { + if(box[j].sxfit >= box[k].sxfit) + box[j].sxfit = MAX(box[j].sxfit, box[k].exfit+1); + if(box[j].exfit <= box[k].exfit) + box[j].exfit = MIN(box[j].exfit, box[k].sxfit-1); + } else { + if(box[k].sxfit >= box[j].sxfit) + box[k].sxfit = MAX(box[k].sxfit, box[j].exfit+1); + if(box[k].exfit <= box[j].exfit) + box[k].exfit = MIN(box[k].exfit, box[j].sxfit-1); + } + } + } + } + +/* Case 2: pretty much the same y start => union */ + for(kp=0; kp exfit) */ + for(k=0; k box[k].exfit) { + for(j=k; jnpersist = nbox; + + + return(0); +} + + +/****************************************************************/ +/* persist_write(): Write all the persistence data for the next image */ +STATIC int persist_write(CELL *cell, const char *outfile, int oldfile) +{ + int i, k, j, err=0; + FILE *fp; + + if(outfile == NULL) return(0); + if( (fp=fopen(outfile, "w")) == NULL) { + fprintf(stderr, "\rerror: cannot open '%s' for writing\n", outfile); + return(-1); + } + +/* Dump out all the burns for the next image... */ +/* FIXME: what to do about generic cell info? */ + fprintf(fp, "# Cell: %d sky= %d rms= %d bias= %d\n", + cell[0].cell, cell[0].sky, cell[0].rms, cell[0].bias); + + /*write new burntool tables (including the eyfit column)*/ + if(!oldfile) { + fprintf(fp, "#Cell time cx cy max y0 sx sy ex ey y0m y0p y1m y1p x0m x0p x1m x1p F up slope nfit sxf exf fiterr eyf\n"); + } else { + fprintf(fp, "#Cell time cx cy max y0 sx sy ex ey y0m y0p y1m y1p x0m x0p x1m x1p F up slope nfit sxf exf fiterr\n"); + } + + for(j=0; j EXPIRE_TRAIL_TIME) + continue; + + if(PERSIST_RETAIN) { +/* Keep fits which have a dubious slope */ + if(cell[j].persist[k].fiterr != FIT_SLOPE_ERROR) { + if(cell[j].persist[k].fiterr) continue; + if(cell[j].persist[k].nfit <= 0) continue; + } + } else { + if(cell[j].persist[k].fiterr) continue; + if(cell[j].persist[k].nfit <= 0) continue; + } + + /*write new burntool tables (including the eyfit column)*/ + if(!oldfile) { + fprintf(fp, "%3d %7d %3d %3d %5d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %1d %1d %9.6f %3d %3d %3d %d %3d\n", + j, cell[j].persist[k].time, + cell[j].persist[k].cx, cell[j].persist[k].cy, + cell[j].persist[k].max, cell[j].persist[k].y0, + cell[j].persist[k].sx, cell[j].persist[k].sy, + cell[j].persist[k].ex, cell[j].persist[k].ey, + cell[j].persist[k].y0m, cell[j].persist[k].y0p, + cell[j].persist[k].y1m, cell[j].persist[k].y1p, + cell[j].persist[k].x0m, cell[j].persist[k].x0p, + cell[j].persist[k].x1m, cell[j].persist[k].x1p, + cell[j].persist[k].func, cell[j].persist[k].up, + cell[j].persist[k].slope, cell[j].persist[k].nfit, + cell[j].persist[k].sxfit, cell[j].persist[k].exfit, + cell[j].persist[k].fiterr, cell[j].persist[k].eyfit); + } else { + fprintf(fp, "%3d %7d %3d %3d %5d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %3d %1d %1d %9.6f %3d %3d %3d %d\n", + j, cell[j].persist[k].time, + cell[j].persist[k].cx, cell[j].persist[k].cy, + cell[j].persist[k].max, cell[j].persist[k].y0, + cell[j].persist[k].sx, cell[j].persist[k].sy, + cell[j].persist[k].ex, cell[j].persist[k].ey, + cell[j].persist[k].y0m, cell[j].persist[k].y0p, + cell[j].persist[k].y1m, cell[j].persist[k].y1p, + cell[j].persist[k].x0m, cell[j].persist[k].x0p, + cell[j].persist[k].x1m, cell[j].persist[k].x1p, + cell[j].persist[k].func, cell[j].persist[k].up, + cell[j].persist[k].slope, cell[j].persist[k].nfit, + cell[j].persist[k].sxfit, cell[j].persist[k].exfit, + cell[j].persist[k].fiterr); + } + + for(i=0; i +#endif + +#define PPIMAGE_BURNTOOL_DEBUG 0 + +#include "ppImage.h" + +bool ppImageBurntoolMask (pmConfig *config, ppImageOptions *options, pmFPAview *view,pmReadout *mask) { + bool status = true; + int burntool_cell; + /* Find input filename */ + pmFPAfile *inputFile = psMetadataLookupPtr(&status , config->files, "PPIMAGE.INPUT"); + if (!status) { + psError(PS_ERR_IO,false, "Unable to identify inputFile"); + return(false); + } + psFits *fits = inputFile->fits; + + /* Read input header, and find the burntool data table. */ + if (!psFitsMoveExtName(fits,"burntool_areas")) { + psError(PS_ERR_IO,false, "Unable to find extension burntool_areas"); + return(false); + } + long Nrows = psFitsTableSize(fits); + long row = 0; + + psLogMsg ("ppImageBurntoolMask", 4, "Inside burntool mask %ld", Nrows); + + /* The new burntool tables and method coming into play in 2022 has a slightly different format (an extra column). Need to differentiate */ + psS16 BURNTOOL_STATE_GOOD = psMetadataLookupS16(NULL, config->camera, "BURNTOOL.STATE.GOOD"); + + /* Redirects and Memory juggling. */ + view->readout = 0; + psImage *image = mask->mask; + + /* Set the maskValue from the recipes. */ + psImageMaskType maskValue = options->burntoolMask; +#if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Status: %ld %d\n",Nrows,maskValue); +#endif + + burntool_cell = view->cell; + burntool_cell = (view->cell % 8) * 8 + (view->cell - (view->cell % 8)) / 8; + psLogMsg("ppImageBurntoolMask", 4, "Cell mapping: %d %d %d\n",view->cell,burntool_cell,-1); + for (row = 0; row < Nrows; row++) { + psMetadata *rowMD = psFitsReadTableRow(fits,row); + + if (psMetadataLookupS32(&status,rowMD,"cell") == burntool_cell) { + if (((options->burntoolTrails & 0x01)&&(psMetadataLookupS32(&status,rowMD,"func") == 4))|| + (((options->burntoolTrails & 0x02)&&(psMetadataLookupS32(&status,rowMD,"up") == 1))|| + ((options->burntoolTrails & 0x04)&&(psMetadataLookupS32(&status,rowMD,"up") == 0)))) { + /* If the fit fails, burntool reports zero here. This + signifies that it expected to see a trail (else why + fit) but did not find it when it attempted to + correct. */ +#if PPIMAGE_BURNTOOL_DEBUG + psLogMsg ("ppImageBurntoolMask", 4, "Masking! %d (%d %d %d) %d %d", + psMetadataLookupS32(&status,rowMD,"cell"), + ((options->burntoolTrails & 0x0001)&&(psMetadataLookupS32(&status,rowMD,"nfit") == 0)), + ((options->burntoolTrails & 0x02)&&(psMetadataLookupS32(&status,rowMD,"up") == 1)), + ((options->burntoolTrails & 0x04)&&(psMetadataLookupS32(&status,rowMD,"up") == 0)), + options->burntoolTrails, + maskValue + ); +#endif + + /* do separate masking strategy for old (14 and lower) and new burntool tables*/ + if (BURNTOOL_STATE_GOOD >= 15) { + for (int i = psMetadataLookupS32(&status,rowMD,"sxfit"); + i <= psMetadataLookupS32(&status,rowMD,"exfit"); + i++) { + + if (psMetadataLookupS32(&status,rowMD,"up") == 0) { + for (int j = psMetadataLookupS32(&status,rowMD,"eyfit"); j <= psMetadataLookupS32(&status,rowMD,"y0"); j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } else { + for (int j = psMetadataLookupS32(&status,rowMD,"sy"); j < psMetadataLookupS32(&status,rowMD,"eyfit") ; j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + } + } else { + for (int i = psMetadataLookupS32(&status,rowMD,"sxfit"); + i <= psMetadataLookupS32(&status,rowMD,"exfit"); + i++) { + + if (psMetadataLookupS32(&status,rowMD,"up") == 0) { + for (int j = 0; j <= psMetadataLookupS32(&status,rowMD,"y1p"); j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + else { + for (int j = psMetadataLookupS32(&status,rowMD,"y1m"); j < image->numRows ; j++) { + #if PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", + i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); + #endif + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + } + + } + + + } + } + psFree(rowMD); + } + + return(status); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageCheckCTE.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageCheckCTE.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageCheckCTE.c (revision 42655) @@ -0,0 +1,113 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// XXX I originally coded this to create a new pmFPAfile, but in retrospect it makes more sense +// to treat this function as an operation on the input image + +// XXX make the choice of stats optional +bool ppImageCheckCTE(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + bool status; + + // this step is complete optional. + if (!options->checkCTE) { + return true; + } + + // psTimerStart("check.cte"); + + // add recipe options supplied on command line + psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); + + // find the currently selected readout + pmReadout *inReadout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + psImage *image = inReadout->image; + psImage *mask = inReadout->mask; + // psImage *variance = inReadout->variance; + + // I have the fine image size, I know the binning factor, determine the ruff image size + psImageBinning *binning = psImageBinningAlloc(); + binning->nXfine = image->numCols; + binning->nYfine = image->numRows; + binning->nXbin = psMetadataLookupS32 (&status, recipe, "CTE.XBIN"); + binning->nYbin = psMetadataLookupS32 (&status, recipe, "CTE.YBIN"); + + psImageBinningSetRuffSize(binning, PS_IMAGE_BINNING_CENTER); + psImageBinningSetSkip(binning, image); + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + psImageBackground (stats, NULL, image, mask, 0xffff, rng); + float cellMedian = stats->robustMedian; + psFree (stats); + + // stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV); + // psStats *statsDefaults = psStatsAlloc (PS_STAT_SAMPLE_STDEV); + + stats = psStatsAlloc (PS_STAT_ROBUST_STDEV); + psStats *statsDefaults = psStatsAlloc (PS_STAT_ROBUST_STDEV); + + // measure median and variance for subimages + psRegion ruffRegion = {0,0,0,0}; + psRegion fineRegion = {0,0,0,0}; + for (int iy = 0; iy < binning->nYruff; iy++) { + for (int ix = 0; ix < binning->nXruff; ix++) { + + // convert the ruff grid cell to the equivalent fine grid cell + ruffRegion = psRegionSet (ix, ix + 1, iy, iy + 1); + fineRegion = psImageBinningSetFineRegion (binning, ruffRegion); + fineRegion = psRegionForImage (image, fineRegion); + if (fineRegion.x0 >= image->numCols) continue; + if (fineRegion.y0 >= image->numRows) continue; + + psImage *subset = psImageSubset (image, fineRegion); + if (!subset->numCols || !subset->numRows) { + psFree (subset); + continue; + } + psImage *submask = NULL; + if (mask) { + submask = psImageSubset (mask, fineRegion); + } + + // reset the default values + statsDefaults->tmpData = stats->tmpData; // XXX this is fairly hackish: tmpData is internal storage for stats; the assign drops the reference... + *stats = *statsDefaults; + statsDefaults->tmpData = NULL; + + psImageStats (stats, subset, submask, 0xffff); + + // XXX need to apply the gain as well + float normVariance = PS_SQR(stats->robustStdev) / cellMedian; + // float normVariance = PS_SQR(stats->sampleStdev) / cellMedian; + // if (!isfinite(normVariance)) fprintf (stderr, "** normVariance is nan **\n"); + + // apply resulting value to the input pixels + for (int jy = fineRegion.y0; jy < fineRegion.y1; jy++) { + if (jy < 0) continue; + if (jy >= image->numRows) continue; + for (int jx = fineRegion.x0; jx < fineRegion.x1; jx++) { + if (jx < 0) continue; + if (jx >= image->numCols) continue; + image->data.F32[jy][jx] = normVariance; + } + } + + psFree (subset); + psFree (submask); + } + } + + psFree (rng); + psFree (binning); + psFree (stats); + psFree (statsDefaults); + + // psLogMsg ("ppImage", 5, "check CTE: %f sec\n", psTimerMark ("check.cte")); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageCheckCTE.v1.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageCheckCTE.v1.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageCheckCTE.v1.c (revision 42655) @@ -0,0 +1,101 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageCheckCTE(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + bool status; + + // this step is complete optional. + if (!options->checkCTE) { + return true; + } + + // add recipe options supplied on command line + psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); + + // find the currently selected readout + pmReadout *inReadout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + psImage *image = inReadout->image; + psImage *mask = inReadout->mask; + // psImage *variance = inReadout->variance; + + // I have the fine image size, I know the binning factor, determine the ruff image size + psImageBinning *binning = psImageBinningAlloc(); + binning->nXfine = image->numCols; + binning->nYfine = image->numRows; + binning->nXbin = psMetadataLookupS32 (&status, recipe, "CTE.XBIN"); + binning->nYbin = psMetadataLookupS32 (&status, recipe, "CTE.YBIN"); + + psImageBinningSetRuffSize(binning, PS_IMAGE_BINNING_CENTER); + psImageBinningSetSkip(binning, image); + + pmCell *inCell = pmFPAfileThisCell (config->files, view, "PPIMAGE.INPUT"); + pmCell *outCell = pmFPAfileThisCell (config->files, view, "PPIMAGE.CTEMAP"); + if (!pmCellCopyStructure(outCell, inCell, binning->nXbin, binning->nYbin)) { + psError(PS_ERR_UNKNOWN, false, "Unable to copy cell structure."); + return false; + } + + pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPIMAGE.CTEMAP"); + psImage *output = outRO->image; + + // Don't care about the bias: get rid of it, if present + psFree(outRO->bias); + outRO->bias = psListAlloc(NULL); + psMetadataItem *biassec = psMetadataLookup(outCell->concepts, "CELL.BIASSEC"); + psFree(biassec->data.V); + biassec->data.V = psListAlloc(NULL); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + psStats *statsDefaults = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + + // measure median and variance for subimages + psRegion ruffRegion = {0,0,0,0}; + psRegion fineRegion = {0,0,0,0}; + for (int iy = 0; iy < output->numRows; iy++) { + for (int ix = 0; ix < output->numCols; ix++) { + + // convert the ruff grid cell to the equivalent fine grid cell + ruffRegion = psRegionSet (ix, ix + 1, iy, iy + 1); + fineRegion = psImageBinningSetFineRegion (binning, ruffRegion); + fineRegion = psRegionForImage (image, fineRegion); + if (fineRegion.x0 >= image->numCols || fineRegion.x1 >= image->numCols || + fineRegion.y0 >= image->numRows || fineRegion.y1 >= image->numRows) { + continue; + } + + psImage *subset = psImageSubset (image, fineRegion); + if (!subset->numCols || !subset->numRows) { + psFree (subset); + continue; + } + psImage *submask = NULL; + if (mask) { + submask = psImageSubset (mask, fineRegion); + } + + // reset the default values + statsDefaults->tmpData = stats->tmpData; // XXX this is fairly hackish: tmpData is internal storage for stats; the assign drops the reference... + *stats = *statsDefaults; + statsDefaults->tmpData = NULL; + + psImageStats (stats, subset, submask, 0); + + // XXX need to apply the gain as well + output->data.F32[iy][ix] = PS_SQR(stats->sampleStdev) / stats->sampleMedian; + + psFree (subset); + psFree (submask); + } + } + + psFree (binning); + psFree (stats); + psFree (statsDefaults); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageCheckNoise.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageCheckNoise.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageCheckNoise.c (revision 42655) @@ -0,0 +1,131 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// XXX I originally coded this to create a new pmFPAfile, but in retrospect it makes more sense +// to treat this function as an operation on the input image + +// XXX make the choice of stats optional +bool ppImageCheckNoise(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + bool status; + + // this step is complete optional. + if (!options->checkNoise) { + return true; + } + + // psTimerStart("check.noise"); + + // add recipe options supplied on command line + psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); + + // find the currently selected readout + pmReadout *inReadout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + psImage *image = inReadout->image; + psImage *mask = inReadout->mask; + // psImage *variance = inReadout->variance; + + // I have the fine image size, I know the binning factor, determine the ruff image size + psImageBinning *binning = psImageBinningAlloc(); + binning->nXfine = image->numCols; + binning->nYfine = image->numRows; + binning->nXbin = psMetadataLookupS32 (&status, recipe, "NOISE.XBIN"); + binning->nYbin = psMetadataLookupS32 (&status, recipe, "NOISE.YBIN"); + + psImageBinningSetRuffSize(binning, PS_IMAGE_BINNING_CENTER); + psImageBinningSetSkip(binning, image); + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + psImageBackground (stats, NULL, image, mask, 0xffff, rng); + // float cellMedian = stats->robustMedian; + psFree (stats); + + // stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV); + // psStats *statsDefaults = psStatsAlloc (PS_STAT_SAMPLE_STDEV); + + stats = psStatsAlloc (PS_STAT_ROBUST_STDEV); + psStats *statsDefaults = psStatsAlloc (PS_STAT_ROBUST_STDEV); + + // measure median and variance for subimages + psRegion ruffRegion = {0,0,0,0}; + psRegion fineRegion = {0,0,0,0}; + for (int iy = 0; iy < binning->nYruff; iy++) { + for (int ix = 0; ix < binning->nXruff; ix++) { + + // convert the ruff grid cell to the equivalent fine grid cell + ruffRegion = psRegionSet (ix, ix + 1, iy, iy + 1); + fineRegion = psImageBinningSetFineRegion (binning, ruffRegion); + fineRegion = psRegionForImage (image, fineRegion); + if (fineRegion.x0 >= image->numCols) continue; + if (fineRegion.y0 >= image->numRows) continue; + + psImage *subset = psImageSubset (image, fineRegion); + if (!subset->numCols || !subset->numRows) { + psFree (subset); + continue; + } + psImage *submask = NULL; + if (mask) { + submask = psImageSubset (mask, fineRegion); + } + + // reset the default values + statsDefaults->tmpData = stats->tmpData; // XXX this is fairly hackish: tmpData is internal storage for stats; the assign drops the reference... + *stats = *statsDefaults; + statsDefaults->tmpData = NULL; + + psImageStats (stats, subset, submask, 0xffff); + + // XXX need to apply the gain as well + float normVariance = stats->robustStdev; + // float normVariance = PS_SQR(stats->sampleStdev) / cellMedian; + // if (!isfinite(normVariance)) fprintf (stderr, "** normVariance is nan **\n"); + psTrace("ppImage.noise",3, + "PPIM.N: %s %s %d %d (%d %d) %f\n", + psMetadataLookupStr(NULL,inReadout->parent->parent->concepts,"CHIP.NAME"), + psMetadataLookupStr(NULL,inReadout->parent->concepts,"CELL.NAME"), + ix,iy,subset->numCols,subset->numRows,normVariance); + + for (int dy = 0; dy < subset->numRows; dy++) { + for (int dx = 0; dx < subset->numCols; dx++) { + psTrace("ppImage.noise",5, + "PPIN: %s %s %d %d %d %d %f %d %f\n", + psMetadataLookupStr(NULL,inReadout->parent->parent->concepts,"CHIP.NAME"), + psMetadataLookupStr(NULL,inReadout->parent->concepts,"CELL.NAME"), + ix,iy,dx,dy, + subset->data.F32[dy][dx], + (int) submask->data.PS_TYPE_VECTOR_MASK_DATA[dy][dx], + normVariance); + } + } + // binning->data.F32[iy][ix] = normVariance; + // apply resulting value to the input pixels + for (int jy = fineRegion.y0; jy < fineRegion.y1; jy++) { + if (jy < 0) continue; + if (jy >= image->numRows) continue; + for (int jx = fineRegion.x0; jx < fineRegion.x1; jx++) { + if (jx < 0) continue; + if (jx >= image->numCols) continue; + image->data.F32[jy][jx] = normVariance; + } + } + + psFree (subset); + psFree (submask); + } + } + + psFree (rng); + psFree (binning); + psFree (stats); + psFree (statsDefaults); + + // psLogMsg ("ppImage", 5, "check noise: %f sec\n", psTimerMark ("check.noise")); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageCleanup.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageCleanup.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageCleanup.c (revision 42655) @@ -0,0 +1,35 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +void ppImageCleanup (pmConfig *config, ppImageOptions *options) +{ + if (psErrorCodeLast() != PS_ERR_NONE) { + pmFPAfileFreeSetStrict(false); + } + + // Free memory used by ppImage + psFree(options); + psFree(config); + + // Free memory used by psModules + pmSourceFitSetDone (); + pmConceptsDone(); + pmConfigDone(); + pmModelClassCleanup(); + pmVisualCleanup (); + + // Free memory used by psLib + psLibFinalize(); + + // psMemBlock **memblocks; + // int Nleaks = psMemCheckLeaks (0, &memblocks, stderr, false); + // fprintf (stderr, "Found %d leaks at %s\n", Nleaks, "ppImage"); + + // fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "ppImage"); + psLogMsg("ppImage", PS_LOG_INFO, "Memory leaks: %d\n", psMemCheckLeaks(0, NULL, stdout, false)); + + return; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageCorrectCrosstalk.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageCorrectCrosstalk.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageCorrectCrosstalk.c (revision 42655) @@ -0,0 +1,18 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +#define ESCAPE(MESSAGE) { \ + psError(PS_ERR_UNKNOWN, false, MESSAGE); \ + psFree(view); \ + return false; \ +} + +// For the moment, this implementation is VERY GPC-specific + +bool ppImageCorrectCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageDefineFile.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDefineFile.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDefineFile.c (revision 42655) @@ -0,0 +1,56 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +bool ppImageDefineFile(pmConfig *config, pmFPA *input, char *filerule, char *argname, + pmFPAfileType fileType, pmDetrendType detrendType) +{ + bool status; + pmFPAfile *file = NULL; // File to be defined + + if (!file) { + // look for the file on the argument list + file = pmFPAfileDefineFromArgs(&status, config, filerule, argname); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to load file definition ARG LIST"); + return false; + } + } + if (!file) { + // look for the file on the RUN metadata + file = pmFPAfileDefineFromRun(&status, NULL, config, filerule); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to load file definition RUN"); + return false; + } + } + if (!file) { + // look for the file in the camera config table + file = pmFPAfileDefineFromConf(&status, config, filerule); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to load file definition CONFIG"); + return false; + } + } + if (!file) { + // look for the file to be loaded from the detrend database + file = pmFPAfileDefineFromDetDB(&status, config, filerule, input, detrendType); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to load file definition DETREND"); + return false; + } + } + + if (!file) { + return false; + } + + if (file->type != fileType) { + psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType)); + return false; + } + return true; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendFree.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendFree.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendFree.c (revision 42655) @@ -0,0 +1,73 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// list of the detrend types to free +static char *detrendTypes[] = { + "PPIMAGE.MASK", + "PPIMAGE.NOISEMAP", + "PPIMAGE.BIAS", + "PPIMAGE.DARK", + "PPIMAGE.FLAT", + "PPIMAGE.SHUTTER", + "PPIMAGE.LINEARITY", + "PPIMAGE.NEWNONLIN", + NULL +}; + +bool ppImageDetrendFree (pmConfig *config, pmFPAview *view) { + + bool status; + + for (int i = 0; detrendTypes[i] != NULL; i++) { + + pmFPAfile *file = psMetadataLookupPtr(&status, config->files, detrendTypes[i]); // File of interest + psTrace("pmFPAfileFree",1,"Working on %s\n",detrendTypes[i]); + if (!file) continue; // not all detrends are used in any given run + + // this only returns false on a failure. if we are not ready to write or close, it is not an error + if (!pmFPAfileWrite (file, view, config)) { + psError(PS_ERR_IO, false, "failed to WRITE %s", file->name); + return false; + } + if (!pmFPAfileClose(file, view)) { + psError(PS_ERR_IO, false, "failed to CLOSE for %s", file->name); + return false; + } + if (!pmFPAfileFreeData(file, view)) { + if (!psMetadataRemoveKey(config->files, file->name)) { + psError(PS_ERR_IO, false, "failed to remove %s in FPA_AFTER block", file->name); + return false; + } + } + } + return true; +} + +bool ppImageFringeFree (pmConfig *config, pmFPAview *view) { + + bool status; + + pmFPAfile *file = psMetadataLookupPtr(&status, config->files, "PPIMAGE.FRINGE"); // File of interest + if (!file) return true; + + // this only returns false on a failure. if we are not ready to write or close, it is not an error + if (!pmFPAfileWrite (file, view, config)) { + psError(PS_ERR_IO, false, "failed to WRITE %s", file->name); + return false; + } + if (!pmFPAfileClose(file, view)) { + psError(PS_ERR_IO, false, "failed to CLOSE for %s", file->name); + return false; + } + if (!pmFPAfileFreeData(file, view)) { + if (!psMetadataRemoveKey(config->files, file->name)) { + psError(PS_ERR_IO, false, "failed to remove %s in FPA_AFTER block", file->name); + return false; + } + } + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendFringe.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendFringe.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendFringe.c (revision 42655) @@ -0,0 +1,393 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageDetrendFringeMeasure(pmReadout *readout, pmCell *fringe, const bool isResidual, const ppImageOptions *options) +{ + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(fringe, false); + PS_ASSERT_PTR_NON_NULL(options, false); + + // Reference fringe measurements (stored on the reference cell->analysis) + psArray *references = psMemIncrRefCounter(psMetadataLookupPtr(NULL, fringe->analysis, "FRINGE.MEASUREMENTS")); + if (!references) { + psError(PS_ERR_IO, false, "Unable to find fringe references.\n"); + return false; + } + + pmFringeStats *reference = references->data[0]; // Take the first as representative + pmFringeRegions *regions = reference->regions; // Regions to measure + pmFringeStats *measurements = pmFringeStatsMeasure(regions, readout, options->maskValue); // Fringe stats + + // Normalise measurements by the exposure time + bool mdok; // Status of MD lookup + float expTime = psMetadataLookupF32(&mdok, readout->parent->concepts, "CELL.EXPOSURE"); // Exp. time + if (!mdok || !isfinite(expTime)) { + psError(PS_ERR_UNKNOWN, false, "CELL.EXPOSURE is not set for --- can't normalise fringes\n"); + psFree(measurements); + return false; + } + if (expTime == 0) { + psWarning("Exposure time is zero --- are you sure you want fringe subtraction?\n"); + expTime = 1.0; + } + // normalize by exposure time : NOTE df is 1/sigma + psBinaryOp(measurements->f, measurements->f, "*", psScalarAlloc(1.0 / expTime, PS_TYPE_F32)); + psBinaryOp(measurements->df, measurements->df, "*", psScalarAlloc(expTime, PS_TYPE_F32)); + + char *scienceFringes = NULL; + if (isResidual) { + scienceFringes = psStringCopy ("FRINGE.RESIDUALS"); + } else { + scienceFringes = psStringCopy ("FRINGE.MEASUREMENTS"); + } + + // Science fringe measurements + pmFringeStats *previous = psMetadataLookupPtr(NULL, readout->parent->analysis, scienceFringes); + if (previous) { + // Multiple readouts: concatenate + psArray *concatenate = psArrayAlloc(2); // Array to hold fringes + + // Concatenate science measurements + concatenate->data[0] = previous; + concatenate->data[1] = measurements; + pmFringeStats *new = pmFringeStatsConcatenate(concatenate, NULL, NULL); // New measurements + psFree(measurements); + measurements = new; + + // Concatenate reference measurements (duplication, so the science and reference line up) + for (int i = 0; i < references->n; i++) { + concatenate->data[0] = concatenate->data[1] = references->data[i]; + pmFringeStats *refNew = pmFringeStatsConcatenate(concatenate, NULL, NULL); + psFree(references->data[i]); + references->data[i] = refNew; + } + concatenate->data[0] = concatenate->data[1] = NULL; + psFree(concatenate); + } + + psMetadataAdd(readout->parent->analysis, PS_LIST_TAIL, scienceFringes, PS_DATA_UNKNOWN | PS_META_REPLACE, "Fringe measurements", measurements); + psFree(measurements); + psFree(scienceFringes); + psFree(references); + + return true; +} + + +// Pull the fringes out of the cell analysis FRINGE.MEASUREMENTS for a chip +// XXX need some error checks +static psArray *getFringes(const pmChip *chip, const char *source) +{ + psArray *cells = chip->cells; // Component cells + psArray *fringes = psArrayAlloc(cells->n); // Fringes, to return + int video_cell_zero = 0; + + for (int i = 0; i < cells->n; i++) { + fringes->data[i] = NULL; + + pmCell *cell = cells->data[i]; // Cell of interest + + psTrace("psModules.detrend",7,"Readouts: Cell %d chip: %ld\n",i,cell->readouts->n); + // XXX for now, skip the video cells (cell->readouts->n > 1) + // CZW: This mess creates a fake set of fringe stats by stealing the previous one. + // We let the fitting code know that this is all lies by scaling the weights by a crazy amount. + + if ( (cell->readouts->n > 1) ) { + psTrace("psModules.detrend",7,"Should be skipping scichip: %d\n",i); + psWarning ("Skipping Video Cell (%d) for ppImageDetrendFringe.c:getFringes", i); + + if (i == 0) { + video_cell_zero = 1; + } + else { + pmFringeStats *prevFringe = fringes->data[i-1]; + pmFringeStats *fringe = pmFringeStatsAlloc(prevFringe->regions); + for (int j = 0; j < fringe->regions->nRequested; j++) { + fringe->f->data.F32[j] = prevFringe->f->data.F32[j]; + fringe->df->data.F32[j] = prevFringe->df->data.F32[j] / 1e6; + } + fringes->data[i] = fringe; + } + + continue; + } + + fringes->data[i] = psMemIncrRefCounter(psMetadataLookupPtr(NULL, cell->analysis, source)); + } + + if (video_cell_zero == 1) { + pmFringeStats *prevFringe = fringes->data[1]; + pmFringeStats *fringe = pmFringeStatsAlloc(prevFringe->regions); + for (int j = 0; j < fringe->regions->nRequested; j++) { + fringe->f->data.F32[j] = NAN; + fringe->df->data.F32[j] = 1.0; + } + fringes->data[0] = fringe; + } + + return fringes; +} + + +// Solve the fringe system: we have science fringe measurements for each cell, and an array of reference +// fringe measurements for each cell. Need to concatenate these together first, and then solve. +bool ppImageDetrendFringeSolve(pmChip *scienceChip, const pmChip *refChip, const bool isResidual, const ppImageOptions *options) +{ + PS_ASSERT_PTR_NON_NULL(scienceChip, NULL); + PS_ASSERT_PTR_NON_NULL(refChip, NULL); + PS_ASSERT_PTR_NON_NULL(options, NULL); + + psArray *science = NULL; + if (isResidual) { + science = getFringes(scienceChip, "FRINGE.RESIDUALS"); // Fringe residuals on science chip + } else { + science = getFringes(scienceChip, "FRINGE.MEASUREMENTS"); // Fringe measurements on science chip + } + + pmFringeStats *scienceCat = pmFringeStatsConcatenate(science, NULL, NULL); // Science fringes + psFree(science); + + // Need to transform the array of cells each with an array of fringes --> array of fringes for the chip as + // a whole + psArray *references = getFringes(refChip, "FRINGE.MEASUREMENTS"); // Fringe measurements on reference chip + int numRefs = ((psArray*)references->data[0])->n; // Number of reference fringes + psArray *referencesCat = psArrayAlloc(numRefs); // Reference fringes + for (int i = 0; i < numRefs; i++) { // Iterate over fringes + psArray *refs = psArrayAlloc(references->n); // Array of fringes for each cell + for (int j = 0; j < references->n; j++) { // Iterate over cells + psArray *ref = references->data[j]; // Array of references for this cell + + refs->data[j] = psMemIncrRefCounter(ref->data[i]); + } + referencesCat->data[i] = pmFringeStatsConcatenate(refs, NULL, NULL); + psFree(refs); + } + psFree(references); + + // Now we can solve + psTrace("ppImage", 3, "Solving fringe system...\n"); + pmFringeScale *solution = pmFringeScaleMeasure(scienceCat, referencesCat, options->fringeRej, + options->fringeIter, options->fringeKeep); + + if (isResidual) { + psMetadataAdd(scienceChip->analysis, PS_LIST_TAIL, "FRINGE.RESIDUAL.SOLUTION", PS_DATA_UNKNOWN, "Fringe solution", solution); + } else { + psMetadataAdd(scienceChip->analysis, PS_LIST_TAIL, "FRINGE.SOLUTION", PS_DATA_UNKNOWN, "Fringe solution", solution); + } + +# if (0) + // write the fringe amplitude or residual amplitude to the header + // XXX this is measured per cell, but we only have headers per chip + pmHDU *hdu = pmHDUFromCell(science);// HDU of interest + for (int i = 0; i < solution->nFringeFrames; i++) { + // write metadata header value + psString keyword = NULL; + if (isResidual) { + psStringAppend (&keyword, "FRES_%02dV", i); + } else { + psStringAppend (&keyword, "FRNG_%02dV", i); + } + psMetadataAddF32(hdu->header, PS_LIST_TAIL, keyword, PS_META_REPLACE, "Fringe Amplitude", solution->coeff->data.F32[i + 1]); + psFree (keyword); + + keyword = NULL; + if (isResidual) { + psStringAppend (&keyword, "FRES_%02dE", i); + } else { + psStringAppend (&keyword, "FRNG_%02dE", i); + } + psMetadataAddF32(hdu->header, PS_LIST_TAIL, keyword, PS_META_REPLACE, "Fringe Amplitude error", solution->coeffErr->data.F32[i + 1]); + psFree (keyword); + } +# endif + + psFree(solution); + psFree(scienceCat); + psFree(referencesCat); + + return true; +} + + +bool ppImageDetrendFringeGenerate(pmCell *science, pmCell *fringes, const ppImageOptions *options) +{ + PS_ASSERT_PTR_NON_NULL(science, false); + PS_ASSERT_PTR_NON_NULL(fringes, false); + + pmFringeScale *solution = psMetadataLookupPtr(NULL, science->parent->analysis, "FRINGE.SOLUTION"); + if (!solution) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find fringe solution.\n"); + return false; + } + assert(fringes->readouts->n == solution->nFringeFrames); + + psImageMaskType maskVal = options->flatMask; + + bool mdok; // Status of MD lookup + float expTime = psMetadataLookupF32(&mdok, science->concepts, "CELL.EXPOSURE"); // Exp. time + if (!mdok || !isfinite(expTime)) { + psError(PS_ERR_UNKNOWN, true, "CELL.EXPOSURE is not set --- can't renormalise fringes\n"); + return false; + } + if (expTime == 0) { + psWarning("Exposure time is zero --- are you sure you want fringe subtraction?\n"); + expTime = 1.0; + } + + pmHDU *hdu = pmHDUFromCell(science);// HDU of interest + + // Construct the fringe image from the scale + psTrace("ppImage", 3, "Generating fringe correction...\n"); + psImage *sumFringe = NULL; // Sum of the fringes + for (int i = 0; i < solution->nFringeFrames; i++) { + pmReadout *fringeRO = fringes->readouts->data[i]; // Fringe readout + psImage *fringe = fringeRO->image; // Fringe image + psTrace("ppImage", 5, "Scale for fringe component %d is %f\n", + i, solution->coeff->data.F32[i + 1]); + psBinaryOp(fringe, fringe, "*", psScalarAlloc(solution->coeff->data.F32[i + 1], PS_TYPE_F32)); + if (!sumFringe) { + sumFringe = psImageCopy(NULL, fringe, PS_TYPE_F32); + } else { + psBinaryOp(sumFringe, sumFringe, "+", fringe); + } + + psVector *md5 = psImageMD5(fringe); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psStringPrepend(&md5string, "Fringe image %d (scale %.3f) MD5: ", + i, solution->coeff->data.F32[i + 1]); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + md5string, ""); + psFree(md5string); + +# if (0) + // write metadata header value + // XXX this is measured per cell, but we only have headers per chip + psString keyword = NULL; + psStringAppend (&keyword, "FRNG_%02dV", i); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, keyword, PS_META_REPLACE, "Fringe Amplitude", solution->coeff->data.F32[i + 1]); + psFree (keyword); + + keyword = NULL; + psStringAppend (&keyword, "FRNG_%02dE", i); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, keyword, PS_META_REPLACE, "Fringe Amplitude error", solution->coeffErr->data.F32[i + 1]); + psFree (keyword); +# endif + + } + if (expTime != 1.0) { + psBinaryOp(sumFringe, sumFringe, "*", psScalarAlloc(expTime, PS_TYPE_F32)); + } + + // Apply the correction to all readouts + psArray *readouts = science->readouts; // Array of science readouts + for (int i = 0; i < readouts->n; i++) { + pmReadout *readout = readouts->data[i]; // Readout of interest + if (!readout || !readout->data_exists) { + continue; + } + + // subtract fringe and update mask if fringe value is NAN + for (int iy = 0; iy < readout->image->numRows; iy++) { + for (int ix = 0; ix < readout->image->numCols; ix++) { + readout->image->data.F32[iy][ix] -= sumFringe->data.F32[iy][ix]; + if (!isfinite(sumFringe->data.F32[iy][ix]) && readout->mask) { + readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= maskVal; + } + } + } + + // XXX: Make generic, so subregions may be subtracted as well + // if (!psBinaryOp(readout->image, readout->image, "-", sumFringe)) { + // psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to subtract fringe.\n"); + // return false; + // } + + // measure residual fringe amplitude. results go to FRINGE.RESIDUALS + ppImageDetrendFringeMeasure (readout, fringes, true, options); + } + psFree(sumFringe); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now, used for reporting + psString timeString = psTimeToISO(time); // String with time + psFree(time); + psStringPrepend(&timeString, "Fringe correction completed at "); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + timeString, ""); + psFree(timeString); + + return true; +} + +bool ppImageDetrendFringeApply (pmConfig *config, pmChip *chip, const pmFPAview *inputView, const ppImageOptions *options) { + + pmCell *cell = NULL; + + if (!options->doFringe) return true; + + assert (inputView->chip != -1); + assert (inputView->cell == -1); + assert (inputView->readout == -1); + + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + // select the reference chip + pmChip *fringe = pmFPAfileThisChip(config->files, view, "PPIMAGE.FRINGE"); + if (!fringe) { + psError(PS_ERR_UNKNOWN, false, "missing fringe reference data.\n"); + psFree (view); + return false; + } + + // Solve the fringe system + if (!ppImageDetrendFringeSolve(chip, fringe, false, options)) { + psError(PS_ERR_UNKNOWN, false, "failed to solve the fringe system.\n"); + psFree (view); + return false; + } + + // Go back over the cells to apply the fringe correction + view->cell = view->readout = -1; + while ((cell = pmFPAviewNextCell(view, chip->parent, 1)) != NULL) { + if (!cell->process || !cell->file_exists) { + continue; + } + + // Apply the fringe correction + psTrace("ppImage", 3, "Applying fringe correction...\n"); + pmCell *fringeCell = pmFPAfileThisCell(config->files, view, "PPIMAGE.FRINGE"); + if (!fringeCell) { + psError(PS_ERR_UNKNOWN, false, "missing fringe reference data.\n"); + psFree (view); + return false; + } + + // XXX for now, skip the video cells (cell->readouts->n > 1) + if (cell->readouts->n > 1) { + psWarning ("Skipping Video Cell for ppImageDetrendFringeApply"); + continue; + } + + if (!ppImageDetrendFringeGenerate(cell, fringeCell, options)) { + psError(PS_ERR_UNKNOWN, false, "failed to apply fringe image.\n"); + psFree (view); + return false; + } + } + + // Solve the residual fringe system + if (!ppImageDetrendFringeSolve(chip, fringe, true, options)) { + psError(PS_ERR_UNKNOWN, false, "failed to solve the residual fringe system.\n"); + psFree (view); + return false; + } + + // psLogMsg ("ppImage", 5, "apply fringe: %f sec\n", psTimerMark ("apply.fringe")); + + psFree (view); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendNewNonLinear.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendNewNonLinear.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendNewNonLinear.c (revision 42655) @@ -0,0 +1,42 @@ +#include "ppImage.h" + +bool ppImageDetrendNewNonLinear(pmReadout *input, pmFPAview *detview, pmConfig *config) { + + bool status; + + pmFPAfile *linearity_file = psMetadataLookupPtr(&status, config->files, "PPIMAGE.NEWNONLIN"); + psFits *linearity_fits = linearity_file->fits; + + char *extname = psMetadataLookupStr(&status, input->parent->concepts, "CELL.NAME"); + if (!extname) { + psError(PS_ERR_IO, false, "missing CELL.NAME in concepts"); + return(false); + } + + // if pmFPAfile has been loaded (by ppImageDefineFile in ppImageParseCamera), then + // the file corresponding to the current chip is found and opened + // NOTE: if the extname is missing, we skip the correction + if (!psFitsMoveExtName(linearity_fits, extname)) { + psLogMsg ("ppImageDetrendNewNonLinear", 4, "Unable to move to non-linearity (v2023) table %s, skipping", extname); + return(true); + } + + psArray *table = psFitsReadTable(linearity_fits); + if (!table) { + psError(PS_ERR_IO, false, "Unable to read non-linearity table.\n"); + return(false); + } + + if (!pmNewNonLinearityApply(input,table)) { + psError(PS_ERR_UNKNOWN, false, "Unable to apply non-linearity corrections.\n"); + psFree (table); + return(false); + } + psFree (table); + + return true; +} + +// the new non-linearity correction is a set of splines, one per cell, with the +// xKnots equal to the log10(DN) in the pixel, and the spline value at a point +// equal to a fractional multiplier (i.e., 1 + dF) Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendNonLinear.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendNonLinear.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendNonLinear.c (revision 42655) @@ -0,0 +1,145 @@ +#include "ppImage.h" + +bool ppImageDetrendNonLinearPolynomial(pmReadout *input, psMetadataItem *dataItem) { + + // These are the polynomial coefficients + psVector *coeff = dataItem->data.V; // The coefficient vector + if (coeff->type.type != PS_TYPE_F64) { + psVector *temp = psVectorCopy(NULL, coeff, PS_TYPE_F64); // F64 version + psFree (coeff); + coeff = temp; + } + psPolynomial1D *correction = psPolynomial1DAlloc(coeff->n - 1, PS_POLYNOMIAL_ORD); + psFree(correction->coeff); + correction->coeff = psMemIncrRefCounter(coeff->data.F64); + pmNonLinearityPolynomial(input, correction); + psFree(coeff); + psFree(correction); + return true; +} + +bool ppImageDetrendNonLinearLookup(pmReadout *input, psMetadataItem *dataItem) { + + // This is a filename: lookup table + char *name = dataItem->data.V; // Filename + psLookupTable *table = psLookupTableAlloc(name, "%f %f", 0); + if (psLookupTableRead(table) <= 0) { + psErrorStackPrint(stderr, "Unable to read non-linearity correction file " + "%s --- ignored\n", name); + return false; + } +#ifdef PRODUCTION + pmNonLinearityLookup(input, table); +#else + psVector *influx = table->values->data[0]; + psVector *outflux = table->values->data[1]; + pmNonLinearityLookup(input, influx, outflux); +#endif + psFree(table); + return true; +} + + +bool ppImageDetrendNonLinear(pmReadout *input, pmFPAview *detview, pmConfig *config) { + bool status; + + pmFPAfile *linearity_file = psMetadataLookupPtr(&status,config->files,"PPIMAGE.LINEARITY"); + psFits *linearity_fits = linearity_file->fits; + + char *extname = psMetadataLookupStr(&status,input->parent->concepts,"CELL.NAME"); + if (!extname) { + psError(PS_ERR_IO, false, "missing CELL.NAME in concepts"); + return(false); + } + + if (!psFitsMoveExtName(linearity_fits,extname)) { + psError(PS_ERR_IO, false, "Unable to move to non-linearity table %s", extname); + return(false); + } + + psArray *table = psFitsReadTable(linearity_fits); + if (!table) { + psError(PS_ERR_IO, false, "Unable to read non-linearity table.\n"); + return(false); + } + + // It might be better to pack lookup table here... + // Why? I only use that lookup table once for the single cell it matches. + + if (!pmNonLinearityApply(input,table)) { + psError(PS_ERR_UNKNOWN, false, "Unable to apply non-linearity corrections.\n"); + psFree (table); + return(false); + } + psFree (table); + + return true; +} + +bool ppImageDetrendNonLinear_Original(pmReadout *input, ppImageOptions *options) { + + psMetadataItem *concept; + pmCell *cell = input->parent; + + switch (options->nonLinearType) { + case PS_DATA_VECTOR: + ppImageDetrendNonLinearPolynomial (input, options->nonLinearData); + return true; + + case PS_DATA_STRING: + ppImageDetrendNonLinearLookup (input, options->nonLinearData); + return true; + + case PS_DATA_METADATA: + // Go looking for the value in the hierarchy + concept = psMetadataLookup(cell->concepts, options->nonLinearSource); + if (! concept) { + pmChip *chip = cell->parent;// Parent chip + concept = psMetadataLookup(chip->concepts, options->nonLinearSource); + if (! concept) { + pmFPA *fpa = chip->parent; // Parent FPA + concept = psMetadataLookup(fpa->concepts, options->nonLinearSource); + if (! concept) { + psLogMsg("phase2", PS_LOG_WARN, "Unable to find value of concept %s " + "for non-linearity correction --- ignored.\n", (char *)options->nonLinearSource); + return false; + } + } + } + + if (concept->type != PS_DATA_STRING) { + psLogMsg("phase2", PS_LOG_WARN, "Type for concept %p isn't STRING, as" + " expected for non-linearity correction --- ignored.\n", + concept); + return false; + } + + // Get the value of the concept + psString conceptValue = concept->data.V; + psMetadata *folder = (psMetadata *)options->nonLinearData->data.V; + psMetadataItem *optionItem = psMetadataLookup(folder, conceptValue); + if (!optionItem) { + psLogMsg("phase2", PS_LOG_WARN, "Unable to find %s in NONLIN.DATA" + " --- ignored.\n", conceptValue); + return false; + } + + switch (optionItem->type) { + case PS_DATA_VECTOR: + ppImageDetrendNonLinearPolynomial (input, optionItem); + return true; + case PS_DATA_STRING: + ppImageDetrendNonLinearLookup (input, optionItem); + return true; + default: + psLogMsg("phase2", PS_LOG_WARN, "Non-linearity correction " + "desired but unable to interpret NONLIN.DATA for %s" + " --- ignored\n", conceptValue); + return false; + } + default: + psAbort("Invalid options->nonLinearType"); + } + return true; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendPattern.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendPattern.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendPattern.c (revision 42655) @@ -0,0 +1,544 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +#define ESCAPE(STATUS,...) { \ + psError(PS_ERR_UNKNOWN, STATUS, __VA_ARGS__); \ + psFree(view); \ + return false; \ + } + +static bool doPatternForView (bool *doit, const pmConfig *config, const pmChip *chip, const pmFPAview *view, const char *recipename, const char *recipevalue); + +bool ppImageDetrendPatternRowApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options); +bool ppImageDetrendPatternContinuityApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options); +bool ppImageDetrendPatternCellApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options); +bool ppImageDetrendPatternDeadCellsApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options); + +// Apply the desired pattern corrections (if any). This function is passed the chip for the +// image being processed, but may interact with pmFPAfiles containing the pattern reference information +bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options) +{ + assert(inputView->chip != -1); + assert(inputView->cell == -1); + assert(inputView->readout == -1); + + if (!ppImageDetrendPatternRowApply (config, chip, inputView, options)) { + return false; + } + if (!ppImageDetrendPatternContinuityApply (config, chip, inputView, options)) { + return false; + } + if (!ppImageDetrendPatternCellApply (config, chip, inputView, options)) { + return false; + } + if (!ppImageDetrendPatternDeadCellsApply (config, chip, inputView, options)) { + return false; + } + return(true); +} + +bool ppImageDetrendPatternRowApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options) +{ + bool status; + + if (!options->doPatternRow) return true; + + assert(inputView->chip != -1); + assert(inputView->cell == -1); + assert(inputView->readout == -1); + + // PATTERN.ROW is selected by the recipe. If it is selected, we search for the table + // PATTERN.ROW.SUBSET. If this is found in our format file, we use that version; + // otherwise, we use the table provided in the recipe file "ppImage.config". Within that + // table, we select the entry that matches our CHIP.NAME. This will be either a boolean or + // a string of bits. If it is a boolean, it specified whether or not to correct the entire + // chip; if it is a string, the bits specify which cells to correct (sequence is order of + // CELLS in the format:CHIPS metadata table) + + // New 2021 October : During ppImageParseCamera, we have loaded a file identified by + // PPIMAGE.PATTERN.ROW.AMP on config->files. This file contains a collection of FITS + // tables, one per chip. Each table lists the typical bias-drift amplitude for cell, + // measured from a collection of dark images. These values are passed (via + // cell->analysis) to pmPatternRow which chooses to apply the correction based on the + // ratio of the poisson noise due to the median background (+ read noise) and the + // typical amplitude. Tests show that if the amplitude / noise < 1/6, then the + // row-by-row variations are insignificant. + + // We also check the chip header for the boolean 'PTRN_ROW' : if this is true, we have + // already applied this correct to this data, so we simply skip the correction for this + // chip. + + pmHDU *hdu = pmHDUFromChip(chip); + if (psMetadataLookupBool(&status, hdu->header, "PTRN_ROW")) { + psLogMsg("ppImage", PS_LOG_INFO, "Not performing row pattern correction as it has already been done."); + return true; + } + + const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME"); + psLogMsg("ppImage", PS_LOG_INFO, "Performing row pattern correction for %s\n", chipName); + + // XXX we use input->fpa below, but could we just use chip->parent? + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT"); + + // grab the PATTERN.ROW.AMP file + pmFPAfile *PRAfile = psMetadataLookupPtr (&status, config->files, "PPIMAGE.PATTERN.ROW.AMP"); + if (!PRAfile) { + psLogMsg("ppImage", PS_LOG_INFO, "Pattern Row Amplitude file not found, applying to all (with limits from ROW.SUBSET) "); + } + + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + pmCell *cell = NULL; + while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + // const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + // const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + // psWarning ("Looping through %s, %s\n", chipName, cellName); + + if (!cell->process || !cell->file_exists) { + continue; + } + + // this forces pmFPAfileRead of the PATTERN.ROW.AMP file (XXX but is this needed?) + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE(false, "load failure for Cell"); + } + + if (!cell->data_exists) { + continue; + } + + if (cell->readouts->n > 1) { + psWarning ("Skipping Video Cell for ppImageDetrendPatternApply"); + continue; + } + + // grab the corresponding cell + if (PRAfile) { + pmCell *PRAcell = pmFPAviewThisCell (view, PRAfile->fpa); + psAssert (PRAcell, "found Pattern Row Amplitude file, but not cell?"); + + // find the nominal signal amplitude (check the ghost and/or crosstalk recipe file) + float amplitude = psMetadataLookupF32 (&status, PRAcell->analysis, "PTN.ROW.AMP"); + if (!status) amplitude = NAN; + + // put the value on the science cell + psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "PTN.ROW.AMP", PS_META_REPLACE, "", amplitude); + } + + bool doPattern = false; + if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.ROW.SUBSET")) { + ESCAPE(false, "Unable to determine whether row pattern matching should be applied."); + } + if (!doPattern) continue; + + // switch to test threaded version + if (true) { + // I need to allocate a view here to be freed by the + // called function below. + pmFPAview *myView = pmFPAviewAlloc(0); // View for local processing + *myView = *view; + + // allocate a job, construct the arguments for this job + psThreadJob *job = psThreadJobAlloc("PPIMAGE_PATTERN_ROW_CELL"); + psArrayAdd(job->args, 1, config); + psArrayAdd(job->args, 1, input->fpa); + psArrayAdd(job->args, 1, chip); + psArrayAdd(job->args, 1, cell); + psArrayAdd(job->args, 1, myView); + psArrayAdd(job->args, 1, options); + if (!psThreadJobAddPending(job)) { + return false; + } + } else { + // bump the counter since it must be freed by the function below. + psMemIncrRefCounter (view); // View for local processing + if (!ppImageDetrendPatternApplyCell (config, input->fpa, chip, cell, view, options)) { + return false; + } + } + } + + // wait here for the threaded jobs to finish + // if no threads are allocated, this + if (!psThreadPoolWait(true, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to apply bias correction."); + return false; + } + + psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_ROW",PS_META_REPLACE,"PATTERN.ROW correction applied",true); + psFree(view); + + return true; +} + +bool ppImageDetrendPatternContinuityApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options) +{ + bool status; + + if (!options->doPatternContinuity) return true; + + assert(inputView->chip != -1); + assert(inputView->cell == -1); + assert(inputView->readout == -1); + + // see the comment for PATTERN.ROW; the same rules apply for PATTERN.CELL + + int numCells = chip->cells->n; // Number of cells + psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell? + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + pmHDU *hdu = pmHDUFromChip(chip); + if (psMetadataLookupBool(&status, hdu->header, "PTRN_CON")) { + psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction as it has already been done."); + return true; + } + + const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME"); + psLogMsg("ppImage", PS_LOG_INFO, "Performing pattern continuity correction for %s\n", chipName); + + for (int i = 0; i < chip->cells->n; i++) { + view->cell = i; + + pmCell *cell = chip->cells->data[i]; // Cell of interest + + if (cell->readouts->n > 1) { + psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction on video cell."); + continue; + } + + bool doPattern = false; + if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CONTINUITY.SUBSET")) { + ESCAPE(false, "Unable to determine whether row pattern matching should be applied."); + } + + if (doPattern) { + tweak->data.U8[i] = 0xFF; + } + } + + // Tweak the cells + if (!pmPatternContinuity(chip, tweak, options->patternCellBG, options->patternCellMean, + options->maskValue, options->darkMask,options->patternContinuityEdgeWidth)) { + psFree(tweak); + psFree(view); + return false; + } + psFree(tweak); + psFree(view); + + psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_CON",PS_META_REPLACE,"PATTERN.CONTINUITY correction applied",true); + return true; +} + +bool ppImageDetrendPatternCellApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options) +{ + bool status; + + if (!options->doPatternCell) return true; + + assert(inputView->chip != -1); + assert(inputView->cell == -1); + assert(inputView->readout == -1); + + int numCells = chip->cells->n; // Number of cells + psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell? + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + pmHDU *hdu = pmHDUFromChip(chip); + if (psMetadataLookupBool(&status, hdu->header, "PTRN_CEL")) { + psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell pattern correction as it has already been done."); + return true; + } + + const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME"); + psLogMsg("ppImage", PS_LOG_INFO, "Performing cell pattern correction for %s\n", chipName); + + for (int i = 0; i < chip->cells->n; i++) { + view->cell = i; + + pmCell *cell = chip->cells->data[i]; // Cell of interest + + if (cell->readouts->n > 1) { + psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell pattern correction on video cell."); + continue; + } + + bool doPattern = false; + if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CELL.SUBSET")) { + ESCAPE(false, "Unable to determine whether row pattern matching should be applied."); + } + if (doPattern) { + tweak->data.U8[i] = 0xFF; + } + } + + // Tweak the cells + if (!pmPatternCell(chip, tweak, options->patternCellBG, options->patternCellMean, + options->maskValue, options->darkMask)) { + psFree(tweak); + psFree(view); + return false; + } + psFree(tweak); + psFree(view); + + psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_CEL",PS_META_REPLACE,"PATTERN.CELL correction applied",true); + return true; +} + +bool ppImageDetrendPatternDeadCellsMask (pmChip *chip, psImageMaskType maskVal) { + + int numCells = chip->cells->n; // Number of cells + + // now mask bad cells + for (int i = 0; i < numCells; i++) { + pmCell *cell = chip->cells->data[i]; // Cell of interest + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + + psImage *mask = ro->mask; // mask of interest + int numCols = mask->numCols, numRows = mask->numRows; // Size of image + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskVal; + } + } + } + return true; +} + +bool ppImageDetrendPatternDeadCellsXY42 (pmChip *chip, psImageMaskType maskVal) { + + bool status; + + // extract the MEDIAN_CELL_BACKGROUND values to check for problems + int numCells = chip->cells->n; // Number of cells + psVector *cellBackground = psVectorAllocEmpty (numCells, PS_DATA_F32); + + for (int i = 0; i < numCells; i++) { + pmCell *cell = chip->cells->data[i];; + + if (!cell->process || !cell->file_exists || !cell->data_exists) continue; + + // select the HDU for this cell + pmHDU *cellHDU = pmHDUFromCell(cell); // HDU of interest + + psF32 BackValue = psMetadataLookupF32(&status, cellHDU->header, "BACK_VAL"); + if (!status) continue; + + psF32 BackError = psMetadataLookupF32(&status, cellHDU->header, "BACK_ERR"); + if (!status) continue; + + if (BackValue == 0.0) continue; + if (BackError == 0.0) continue; + + psVectorAppend (cellBackground, BackValue / BackError); + } + + if (cellBackground->n < 0.375*numCells) { + // Chip is bad, mask the whole thing + ppImageDetrendPatternDeadCellsMask (chip, maskVal); + psFree (cellBackground); + return true; + } + + // Second, calculate the median + psVectorSortInPlace (cellBackground); + int midPt = cellBackground->n / 2.0; + float median = cellBackground->n % 2 ? cellBackground->data.F32[midPt] : 0.5*(cellBackground->data.F32[midPt] + cellBackground->data.F32[midPt-1]); + psFree (cellBackground); + + // XXX hardwired number + if (median < 3.0) { + // Chip is bad, mask the whole thing + ppImageDetrendPatternDeadCellsMask (chip, maskVal); + } + psFree (cellBackground); + return true; +} + +// to apply the dead cell pattern, we need to transfer the pattern for this chip from the +// pmFPAfile for the pattern to the one for the image being processed. +bool ppImageDetrendPatternDeadCellsApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options) +{ + bool status; + + if (!options->doPatternDeadCells) return true; + + assert(inputView->chip != -1); + assert(inputView->cell == -1); + assert(inputView->readout == -1); + + const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME"); + psLogMsg("ppImage", PS_LOG_INFO, "Performing cell pattern correction for %s\n", chipName); + + pmHDU *hdu = pmHDUFromChip(chip); + if (psMetadataLookupBool(&status, hdu->header, "PTRN_DED")) { + psLogMsg("ppImage", PS_LOG_INFO, "Not performing dead cell pattern correction as it has already been done."); + return true; + } + + if (!strcmp (chipName, "XY40") || !strcmp (chipName, "XY42")) { + // special case : check for BACK_VAL / BACK_ERR > 3 or < 3 + psLogMsg("ppImage", PS_LOG_INFO, "Using special case for XY40 and XY42"); + ppImageDetrendPatternDeadCellsXY42 (chip, options->blankMask); + return true; + } + + pmFPAfile *pattern = psMetadataLookupPtr(&status, config->files, "PPIMAGE.PATTERN.DEAD.CELLS"); + if (!pattern) { + psLogMsg("ppImage", PS_LOG_INFO, "Pattern Dead Cells file not found, skipping"); + return true; + } + + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + pmChip *patternChip = pmFPAviewThisChip (view, pattern->fpa); + + // grab the pattern from the input pattern file chip + psImage *deadCellPattern = (psImage *) psMetadataLookupPtr (&status, patternChip->analysis, "PTN.DEAD.CELL"); + if (!deadCellPattern) { + psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No DEAD CELL pattern for chip, skipping\n"); + psFree (view); + return true; + } + + // copy the pattern pointer to the chip of the image being processed + psMetadataAddImage (chip->analysis, PS_LIST_TAIL, "PTN.DEAD.CELL", PS_META_REPLACE, "", deadCellPattern); + + // extract the MEDIAN_CELL_BACKGROUND values to check for problems + int numCells = chip->cells->n; // Number of cells + psVector *cellBackground = psVectorAllocEmpty (numCells, PS_DATA_F32); + + pmCell *cell = NULL; + while ((cell = pmFPAviewNextCell(view, chip->parent, 1)) != NULL) { + if (!cell->process || !cell->file_exists || !cell->data_exists) { + psVectorAppend (cellBackground, NAN); + continue; + } + + // select the HDU for this cell + pmHDU *cellHDU = pmHDUFromCell(cell); // HDU of interest + + psF32 value = psMetadataLookupF32(&status, cellHDU->header, "BACK_VAL"); + if (!status) { + psVectorAppend (cellBackground, NAN); + continue; + } + psVectorAppend (cellBackground, value); + } + + // match cellBackground pattern to registered patterns and mask as needed + if (!pmPatternDeadCells(chip, cellBackground, options->blankMask)) { + psFree(cellBackground); + psFree(view); + return false; + } + psFree(cellBackground); + psFree(view); + + psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_DED", PS_META_REPLACE, "PATTERN.DEADCELLS applied", true); + return true; +} + +static bool doPatternForView (bool *doit, const pmConfig *config, const pmChip *chip, const pmFPAview *view, const char *recipeName, const char *recipeValue) { + + *doit = false; + + psMetadataItem *doPattern; + + doPattern = pmConfigRecipeValueByView(config, recipeName, recipeValue, chip->parent, view); + + if (!doPattern) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine whether row pattern matching should be applied."); + return false; + } + if (doPattern->type == PS_DATA_BOOL) { + *doit = doPattern->data.B; + return true; + } + if (doPattern->type == PS_DATA_STRING) { + // expect a string of the form "000110001001" with at least view->cell entries + char *string = doPattern->data.str; + if (strlen(string) < view->cell) { + psError(PS_ERR_UNKNOWN, true, "error in PATTERN.ROW.SUBSET chip string (too few elements %d)", (int) strlen(string)); + return false; + } + switch (string[view->cell]) { + case '0': + case 'f': + case 'F': + case 'n': + case 'N': + *doit = false; + return true; + case '1': + case 't': + case 'T': + case 'y': + case 'Y': + *doit = true; + return true; + default: + psError(PS_ERR_UNKNOWN, true, "error in PATTERN.ROW.SUBSET chip string %s (unknown value %c))", string, string[view->cell]); + return false; + } + psAbort("imposible to reach here"); + } + psError(PS_ERR_UNKNOWN, true, "error in PATTERN.ROW.SUBSET : invalid data type"); + return false; +} + +// thread safety : +bool ppImageDetrendPatternApplyCell (pmConfig *config, pmFPA *fpa, pmChip *chip, pmCell *cell, pmFPAview *view, ppImageOptions *options) { + + // process each of the readouts + pmReadout *readout; // Readout from cell + while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) { + if (!readout->data_exists) { + continue; + } + + // Perform pattern correction + if (!pmPatternRow(readout, options->patternRowOrder, options->patternRowIter, + options->patternRowRej, options->patternRowThresh, options->patternRowMean, + options->patternRowStdev, options->maskValue, options->darkMask)) { + psFree (view); + return false; + } + } + psFree (view); // supplied view must be freeable + return true; +} + +psVector *ppImageDetrendPatternCellFailures(pmFPAfile *input, const pmFPAview *inputView) { + + bool status = false; + pmCell *cell = NULL; + + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + // extract the MEDIAN_CELL_BACKGROUND values to check for problems + psVector *cellBackground = psVectorAllocEmpty (64, PS_DATA_F32); + while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists || !cell->data_exists) { + psVectorAppend (cellBackground, NAN); + continue; + } + + // select the HDU for this cell + pmHDU *hdu = pmHDUFromCell(cell); // HDU of interest + + psF32 value = psMetadataLookupF32(&status, hdu->header, "BACK_VAL"); + psVectorAppend (cellBackground, value); + } + return cellBackground; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendReadout.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendReadout.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendReadout.c (revision 42655) @@ -0,0 +1,363 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageDetrendReadout(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + // psTimerStart("detrend.readout"); + + // construct a view for the detrend images (which have only one readout) + pmFPAview *detview = pmFPAviewAlloc(0); + *detview = *view; + detview->readout = 0; + + // find the currently selected readout + pmReadout *input = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + // Check that the gain is set (this is used by both pmReadoutGenerateMask and pmReadoutGenerateVariance) + { + float gain = psMetadataLookupF32(NULL, input->parent->concepts, "CELL.GAIN"); // Gain for cell + if (!isfinite(gain)) { + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, RECIPE_NAME); // Recipe + psAssert(recipe, "Should be there!"); + bool override = psMetadataLookupBool(NULL, recipe, "GAIN.OVERRIDE"); // Override the bad gain? + if (override) { + psWarning("CELL.GAIN is not set for readout (%d,%d,%d) --- setting to unity.", view->chip, view->cell, view->readout); + psMetadataItem *item = psMetadataLookup(input->parent->concepts, "CELL.GAIN"); // Gain item + psAssert(item, "Should be there!"); + item->data.F32 = 1.0; + + // for unity gain, there is no modification for the readnoise, note that it has (effectively) been updated + psMetadataRemoveKey(input->parent->concepts, "CELL.READNOISE.UPDATE"); + } else { + psWarning("CELL.GAIN is NAN for readout (%d,%d,%d), image will be masked.", view->chip, view->cell, view->readout); + } + } + } + // psLogMsg ("ppImage", 6, "check gain: %f sec\n", psTimerMark ("detrend.readout")); + + // Check to see if we're in a chip that contains video + // NOTE: this is an esoteric test for a GPC1 feature : this is irrelevant & will not work for other cameras + bool hasVideo = false; + { + // XXX test for GPC1? or CAN_HAVE_VIDEO in camera config? + bool status = false; + if (!input) goto done_video_check; + if (!input->parent) goto done_video_check; + if (!input->parent->parent) goto done_video_check; + if (!input->parent->parent->hdu) goto done_video_check; + if (!input->parent->parent->hdu->header) goto done_video_check; + char *ptr = psMetadataLookupStr(&status,input->parent->parent->hdu->header,"CELLMODE"); + if (status) { + psLogMsg ("ppImage.detrend", PS_LOG_DETAIL, "VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); + char *Vptr = strchr(ptr,'V'); + if (Vptr) { + hasVideo = options->hasVideo = true; + psLogMsg ("ppImage.detrend", PS_LOG_INFO, "VIDEO: %d %d %d\n",(int) options->hasVideo,(int) options->useVideoDark, (int) options->useVideoMask); + } + } + } + done_video_check: + // psLogMsg ("ppImage", 6, "check video: %f sec\n", psTimerMark ("detrend.readout")); + + // XXX for GPC2, there are some bizzare cell levels that need to be filtered out + // Here I measure the median background before overscan subtraction, but an alternative + // is to use the overscan-subtracted image. + // XX if (0) { + // XX psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN); + // XX psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + // XX psImageBackground (stats, NULL, input->image, NULL, 0xffff, rng); + // XX readoutBackground = stats->robustMedian; + // XX } + + // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine. + if (options->doMaskBuild) { + psImageMaskType satMask = options->doMaskSat ? options->satMask : 0; + psImageMaskType lowMask = options->doMaskLow ? options->lowMask : 0; + if (!pmReadoutGenerateMask(input, satMask, lowMask)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate a mask."); + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "generate mask: %f sec\n", psTimerMark ("detrend.readout")); + } + // apply the externally supplied mask to the input->mask pixels + if (options->doMask) { + pmReadout *mask; + if ((options->useVideoMask)&&(hasVideo)) { + mask = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.VIDEOMASK"); + } + else { + mask = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.MASK"); + } + pmMaskBadPixels(input, mask, options->maskValue); + // psLogMsg ("ppImage", 6, "apply mask: %f sec\n", psTimerMark ("detrend.readout")); + } + + if (options->doApplyBurntool) + { + // extern bool ppImageBurntoolApply(pmConfig *, ppImageOptions *, pmFPAview *, pmReadout *); + ppImageBurntoolApply(config, options, view, input); + // psLogMsg ("ppImage", 6, "apply burntool: %f sec\n", psTimerMark ("detrend.readout")); + } + + if (options->doMaskBurntool) { + if (options->doApplyBurntool) { + // build burntool mask from data input burntool table + ppImageBurntoolMaskFromTable(config,options,view,input); + } else { + // build burntool mask from data in the input image's fits extension + ppImageBurntoolMask(config,options,view,input); + } + // psLogMsg ("ppImage", 6, "apply burntool mask: %f sec\n", psTimerMark ("detrend.readout")); + } + + // Subtract the overscan + if (options->doOverscan) { + if (!pmOverscanSubtract (input, options->overscan)) { + psError(PS_ERR_UNKNOWN, false, "Unable to subtract overscan."); + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "subtract overscan: %f sec\n", psTimerMark ("detrend.readout")); + } + + // measure the overscan-subtracted readoutBackground here (or subtract the overscan value?) + // XXX this is the measurements and should be independent of the pattern masking + if (options->doPatternDeadCells) { + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + psImageBackground (stats, NULL, input->image, NULL, 0xffff, rng); + + // save this value somewhere + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "BACK_VAL", PS_META_REPLACE, "Median cell background", stats->robustMedian); + psMetadataAddF32(hdu->header, PS_LIST_TAIL, "BACK_ERR", PS_META_REPLACE, "Stdev of cell background", stats->robustStdev); + psFree (stats); + psFree (rng); + } + + // Non-linearity correction + if (options->doNonLin) { + if (!ppImageDetrendNonLinear(input,detview,config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to correct NonLinearity"); + psFree(detview); + return(false); + } + // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout")); + } + // New Non-linearity correction (exclusive of the above) + if (options->doNewNonLin) { + if (!ppImageDetrendNewNonLinear(input, detview, config)) { + psError(PS_ERR_UNKNOWN, false, "Unable to correct Non-Linearity with new version (2023)"); + psFree(detview); + return(false); + } + // psLogMsg ("ppImage", 6, "nonlinear correction: %f sec\n", psTimerMark ("detrend.readout")); + } + + // set up the dark and bias + pmCell *dark = NULL; // Multi-dark + pmReadout *oldDark = NULL; // Old-fashioned dark + pmReadout *bias = NULL; + if (options->doBias) { + bias = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.BIAS"); + } + if (options->doDark) { + bool mdok; // Status of MD lookup + psMetadata *recipe = psMetadataLookupPtr (&mdok, config->recipes, RECIPE_NAME); + assert(mdok && recipe); + + if ((options->useVideoDark)&&(hasVideo)) { + if (psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { + oldDark = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.VIDEODARK"); + } else { + dark = pmFPAfileThisCell(config->files, detview, "PPIMAGE.VIDEODARK"); + } + } + else { + if (psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { + oldDark = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.DARK"); + } else { + dark = pmFPAfileThisCell(config->files, detview, "PPIMAGE.DARK"); + } + } + } + + // Bias and temperature-independent-dark subtraction are merged. + if (options->doBias) { + if (!pmBiasSubtract(input, bias, oldDark, view)) { + psError(PS_ERR_UNKNOWN, false, "Unable to subtract bias."); + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "apply bias: %f sec\n", psTimerMark ("detrend.readout")); + } + + // Weight on the basis of pixel value needs to be done after the overscan has been subtracted + if (options->doVarianceBuild) { + // create the target mask and variance images + psImage *noiseImage = NULL; + if (options->doNoiseMap) { + // XXX convert the noiseMap image to a binned image + pmReadout *noiseMap = NULL; + noiseMap = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.NOISEMAP"); + noiseImage = psImageCopy (NULL, input->image, PS_TYPE_F32); + psImageInit (noiseImage, 0.0); + + // XXX this works, but is not really quite right: the model shoud include the + // offset information, we are not really getting exactly the right mapping from the + // original file. + // CZW 2012-03-21: I do not believe this is true anymore. In any case, this seems to be what + // ppImageReplaceBackground does to do sky subtraction. + psImageBinning *binning = psImageBinningAlloc(); + binning->nXruff = noiseMap->image->numCols; + binning->nYruff = noiseMap->image->numRows; + binning->nXfine = input->image->numCols; + binning->nYfine = input->image->numRows; + psImageBinningSetScale(binning, PS_IMAGE_BINNING_LEFT); + + psImageUnbin (noiseImage, noiseMap->image, binning); + psFree (binning); + } + + if (!pmReadoutGenerateVariance(input, noiseImage, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate a variance image."); + psFree(detview); + return false; + } + psFree (noiseImage); + // psLogMsg ("ppImage", 6, "generate variance: %f sec\n", psTimerMark ("detrend.readout")); + } + + if (options->doDark && dark) { + if (!pmDarkApply(input, dark, options->darkMask)) { + psError(PS_ERR_UNKNOWN, false, "Unable to subtract dark."); + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "apply dark: %f sec\n", psTimerMark ("detrend.readout")); + } + + if (options->doRemnance) { + if (!pmRemnance(input, options->maskValue, options->lowMask, + options->remnanceSize, options->remnanceThresh)) { + psError(PS_ERR_UNKNOWN, false, "Unable to mask remnance."); + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "mask remnance: %f sec\n", psTimerMark ("detrend.readout")); + } + + // Shutter correction + if (options->doShutter) { + pmReadout *shutter = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.SHUTTER"); + if (!pmShutterCorrectionApply(input, shutter, pmConfigMaskGet("FLAT", config))) { + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "shutter correction: %f sec\n", psTimerMark ("detrend.readout")); + } + + // Flat-field correction (no options used?) + if (options->doFlat) { + pmReadout *flat = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.FLAT"); + if (!pmFlatField(input, flat, options->flatMask)) { + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "apply flat: %f sec\n", psTimerMark ("detrend.readout")); + } + +/* // Pattern noise correction */ +/* if (options->doPattern) { */ +/* if (!pmPatternRow(input, options->patternOrder, options->patternIter, options->patternRej, */ +/* options->patternThresh, options->patternMean, options->patternStdev, */ +/* options->maskValue, options->darkMask)) { */ +/* psFree(detview); */ +/* return false; */ +/* } */ +/* } */ + + // Normalization by a single (known) constant + bool mdok; // Status of MD lookup + float norm = psMetadataLookupF32(&mdok, config->arguments, "NORMALIZATION"); + if (mdok && isfinite(norm) && norm != 1.0) { + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psString comment = NULL; // Comment to add + psStringAppend(&comment, "Normalization: %f", norm); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + + psBinaryOp(input->image, input->image, "*", psScalarAlloc(norm, PS_TYPE_F32)); + // psLogMsg ("ppImage", 6, "renormalize: %f sec\n", psTimerMark ("detrend.readout")); + } + +# if (1) + // Normalization by per-class values + psMetadata *normlist = psMetadataLookupMetadata(&mdok, config->arguments, "NORMALIZATION.TABLE"); + if (normlist) { + pmFPAfile *inputFile = psMetadataLookupPtr(&mdok, config->files, "PPIMAGE.INPUT"); + + // get the menu of class IDs + psMetadata *menu = psMetadataLookupMetadata(&mdok, inputFile->camera, "CLASSID"); + if (!menu) { + psError(PS_ERR_IO, false, "Unable to find CLASSID metadata in camera configuration"); + psFree(detview); + return false; + } + // get the rule for class_id for the desired class + const char *rule = psMetadataLookupStr(&mdok, menu, options->normClass); + if (!rule) { + psError(PS_ERR_IO, false, "Unable to find NORM.CLASS value %s in CLASSID in camera configuration", options->normClass); + psFree(detview); + return false; + } + // get the class_id from the rule + char *classID = pmFPAfileNameFromRule(rule, inputFile, view); + if (!classID) { + psError(PS_ERR_IO, false, "error converting CLASSID rule %s to name\n", rule); + psFree(detview); + return false; + } + + // get normalization from the class_id + float norm = psMetadataLookupF32 (&mdok, normlist, classID); + if (!mdok) { + psError(PS_ERR_IO, false, "failed to find class ID %s in normalization table\n", classID); + psFree(detview); + return false; + } + + pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest + psString comment = NULL; // Comment to add + psStringAppend(&comment, "Normalization: %f", norm); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + + // apply the normalization + psBinaryOp(input->image, input->image, "*", psScalarAlloc(norm, PS_TYPE_F32)); + + psFree (classID); + + // psLogMsg ("ppImage", 6, "renormalize by class: %f sec\n", psTimerMark ("detrend.readout")); + } +# endif + + if (options->doFringe) { + pmCell *fringe = pmFPAfileThisCell(config->files, detview, "PPIMAGE.FRINGE"); + if (!ppImageDetrendFringeMeasure(input, fringe, false, options)) { + psFree(detview); + return false; + } + // psLogMsg ("ppImage", 6, "measure fringe: %f sec\n", psTimerMark ("detrend.readout")); + } + + ppImageMemoryDump("detrend"); + + // psLogMsg ("ppImage", 5, "detrend readout: %f sec\n", psTimerMark ("detrend.readout")); + psFree(detview); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageDetrendRecord.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageDetrendRecord.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageDetrendRecord.c (revision 42655) @@ -0,0 +1,84 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +static void detrendRecord( + bool doThis, // Are we supposed to do this? + psMetadata *target, // Target metadata + const pmConfig *config, // Configuration + const pmFPAview *view, // View to cell + const char *filename, // Name of file + const char *name, // Name in header + const char *desc // Description + ) +{ + psAssert(config, "Need configuration"); + psAssert(view, "Need view to cell"); + psAssert(name, "Need file name"); + + if (!doThis) { + return; + } + + pmFPAfile *file = psMetadataLookupPtr(NULL, config->files, filename); // File of interest + psAssert(file, "Should be there"); + + if (target) { + psMetadataAddStr(target, PS_LIST_TAIL, name, 0, desc, file->filename); + } + + pmCell *input = pmFPAfileThisCell(config->files, view, "PPIMAGE.INPUT"); // File we're processing + psAssert(input, "Should be there"); + pmHDU *hdu = pmHDUGetHighest(input->parent->parent, input->parent, input); // HDU for cell + + // Strip off path and Nebulous bits + char *base = file->filename; // Base name of file + for (char *new = base; (new = strpbrk(base, "/:")); base = new + 1); // No action + + // We don't want multiple listings in the header saying the same thing, so make sure we haven't put the + // same entry there. Usually (if the detrend and the image have the same file level) there'll only end up + // being one entry. + psString regexp = NULL; // Regular expression + psStringAppend(®exp, "^%s$", name); + psMetadataIterator *iter = psMetadataIteratorAlloc(hdu->header, PS_LIST_HEAD, regexp); // Iterator for hdr + psFree(regexp); + psMetadataItem *item; // Item from iteration + bool found = false; + while (!found && (item = psMetadataGetAndIncrement(iter))) { + if (item->type == PS_DATA_STRING && strcmp(item->data.str, base) == 0) { + found = true; + } + } + psFree(iter); + if (!found) { + psMetadataAddStr(hdu->header, PS_LIST_TAIL, name, PS_META_DUPLICATE_OK, desc, base); + } + + return; +} + + +bool ppImageDetrendRecord(pmCell *cell, const pmConfig *config, const ppImageOptions *options, + const pmFPAview *view) +{ + psMetadata *detrend = psMetadataAlloc(); // Detrend information + psMetadataAddMetadata(cell->analysis, PS_LIST_TAIL, "DETREND", 0, "Detrend information", detrend); + + detrendRecord(options->doMask, detrend, config, view, "PPIMAGE.MASK", "DETREND.MASK", "Mask filename"); + detrendRecord(options->doNoiseMap, detrend, config, view, "PPIMAGE.NOISEMAP", "DETREND.NOISEMAP", "Noise Map filename"); + detrendRecord(options->doBias, detrend, config, view, "PPIMAGE.BIAS", "DETREND.BIAS", "Bias filename"); + detrendRecord(options->doDark, detrend, config, view, "PPIMAGE.DARK", "DETREND.DARK", "Dark filename"); + detrendRecord(options->doShutter, detrend, config, view, "PPIMAGE.SHUTTER", "DETREND.SHUTTER", "Shutter correction filename"); + detrendRecord(options->doFlat, detrend, config, view, "PPIMAGE.FLAT", "DETREND.FLAT", "Flat filename"); + detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE", "Fringe filename"); + + detrendRecord(options->doNonLin, detrend, config, view, "PPIMAGE.LINEARITY","DETREND.NONLIN", "Non-linearity table filename"); + detrendRecord(options->doNewNonLin, detrend, config, view, "PPIMAGE.NEWNONLIN","DETREND.NEWNONLIN","Non-linearity table filename (v2023)"); + + detrendRecord(options->doDark & options->useVideoDark, detrend, config, view, "PPIMAGE.VIDEODARK", "DETREND.VIDEODARK", "VideoDark filename"); + detrendRecord(options->doMask & options->useVideoMask, detrend, config, view, "PPIMAGE.VIDEOMASK", "DETREND.VIDEOMASK", "VideoMASK filename"); + psFree (detrend); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.c.in =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.c.in (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.c.in (revision 42655) @@ -0,0 +1,26 @@ +/* + * The line + { PPIMAGE_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "ppImageErrorCodes.h" + +void ppImageErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PPIMAGE_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PPIMAGE_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PPIMAGE_ERR_NERROR - PPIMAGE_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + psMemSetPersistent(tmp, true); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.dat =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.dat (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.dat (revision 42655) @@ -0,0 +1,12 @@ +# +# This file is used to generate ppImageErrorClasses.h + +BASE = 5000 First value we use; lower values belong to psLib +# these errors correspond to standard exit conditions +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data +UNKNOWN unspecified error, not defined +NO_PIXELS No good pixels in image Index: /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.h.in =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.h.in (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageErrorCodes.h.in (revision 42655) @@ -0,0 +1,18 @@ +#if !defined(PPIMAGE_ERROR_CODES_H) +#define PPIMAGE_ERROR_CODES_H +/* + * The line + * PPIMAGE_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PPIMAGE_ERR_BASE = 5000, + PPIMAGE_ERR_${ErrorCode}, + PPIMAGE_ERR_NERROR +} ppImageErrorCode; + +void ppImageErrorRegister(void); + +#endif Index: /branches/eam_branches/ppImage.20240412/src/ppImageFileCheck.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageFileCheck.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageFileCheck.c (revision 42655) @@ -0,0 +1,60 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +void ppImageFileCheck (pmConfig *config) { + + // add the output names to the output-type files + psMetadataItem *item = NULL; + psMetadataIterator *iter = psMetadataIteratorAlloc (config->files, PS_LIST_HEAD, NULL); + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + pmFPAfile *file = item->data.V; + pmFPA *fpa = file->fpa; + fprintf (stderr, "file %s\n", file->name); + if (!fpa) { + fprintf (stderr, " has no FPA\n"); + continue; + } + if (fpa->hdu) { + if (fpa->hdu->images) fprintf (stderr, " (%d,%d) images\n", -1, -1); + if (fpa->hdu->variances) fprintf (stderr, " (%d,%d) variances\n", -1, -1); + if (fpa->hdu->masks) fprintf (stderr, " (%d,%d) masks\n", -1, -1); + if (fpa->hdu->header) fprintf (stderr, " (%d,%d) header\n", -1, -1); + } else { + // fprintf (stderr, " has no fpa data (%d,%d)\n", -1, -1); + } + for (int i = 0; i < fpa->chips->n; i++) { + pmChip *chip = fpa->chips->data[i]; + if (chip->hdu) { + if (chip->hdu->images) fprintf (stderr, " (%d,%d) images\n", i, -1); + if (chip->hdu->variances) fprintf (stderr, " (%d,%d) variances\n", i, -1); + if (chip->hdu->masks) fprintf (stderr, " (%d,%d) masks\n", i, -1); + if (chip->hdu->header) fprintf (stderr, " (%d,%d) header\n", i, -1); + } else { + // fprintf (stderr, " has no chip data (%d,%d)\n", i, -1); + } + for (int j = 0; j < chip->cells->n; j++) { + pmCell *cell = chip->cells->data[j]; + if (cell->hdu) { + if (cell->hdu->images) fprintf (stderr, " (%d,%d) images\n", i, j); + if (cell->hdu->variances) fprintf (stderr, " (%d,%d) variances\n", i, j); + if (cell->hdu->masks) fprintf (stderr, " (%d,%d) masks\n", i, j); + if (cell->hdu->header) fprintf (stderr, " (%d,%d) header\n", i, j); + } else { + // fprintf (stderr, " has no cell data (%d,%d)\n", i, j); + } + for (int k = 0; k < cell->readouts->n; k++) { + pmReadout *readout = cell->readouts->data[k]; + if (readout) { + if (readout->image) fprintf (stderr, " (%d,%d,%d) image\n", i, j, k); + if (readout->variance) fprintf (stderr, " (%d,%d,%d) variance\n", i, j, k); + if (readout->mask) fprintf (stderr, " (%d,%d,%d) masks\n", i, j, k); + } + } + } + } + } + psFree (iter); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageLoop.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageLoop.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageLoop.c (revision 42655) @@ -0,0 +1,333 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +#define ESCAPE(MESSAGE) { \ + psError(PS_ERR_UNKNOWN, false, MESSAGE); \ + pmFPAfileIOChecks(config, view, PM_FPA_AFTER); \ + psFree(view); \ + psFree(stats); \ + pmFPAfileFreeSetStrict(false); \ + return false; \ +} + +bool ppImageLoop(pmConfig *config, ppImageOptions *options) +{ + psMetadata *stats = NULL; // Statistics to output + float timeDetrend = 0; // Amount of time spent in detrend + float timePhot = 0; // Amount of time spent in photometry + + if (options->doStats) { + stats = psMetadataAlloc(); + psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", 0, "No problems", 0); + } + + bool status; // Status of MD lookup + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT"); + if (!status) { + psErrorStackPrint(stderr, "Can't find input data!\n"); + ppImageCleanup(config, options); + exit(PS_EXIT_PROG_ERROR); + } + + pmConfigCamerasCull(config, NULL); + pmConfigRecipesCull(config, "PPIMAGE,PPSTATS,PSPHOT,MASKS,PSASTRO,JPEG"); + + pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest + pmHDU *lastHDU = NULL; // Last HDU that was updated + + // files associated with the science image + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for FPA"); + } + + pmChip *chip; // Chip from FPA + while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { + psLogMsg ("ppImageLoop", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (!chip->process || !chip->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for Chip"); + } + + // crosstalk measurement needs to be done on the entire chip at once, and before + // signal levels are modified by the detrending. If crosstalk measurement is + // requested, the read-level for the images is set to CHIP. + if (!ppImageMeasureCrosstalk(config, options, view)) { + ESCAPE("Unable to perform crosstalk correction"); + } + + // crosstalk correction needs to be done on the entire chip at once, and before + // signal levels are modified by the detrending. If crosstalk correction is + // requested, the read-level for the images is set to CHIP. + if (!ppImageCorrectCrosstalk(config, options, view)) { + ESCAPE("Unable to perform crosstalk correction"); + } + + psTimerStart(TIMER_DETREND); + pmCell *cell; // Cell from chip + while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + psLogMsg ("ppImageLoop", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!cell->process || !cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for Cell"); + } + + // Put version information into the header + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + ppImageVersionHeader(hdu->header); + lastHDU = hdu; + } + + // XXX for now, skip the video cells (cell->readouts->n > 1) + if (cell->readouts->n > 1) { + psWarning ("Skipping Video Cell for ppImageDetrendReadout"); + continue; + } + + // process each of the readouts + pmReadout *readout; // Readout from cell + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for Readout"); + } + if (!readout->data_exists) { + continue; + } + + // flip the image to match the native detector orientation (to match bias, flat, etc) + if (!ppImageParityFlip(config, options, view, true)) { + ESCAPE("Unable to detrend readout"); + } + + // XXX set the options->*Mask values here (after the mask images have been loaded + // and before any of the value are used) + if (!ppImageSetMaskBits(config, options)) { + ESCAPE("Unable to set bit masks"); + } + + // perform the detrend analysis + if (!ppImageDetrendReadout(config, options, view)) { + ESCAPE("Unable to detrend readout"); + } + + // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT + if (!ppImageDetrendFree (config, view)) { + ESCAPE("Unable to free detrend images"); + } + + // optionally measure CTE by examining the variance in a box + if (!ppImageCheckCTE (config, options, view)) { + ESCAPE("Unable to measure CTE"); + } + + if (!ppImageCheckNoise (config, options, view)) { + ESCAPE("Unable to generate noisemap"); + } + + // optionally degrade a MD image to 3pi exposure times + if (!ppImageAddNoise(config, options, view, input->fpa)){ + ESCAPE("Unable to degrade MD image to 3pi"); + } + + + if (!ppImageSquashNANs(config, options, view)) { + ESCAPE("Unable to squash NAN pixels"); + } + + } + + if (cell->data_exists) { + ppImageDetrendRecord(cell, config, options, view); + } + + // process each of the readouts + // XXX reset the view to the first readout? + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + if (!readout->data_exists) { + continue; + } + // flip the image to match the raw readout orientation + if (!ppImageParityFlip(config, options, view, false)) { + ESCAPE("Unable to detrend readout"); + } + } + + // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT + if (!ppImageDetrendFree (config, view)) { + ESCAPE("Unable to free detrend images"); + } + + } + // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT + if (!ppImageDetrendFree (config, view)) { + ESCAPE("Unable to free detrend images"); + } + + // Apply the fringe correction + if (!ppImageDetrendFringeApply(config, chip, view, options)) { + ESCAPE("Unable to defringe"); + } + + // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT + if (!ppImageFringeFree (config, view)) { + ESCAPE("Unable to free fringe images"); + } + + // Apply the pattern correction (only done if requested) + if (!ppImageDetrendPatternApply(config,chip,view,options)) { + ESCAPE("Problem applying pattern corrections"); + } + + // measure various pixel-based statistics for this image + if (!ppImagePixelStats(config, stats, options, view)) { + ESCAPE("Unable to measures pixel stats for image"); + } + + if (!ppImageMosaicChip(config, options, view, "PPIMAGE.CHIP", "PPIMAGE.OUTPUT")) { + ESCAPE("Unable to mosaic chip"); + } + + if (!ppImageAuxiliaryMask(config, view, options, stats)) { + ESCAPE("Unable to apply auxiliary mask"); + } + + timeDetrend += psTimerClear(TIMER_DETREND); + + // we perform photometry on the readouts of this chip in the output + + psTimerStart(TIMER_PHOT); + if (options->doPhotom) { + if (!ppImagePhotom(stats, config, view)) { + ESCAPE("error running photometry."); + } + } + timePhot += psTimerClear(TIMER_PHOT); + + // replace the masked pixels with the background level + if (options->doBG) { + if (!ppImageSubtractBackground(config, view, options)) { + ESCAPE("Unable to subtract background"); + } + } + + if (options->doMaskStats) { + //if (!ppImageMaskStats(config, view, options)) { + if (!ppImageMaskStats(config, view, stats)) { + ESCAPE("Unable to do Mask stats"); + } + } + + // these may be used by ppImageSubtractBackground. + // if these are defined as internal files, drop them here + status = true; + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files"); + psFree (view); + return false; + } + + // binning (used for display) must take place after the background is replaced, if desired + if ((options->Bin1FITS)||(options->Bin1JPEG)||(options->FPA1FITS)) { + if (!ppImageRebinChip(config, view, options, "PPIMAGE.BIN1")) { + ESCAPE("Unable to bin chip (level 1)."); + } + } + if ((options->Bin2FITS)||(options->Bin2JPEG)||(options->FPA2FITS)) { + if (!ppImageRebinChip(config, view, options, "PPIMAGE.BIN2")) { + ESCAPE("Unable to bin chip (level 2)."); + } + } + + if (options->doBackgroundContinuity) { + pmFPAfile *out = psMetadataLookupPtr(NULL, config->files, "PPIMAGE.BACKMDL"); + pmFPAfileCopyView(out->fpa,out->src,view); + } + + // Close cells (XXX shouldn't pmFPAfileClose iterate down as needed?) + view->cell = -1; + while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + ESCAPE("save failure for Cell"); + } + } + + // Close chip + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + ESCAPE("save failure for Chip"); + } + } + + // Do background model continuity updates + if (!ppImageMosaicBackground(config, options )) { + ESCAPE("failure in Background Mosaic"); + } + + // generate the full-scale FPA mosaic + if ((options->FPA1FITS)||(options->Bin1JPEG)) { + if (!ppImageMosaicFPA(config, options, "PPIMAGE.OUTPUT.FPA1", "PPIMAGE.BIN1")) { + ESCAPE("failure in FPA Mosaic (level 1)"); + } + } + if ((options->FPA2FITS)||(options->Bin2JPEG)) { + if (!ppImageMosaicFPA(config, options, "PPIMAGE.OUTPUT.FPA2", "PPIMAGE.BIN2")) { + ESCAPE("failure in FPA Mosaic (level 2)"); + } + } + + // we perform astrometry on all chips after sources have been detected + // this also performs the psastro file IO + if (options->doAstromChip || options->doAstromMosaic) { + if (!ppImageAstrom(config, stats)) { + ESCAPE("error running astrometry."); + } + } + + if (psTraceGetLevel("ppImage") >= 3) { + ppImageFileCheck(config); + } + + // Calculate summary statistics from FPA Metadata + if (!ppImageMetadataStats(config, stats, options)) { + ESCAPE("Unable to determine FPA-level metadata statistics."); + } + + // Output and Close FPA + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + ESCAPE("save failure for FPA"); + } + psFree(view); + + // Dump configuration + psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); + if (dump_file) { + if (!pmConfigDump(config, dump_file)) { + ESCAPE("Unable to dump configuration."); + } + } + + // Write out summary statistics + if (options->doStats) { + psMetadataAddF32(stats, PS_LIST_TAIL, "DT_DET", 0, "Time spent detrending (sec)", timeDetrend); + psMetadataAddF32(stats, PS_LIST_TAIL, "DT_PHOT", 0, "Time spent photometering (sec)", timePhot); + psMetadataAddF32(stats, PS_LIST_TAIL, "DT_TOTAL", 0, "Total time (sec)", psTimerMark(TIMER_TOTAL)); + if (!ppImageStatsOutput(config, stats, options)) { + ESCAPE("Unable to write statistics file."); + } + } + psFree (stats); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageMaskStats.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageMaskStats.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageMaskStats.c (revision 42655) @@ -0,0 +1,99 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + + +bool ppImageMaskStats(pmConfig *config, pmFPAview *view, psMetadata *stats) +{ + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + if (!stats) { + return true; + } + + bool status; + + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.CHIP"); + if (!status) { + psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined"); + return(false); + } + + psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, "PPIMAGE"); + + psU16 staticMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.STATIC"); + psU16 magicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.MAGIC"); + psU16 dynamicMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.DYNAMIC"); + psU16 advisoryMaskVal = psMetadataLookupU32(&status, recipe, "MASKSTAT.ADVISORY"); + + psS32 Npix_valid = 0; + psS32 Npix_static = 0; + psS32 Npix_magic = 0; + psS32 Npix_dynamic = 0; + psS32 Npix_advisory = 0; + + pmChip *chip = pmFPAviewThisChip(view, input->fpa); // Chip of interest + if (!chip) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find chip"); + return(false); + } + if (chip->cells->n == 0) { + psWarning("Chip has no cells"); + return(true); + } + if (chip->cells->n > 1) { + psWarning("Chip has %ld cells; only the first will be processed", chip->cells->n); + } + pmCell *cell = chip->cells->data[0]; // Cell of interest + if (!cell || !cell->process || !cell->file_exists) { + // Nothing to process + return(true); + } + if (cell->readouts->n == 0) { + psWarning("Cell has no readouts"); + return(true); + } + if (cell->readouts->n > 1) { + psWarning("Cell has %ld readouts; only the first will be processed", cell->readouts->n); + } + pmReadout *readout = cell->readouts->data[0]; // Readout of interest; + if (!readout || !readout->data_exists) { + // Nothing to process + return(true); + } + + psImage *mask = readout->mask; // Mask of interest; + psWarning("In ppImageMaskStats: %d %ld\n",Npix_valid, (long) mask); + + if (!pmSingleImageMaskStats(mask,&Npix_valid,&Npix_static,&Npix_magic, + &Npix_dynamic,&Npix_advisory, + staticMaskVal,magicMaskVal, + dynamicMaskVal,advisoryMaskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate masks for readout."); + return(false); + } + psMetadataAddS32(stats, PS_LIST_TAIL,"MASKFRAC_NPIX", 0, + "Number of valid pixels", Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_STATIC", 0, + "Fraction of pixels statically masked", (float) Npix_static / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_DYNAMIC", 0, + "Fraction of pixels dynamically masked", (float) Npix_dynamic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_MAGIC", 0, + "Fraction of pixels magically masked", (float) Npix_magic / Npix_valid); + psMetadataAddF32(stats,PS_LIST_TAIL, "MASKFRAC_ADVISORY", 0, + "Fraction of pixels masked as an advisory", (float) Npix_advisory / Npix_valid); + psWarning("In ppImageMaskStats: %d %f %f %f %f\n",Npix_valid, (float) Npix_static / Npix_valid, + (float) Npix_dynamic / Npix_valid, (float) Npix_magic / Npix_valid, + (float) Npix_advisory / Npix_valid); + + if ((Npix_valid == 0)||(Npix_static + Npix_dynamic >= Npix_valid)) { + if (psMetadataLookupS32(NULL, stats, "QUALITY") == 0) { + psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "No good pixels in image.", PPIMAGE_ERR_NO_PIXELS); + } + } + + return(true); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageMeasureCrosstalk.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageMeasureCrosstalk.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageMeasureCrosstalk.c (revision 42655) @@ -0,0 +1,187 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +#define ESCAPE(MESSAGE) { \ + psError(PS_ERR_UNKNOWN, false, MESSAGE); \ + psFree(view); \ + return false; \ + } + +// For the moment, this implementation is VERY GPC-specific + +bool ppImageMeasureCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + bool status; + const float CUTOFF = 60000; + const int MIN_MATCHED_PIXELS = 500; // require a significant area of high signal + char name[64]; + + if (!options->doCrosstalkMeasure) return true; + + psTimerStart("crosstalk"); + + // confirm that this camera is GPC1 + + // find the currently selected chip + pmChip *chip = pmFPAfileThisChip(config->files, view, "PPIMAGE.INPUT"); + pmFPA *fpa = chip->parent; + if (chip->cells->n != 64) return true; + + // for each cell (xyNM), we want to find the pixels with values above a cutoff (40k, 60k, ?) + // for all of the other cells in the row (xyJM), we want to measure the (robust) median flux in the pixels + // corresponding to the high pixels in xyNM, and compare with the median flux for the cell + + psArray *vectorSet = psArrayAlloc(8); + psArray *cellRows = psArrayAlloc(8); + psArray *imageRows = psArrayAlloc(8); + for (int i = 0; i < 8; i++) { + psArray *cellRow = psArrayAlloc(8); + cellRows->data[i] = cellRow; + psArray *imageRow = psArrayAlloc(8); + imageRows->data[i] = imageRow; + psVector *vector = psVectorAllocEmpty(100, PS_TYPE_F32); + vectorSet->data[i] = vector; + } + + // assign the cells to arrays by row and column + for (int i = 0; i < chip->cells->n; i++) { + + pmCell *cell = chip->cells->data[i]; + + // skip the video cells and empty cells (cell->readouts->n != 1) + if (cell->readouts->n != 1) { + psWarning ("Skipping Empty and Video Cell for ppImageCrosstalk"); + continue; + } + + // place the image from this cell in the 2D array: + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + int row = cellName[3] - '0'; + int col = cellName[2] - '0'; + assert (row >= 0 && row < 8); + assert (col >= 0 && col < 8); + + pmReadout *readout = cell->readouts->data[0]; + psImage *image = readout->image; + + psArray *cellRow = cellRows->data[row]; + cellRow->data[col] = psMemIncrRefCounter(cell); + + psArray *imageRow = imageRows->data[row]; + imageRow->data[col] = psMemIncrRefCounter(image); + } + + psVector *sample = NULL; + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN); + + // measure the background for each cell + for (int row = 0; row < cellRows->n; row++) { + psArray *cellRow = cellRows->data[row]; + psArray *imageRow = imageRows->data[row]; + for (int col = 0; col < cellRow->n; col++) { + pmCell *cell = cellRow->data[col]; + if (!cell) continue; + psImage *image = imageRow->data[col]; + if (!image) continue; // XXX assert on this? + + psStatsInit (stats); + if (!psImageBackground (stats, &sample, image, NULL, 0, rng)) continue; + psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "XTALK.REF", PS_META_REPLACE, "crosstalk measurement", stats->robustMedian); + // fprintf (stderr, "xtalk.ref (xy%d%d) : %f\n", col, row, stats->robustMedian); + } + } + + for (int row = 0; row < cellRows->n; row++) { + psArray *cellRow = cellRows->data[row]; + psArray *imageRow = imageRows->data[row]; + for (int col = 0; col < cellRow->n; col++) { + psImage *image = imageRow->data[col]; + if (!image) continue; + + // initialize the storage vectors + for (int i = 0; i < 8; i++) { + psVector *vector = vectorSet->data[i]; + vector->n = 0; + } + + int nBright = 0; + + // generate a vector for each of the other cells in this row + // containing only the pixels for which this cell is > CUTOFF + for (int iy = 0; iy < image->numRows; iy++) { + for (int ix = 0; ix < image->numCols; ix++) { + + if (image->data.F32[iy][ix] < CUTOFF) continue; + nBright ++; + + // this is a pixel of interest in the target cell; extract the matched pixels + + for (int i = 0; i < 8; i++) { + if (i == col) continue; + + psImage *matchImage = imageRow->data[i]; + if (!matchImage) continue; + + psVector *vector = vectorSet->data[i]; + psVectorAppend (vector, matchImage->data.F32[iy][ix]); + } + } + } + // fprintf (stderr, "nBright for xy%d%d : %d\n", col, row, nBright); + if (nBright < MIN_MATCHED_PIXELS) continue; + + // pmCell *refCell = cellRow->data[col]; + // float refBackground = psMetadataLookupF32 (&status, refCell->analysis, "XTALK.REF"); + // float swing = CUTOFF - refBackground; + + // now we need to measure the median for these vectors + for (int i = 0; i < 8; i++) { + if (i == col) continue; + psVector *vector = vectorSet->data[i]; + if (vector->n < MIN_MATCHED_PIXELS) continue; + + pmCell *cell = cellRow->data[i]; + if (!cell) continue; // XXX assert on this? + + psStatsInit (stats); + if (!psVectorStats (stats, vector, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); + return false; + } + + float background = psMetadataLookupF32 (&status, cell->analysis, "XTALK.REF"); + float xtalk = (stats->robustMedian - background) / CUTOFF; + + // Put version information into the header + pmHDU *hdu = pmHDUGetHighest(fpa, chip, cell); + if (hdu) { + // add this to the PHU header + snprintf (name, 64, "XT_%d%d_%d%d", col, row, i, row); + psMetadataAddF32 (hdu->header, PS_LIST_TAIL, name, PS_META_REPLACE, "crosstalk measurement", xtalk); + } + + // keyword for resulting value: + snprintf (name, 64, "XTALK_%d%d", i, row); + + psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, name, PS_META_REPLACE, "crosstalk measurement", xtalk); + // fprintf (stderr, "xtalk (xy%d%d on xy%d%d) : %f -> delta is %f, slope if %e\n", col, row, i, row, stats->robustMedian, stats->robustMedian - background, xtalk); + } + } + } + psLogMsg ("ppImage", 3, "crosstalk measurement: %f sec\n", psTimerMark ("crosstalk")); + + psFree (vectorSet); + psFree (cellRows); + psFree (imageRows); + psFree (stats); + psFree (rng); + psFree (sample); + + // need to free all sorts of things here.... + return true; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImageMemory.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageMemory.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageMemory.c (revision 42655) @@ -0,0 +1,38 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppImage.h" + +//#define TESTING + +void ppImageMemoryDump(const char *description) +{ +#ifndef TESTING + return; +#else + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true); + + static int num = 0; + psString filename = NULL; + psStringAppend(&filename, "%s_%d.txt", description, num++); + FILE *fp = fopen(filename, "w"); + psFree(filename); + + for (int i = 0; i < numLeaks; i++) { + psMemBlock *memBlock = leaks[i]; + fprintf(fp, "%ld : %s at (%s:%d) ID: %lu Ref: %lu\n", + (unsigned long)memBlock->userMemorySize, memBlock->func, memBlock->file, + (int)memBlock->lineno, (unsigned long)memBlock->id, memBlock->refCounter); + } + psFree(leaks); + + fclose(fp); + + return; +#endif +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageMetadataStats.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageMetadataStats.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageMetadataStats.c (revision 42655) @@ -0,0 +1,46 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// calculate stats from headers and concepts +bool ppImageMetadataStats(pmConfig *config, psMetadata *stats, const ppImageOptions *options) +{ + bool mdok; // Status of MD lookup + + if (!options->doStats) { + return true; + } + + // Extract statistics from the last output fpa + pmFPAfile *outImage = psMetadataLookupPtr(&mdok, config->files, "PPIMAGE.OUTPUT"); + pmFPAfile *outPhotom = psMetadataLookupPtr(&mdok, config->files, "PSPHOT.OUTPUT"); + pmFPAfile *outAstrom = psMetadataLookupPtr(&mdok, config->files, "PSASTRO.OUTPUT"); + + if (!outImage && !outPhotom && !outAstrom) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Unable to find any output file (PPIMAGE.OUTPUT, PSPHOT.OUTPUT, PSASTRO.OUTPUT)."); + return false; + } + + // get the latest output product available + pmFPAfile *output = outAstrom; + if (!output) { + output = outPhotom; + } + if (!output) { + output = outImage; + } + + // extract stats for the complete fpa + pmFPAview *view = pmFPAviewAlloc(0); + if (!ppStatsMetadata(stats, output->fpa, view, options->maskValue, config)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate stats for image."); + psFree(view); + return false; + } + + psFree(view); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageMosaic.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageMosaic.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageMosaic.c (revision 42655) @@ -0,0 +1,107 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageMosaicChip(pmConfig *config, const ppImageOptions *options, const pmFPAview *view, + const char *outFile, const char *inFile) +{ + bool status; // Status of MD lookup + + // psTimerStart("mosaic.chip"); + + pmFPAfile *in = psMetadataLookupPtr(&status, config->files, inFile); // Input file + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + + pmFPAfile *out = psMetadataLookupPtr(&status, config->files, outFile); // Output file + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + + pmChip *outChip = pmFPAviewThisChip(view, out->fpa); + pmChip *inChip = pmFPAviewThisChip(view, in->fpa); + if (!outChip->hdu && !outChip->parent->hdu) { + pmFPAAddSourceFromView(out->fpa, view, out->format); + } + + // If the input is a dark (normalising and mosaicking) then it looks like a video cell and will be ignored + // by pmChipMosaic. We therefore hack the structure so it doesn't look like a video cell. + psVector *darkNumbers = NULL; // Number of dark readouts for each cell + if (psMetadataLookupBool(&status, config->arguments, "INPUT_IS_DARK")) { + darkNumbers = psVectorAlloc(inChip->cells->n, PS_TYPE_S32); + psVectorInit(darkNumbers, 0); + for (int i = 0; i < inChip->cells->n; i++) { + pmCell *cell = inChip->cells->data[i]; + if (!cell) { + continue; + } + darkNumbers->data.S32[i] = cell->readouts->n; + cell->readouts->n = 1; + } + } + + psTrace("pmChipMosaic", 5, "mosaic chip %s to %s (xbin,ybin: %d,%d to %d,%d)\n", + in->name, out->name, in->xBin, in->yBin, out->xBin, out->yBin); + + // Mosaic the chip, making a deep copy. This has the side effect of making the output + // image products pure trimmed images, but also increases the memory footprint. + status = pmChipMosaic(outChip, inChip, true, options->blankMask); + + // Restore dark structure + if (darkNumbers) { + for (int i = 0; i < inChip->cells->n; i++) { + pmCell *cell = inChip->cells->data[i]; + if (!cell) { + continue; + } + cell->readouts->n = darkNumbers->data.S32[i]; + } + psFree(darkNumbers); + } + + // psLogMsg ("ppImage", 5, "mosaic chip: %f sec\n", psTimerMark ("mosaic.chip")); + + return status; +} + +bool ppImageMosaicFPA (pmConfig *config, const ppImageOptions *options, const char *outFile, + const char *inFile) +{ + bool status; // Status of MD lookup + + pmFPAfile *in = psMetadataLookupPtr(&status, config->files, inFile); // Input file + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + + pmFPAfile *out = psMetadataLookupPtr(&status, config->files, outFile); // Output file + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + + // XXX test printing of all concepts + #if 0 + for (int i = 0; i < in->fpa->chips->n; i++) { + pmChip *chip = in->fpa->chips->data[i]; + for (int j = 0; j < chip->cells->n; j++) { + pmCell *cell = chip->cells->data[j]; + psMetadataPrint(stdout, cell->concepts, 2); + } + } + #endif + + pmFPAview *view = pmFPAviewAlloc(0); + pmFPAAddSourceFromView(out->fpa, view, out->format); + psFree(view); + + psTrace ("pmFPAMosaic", 5, "mosaic fpa %s to %s (xbin,ybin: %d,%d to %d,%d)\n", + in->name, out->name, in->xBin, in->yBin, out->xBin, out->yBin); + return pmFPAMosaic(out->fpa, in->fpa, false, options->blankMask); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageMosaicBackground.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageMosaicBackground.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageMosaicBackground.c (revision 42655) @@ -0,0 +1,77 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + + +bool ppImageMosaicBackground(pmConfig *config, const ppImageOptions *options) { + assert(config); + assert(options); + + if (options->doBackgroundContinuity) { + bool status; + pmFPAfile *in = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT"); + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + pmFPAfile *out = psMetadataLookupPtr(&status, config->files, "PPIMAGE.BACKMDL"); + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + //pmReadout *model = pmFPAGenerateReadout(config, view, psphotGetFilerule("PSPHOT.BACKMDL"), inFPA, binning, index); + + pmFPAview *view = pmFPAviewAlloc(0); + // pmFPAAddSourceFromView(out->fpa, view, out->format); + // psFree(view); + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + // ESCAPE("load failure for Chip"); + } + + pmChip *chip; + + if (!pmPatternContinuityBackground(in,out,options->patternCellBG,options->patternCellMean, + options->maskValue, options->darkMask, options->patternContinuityEdgeWidth)) { + // Free things? + psFree(view); + return(false); + } + + // Write out output models + psFree(view); + view = pmFPAviewAlloc(0); + while ((chip = pmFPAviewNextChip(view, out->fpa, 1)) != NULL) { + pmCell *cell; + while ((cell = pmFPAviewNextCell(view, out->fpa, 1)) != NULL) { + pmReadout *readout; + while ((readout = pmFPAviewNextReadout(view, out->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psError(PS_ERR_UNKNOWN, false, "I/O failure in ppImageMosaicBackground"); + psFree(view); + return(false); + } + psFree(view); + } + + return(true); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageOptions.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageOptions.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageOptions.c (revision 42655) @@ -0,0 +1,492 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +static void imageOptionsFree(ppImageOptions *options) +{ + psFree(options->overscan); + // psFree(options->nonLinearData); + // psFree(options->nonLinearSource); + psFree(options->auxVideoMask); +} + +ppImageOptions *ppImageOptionsAlloc(void) +{ + ppImageOptions *options = psAlloc(sizeof(ppImageOptions)); + psMemSetDeallocator(options, (psFreeFunc)imageOptionsFree); + + // actions which ppImage should perform + options->doMaskBuild = false; // Build internal mask + options->doMaskSat = false; // mask saturated pixels + options->doMaskLow = false; // mask low pixels + options->doMaskBurntool = false; // mask potential burntool trails + options->doApplyBurntool = false; // apply burntool correction + options->doVarianceBuild = false; // Build internal variance + options->doMask = false; // Mask bad pixels + options->doAuxMask = false; // apply auxillary mask + options->doNonLin = false; // Non-linearity correction + options->doNewNonLin = false; // New Non-linearity correction (v2023) + options->doOverscan = false; // Overscan subtraction + options->doNoiseMap = false; // Apply Read Noise Map + options->doBias = false; // Bias subtraction + options->doDark = false; // Dark subtraction + options->doRemnance = false; // Remnance masking + options->doShutter = false; // Shutter correction + options->doFlat = false; // Flat-field normalisation + options->doPatternRow = false; // Row pattern correction + options->doPatternCell = false; // Cell pattern correction + options->doPatternContinuity = false; // Cell continuity correction + options->doBackgroundContinuity = false; // Chip level background continuity correction + options->doFringe = false; // Fringe subtraction + options->doPhotom = false; // Source identification and photometry + options->doAstromChip = false; // Astrometry (per-chip) + options->doAstromMosaic = false; // Astrometry (full-mosaic) + options->doStats = false; // Measure and save image statistics + options->checkCTE = false; // Measure pixel-based variance + options->checkNoise = false; // Measure cell-level variances. + options->squashNANs = false; // Measure cell-level variances. + options->applyParity = false; // Apply Cell parities + options->doMaskStats = false; // Calculate mask fractions + options->addNoise = false; //Degrade an MD image to a 3pi image + options->hasVideo = false; // Determine if this OTA has a video cell + options->useVideoDark = false; // Use video dark if we can? + options->useVideoMask = false; // Use video mask if we can? + options->doApplyPixelZero = true; // option for zero'ing pixels under masks + + // output files requested + options->BaseFITS = false; // create output image + options->BaseMaskFITS = false; // create output mask image + options->BaseVarianceFITS = false; // create output variance image + + options->ChipFITS = false; // create output chip-mosaic image + options->ChipMaskFITS = false; // create output chip-mosaic mask image + options->ChipVarianceFITS = false; // create output chip-mosaic variance image + + options->FPA1FITS = false; // create fpa-mosaic binned image (scale 1) + options->FPA2FITS = false; // create fpa-mosaic binned image (scale 2) + options->Bin1FITS = false; // create binned image (scale 1) + options->Bin2FITS = false; // create binned image (scale 2) + options->Bin1JPEG = false; // create jpeg of binned image (scale 1) + options->Bin2JPEG = false; // create jpeg of binned image (scale 2) + + // default flags for various activities + options->maskValue = 0x00; // Default mask value (used to skip / ignore pixels) + options->satMask = 0x00; // Saturated pixels (supplied to pmReadoutGenerateMask) + options->lowMask = 0x00; // out-of-bounds (low) pixels (supplied to pmReadoutGenerateMask) + options->flatMask = 0x00; // Bad flat pixels (supplied to pmFlatField) + options->darkMask = 0x00; // Bad dark pixels (supplied to pmDarkApply) + options->blankMask = 0x00; // Blank (no data, cell gap) pixels (supplied to pmChipMosaic, pmFPAMosaic) + options->markValue = 0x00; // A safe bit for internal marking + options->burntoolMask = 0x00; // Suspect pixels that fall where a burntool trail is expected. + options->burntoolTrails = 0x07; // Which types of burntool areas to mask. + // crosstalk options + options->doCrosstalkMeasure = false; // measure crosstalk + options->doCrosstalkCorrect = false; // correct crosstalk + + // Non-linearity default options + options->nonLinearType = 0; // Type of non-linearity data (vector, string or metadata) + options->nonLinearData = NULL; // The non-linearity data + options->nonLinearSource = NULL; // If the non-linearity data is a menu, this provides the key + + // Overscan defaults + options->overscan = NULL; // Overscan options + + // binning parameters + options->xBin1 = 16; // x-binning, scale 1 + options->yBin1 = 16; // y-binning, scale 1 + options->xBin2 = 16; // x-binning, scale 2 + options->yBin2 = 16; // y-binning, scale 2 + + // Fringe defaults + options->fringeRej = NAN; // Fringe rejection limit + options->fringeIter = 0; // Fringe iterations + options->fringeKeep = 1.0; // Fringe keep fraction + + // Pattern correction values + + options->patternRowOrder = 0; // Polynomial order + options->patternRowIter = 0; // Clipping iterations + options->patternRowRej = NAN; // Clipping rejection threshold + options->patternRowThresh = NAN; // Threshold for ignoring pixels (pixels with counts > median + thresh * stdev are ignored) + options->patternRowMean = PS_STAT_NONE; // Statistic for mean + options->patternRowStdev = PS_STAT_NONE; // Statistic for standard deviation + options->patternCellBG = PS_STAT_NONE; // Statistic for background + options->patternCellMean = PS_STAT_NONE; // Statistic for mean + + // Remnance values + options->remnanceSize = 30; // Size for remnance detection + options->remnanceThresh = 25.0; // Threshold for remnance detection + + // per-class normalization source + options->normClass = NULL; // per-class normalizations refer to this class + + options->auxVideoMask = NULL; // auxillary video mask file name + + return options; +} + +ppImageOptions *ppImageOptionsParse(pmConfig *config) +{ + bool status; + ppImageOptions *options = ppImageOptionsAlloc (); + + // select the recipe for this analysis + bool mdStatus = false; // Result of MD lookup + psMetadata *recipe = psMetadataLookupMetadata(&mdStatus, config->recipes, RECIPE_NAME); + if (! mdStatus || !recipe) { + psLogMsg("ppImage", PS_LOG_ERROR, "Can't find recipe %s in the RECIPES.\n", RECIPE_NAME); + exit(EXIT_FAILURE); + } + psMetadata *format = config->format; + + // Non-linearity recipe options + if (psMetadataLookupBool(NULL, recipe, "NONLIN")) { + psMetadataItem *dataItem = psMetadataLookup(recipe, "NONLIN.DATA"); + if (! dataItem) { + psLogMsg("ppImage", PS_LOG_ERROR, "Non-linearity correction desired, but unable to find NONLIN.DATA in recipe %s.", RECIPE_NAME); + exit(EXIT_FAILURE); + } + + options->doNonLin = true; + options->nonLinearType = dataItem->type; + options->nonLinearData = dataItem; + + switch (dataItem->type) { + // No immediate action required + case PS_DATA_VECTOR: + case PS_DATA_STRING: + break; + + // This is a menu; we need the key + case PS_DATA_METADATA: + options->nonLinearSource = psMetadataLookupStr(&status, recipe, "NONLIN.SOURCE"); + if (! status || ! options->nonLinearSource) { + psLogMsg("ppImage", PS_LOG_ERROR, "Non-linearity correction desired, but unable to find NONLIN.SOURCE in recipe %s.", RECIPE_NAME); + exit(EXIT_FAILURE); + } + break; + default: + psLogMsg("ppImage", PS_LOG_ERROR, "Non-linearity correction desired, but NONLIN.DATA is of invalid type in recipe %s.", RECIPE_NAME); + exit(EXIT_FAILURE); + } + } + + // XXX PAP: The overscan stuff needs to be updated following the reworked API + + // New Non-linearity (v 2023) recipe options + // non-linearity corrections are loaded from a file defined in the detrend system or on the command-line + if (psMetadataLookupBool(NULL, recipe, "NEWNONLIN")) { + options->doNewNonLin = true; + } + + // XXX PAP: The overscan stuff needs to be updated following the reworked API + + // Overscan recipe options + // XXX EAM : we should abort on invalid options. default options? + if (psMetadataLookupBool(NULL, recipe, "OVERSCAN")) { + options->doOverscan = true; + + // Do the overscan as a single value? + bool overscanSingle = psMetadataLookupBool(NULL, recipe, "OVERSCAN.SINGLE"); + + // How do we fit it? + pmFit overscanFit = PM_FIT_NONE; // Fit type for overscan + int overscanOrder = 0; // Order for overscan fit + psString fit = psMetadataLookupStr(NULL, recipe, "OVERSCAN.FIT"); + if (! strcasecmp(fit, "POLYNOMIAL")) { + overscanFit = PM_FIT_POLY_ORD; + overscanOrder = psMetadataLookupS32(NULL, recipe, "OVERSCAN.ORDER"); + } else if (! strcasecmp(fit, "CHEBYSHEV")) { + overscanFit = PM_FIT_POLY_CHEBY; + overscanOrder = psMetadataLookupS32(NULL, recipe, "OVERSCAN.ORDER"); + } else if (! strcasecmp(fit, "SPLINE")) { + overscanFit = PM_FIT_SPLINE; + } else if (strcasecmp(fit, "NONE")) { + psLogMsg(__func__, PS_LOG_WARN, + "OVERSCAN.FIT (%s) in recipe %s is not one of NONE, POLYNOMIAL, or SPLINE", + fit, RECIPE_NAME); + exit(EXIT_FAILURE); + } + + // What method do we use to measure the overscan statistics? + // XXX allow user to specify psStats types by name + psStats *overscanStats = NULL; // Statistics for overscan + psString stat = psMetadataLookupStr(NULL, recipe, "OVERSCAN.STAT"); + if (! strcasecmp(stat, "MEAN")) { + // overscanStats = psStatsAlloc(PS_STAT_SAMPLE_MEAN); + overscanStats = psStatsAlloc(PS_STAT_SAMPLE_MEAN); + } else if (! strcasecmp(stat, "MEDIAN")) { + overscanStats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); + } else { + psErrorStackPrint(stderr, "OVERSCAN.STAT (%s) in recipe %s is not one of MEAN or MEDIAN", + stat, RECIPE_NAME); + exit(EXIT_FAILURE); + } + + bool mdok; + int boxcar = psMetadataLookupS32(NULL, recipe, "OVERSCAN.BOXCAR"); + float gauss = psMetadataLookupF32(NULL, recipe, "OVERSCAN.GAUSS"); + float minValid = psMetadataLookupF32(&mdok, recipe, "OVERSCAN.MIN.VALID"); + if (!mdok) { minValid = 0.0; } + + float maxValid = psMetadataLookupF32(&mdok, recipe, "OVERSCAN.MAX.VALID"); + if (!mdok) { maxValid = (float) 0x10000; } + + // Fill in the options + options->overscan = pmOverscanOptionsAlloc(overscanSingle, overscanFit, overscanOrder, + overscanStats, boxcar, gauss); + + options->overscan->constant = psMetadataLookupBool(NULL, recipe, "OVERSCAN.CONSTANT"); + options->overscan->value = psMetadataLookupF32(NULL, recipe, "OVERSCAN.VALUE"); + options->overscan->minValid = minValid; + options->overscan->maxValid = maxValid; + options->overscan->maskVal = 0x0001; + + psFree(overscanStats); + } + + // for these images, even if not required otherwise + options->doMaskBuild = psMetadataLookupBool(NULL, recipe, "MASK.BUILD"); + options->doMaskSat = psMetadataLookupBool(NULL, recipe, "MASK.SATURATED"); + options->doMaskLow = psMetadataLookupBool(NULL, recipe, "MASK.LOW"); + options->doMaskBurntool = psMetadataLookupBool(NULL, recipe, "MASK.BURNTOOL"); + options->doApplyBurntool = psMetadataLookupBool(NULL, recipe, "APPLY.BURNTOOL"); + //TdB: read in switch for zero'ing pixels under masks + options->doApplyPixelZero = psMetadataLookupBool(NULL, recipe, "APPLY.PIXELZERO"); + options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD"); + options->doAuxMask = psMetadataLookupBool(NULL, recipe, "MASK.AUXMASK"); + if (options->doAuxMask) { + // if we are applying an auxiliary mask we can optionally apply another + // mask to video cells only. + psString auxVideoMask = psMetadataLookupStr(NULL, recipe, "AUX.VIDEO.MASK"); + // save the value if defined and not the value "NULL" + if (auxVideoMask && strcmp(auxVideoMask, "NULL")) { + options->auxVideoMask = psStringCopy(auxVideoMask); + } + } + + + // Mask recipe options (note that mask bit values are set in ppImageSetMaskBits.c) + options->doMask = psMetadataLookupBool(NULL, recipe, "MASK"); + + // Mask recipe options (note that mask bit values are set in ppImageSetMaskBits.c) + options->doCrosstalkMeasure = psMetadataLookupBool(NULL, recipe, "CROSSTALK.MEASURE"); + options->doCrosstalkCorrect = psMetadataLookupBool(NULL, recipe, "CROSSTALK.CORRECT"); + + options->doNoiseMap = psMetadataLookupBool(NULL, recipe, "NOISEMAP"); + options->doBias = psMetadataLookupBool(NULL, recipe, "BIAS"); + options->doDark = psMetadataLookupBool(NULL, recipe, "DARK"); + options->doRemnance = psMetadataLookupBool(NULL, recipe, "REMNANCE"); + options->doFlat = psMetadataLookupBool(NULL, recipe, "FLAT"); + options->doFringe = psMetadataLookupBool(NULL, recipe, "FRINGE"); + options->doShutter = psMetadataLookupBool(NULL, recipe, "SHUTTER"); + + // PATTERN.ROW is selected by the recipe. If it is selected, we search for the table + // PATTERN.ROW.SUBSET. If this is found in our format file, we use that version; + // otherwise, we use the table provided in the recipe file "ppImage.config". Within that + // table, we select the entry that matches our CHIP.NAME. This will be either a boolean or + // a string of bits. If it is a boolean, it specified whether or not to correct the entire + // chip; if it is a string, the bits specify which cells to correct (sequence is order of + // CELLS in the format:CHIPS metadata table) + + // We also check the chip header for the boolean 'PTRN_ROW' : if this is true, we have + // already applied this correct to this data, so we simply skip the correction for this + // chip. + + options->doPatternRow = psMetadataLookupBool(NULL, recipe, "PATTERN.ROW"); + options->doPatternCell = psMetadataLookupBool(NULL, recipe, "PATTERN.CELL"); + options->doPatternContinuity = psMetadataLookupBool(NULL, recipe, "PATTERN.CONTINUITY"); + options->doPatternDeadCells = psMetadataLookupBool(NULL, recipe, "PATTERN.DEAD.CELLS"); + + options->doMaskStats = psMetadataLookupBool(NULL, recipe, "MASK.STATS"); + options->addNoise = psMetadataLookupBool(NULL, recipe, "ADDNOISE"); + + options->doStats = false; + char *statsName = psMetadataLookupStr(&status, config->arguments, "STATS"); // Filename for statistics + if (statsName) { + options->doStats = true; + } + + options->applyParity = psMetadataLookupBool(NULL, recipe, "APPLY.CELL.PARITY"); + + options->burntoolTrails = psMetadataLookupS32(&status, recipe, "BURNTOOL.TRAILS"); + psTrace("psModules.detrend", 7, "burntoolTrails: %d BURNTOOL.TRAILS: %d Status: %d\n", + options->burntoolTrails,psMetadataLookupS32(&status,recipe,"BURNTOOL.TRAILS"),status); + if (!status) { + psWarning("BURNTOOL.TRAILS not found in recipe: setting to default value.\n"); + } + + // binned image options + options->xBin1 = psMetadataLookupS32(&status, recipe, "BIN1.XBIN"); + if (!status) { + psWarning("BIN1.XBIN not found in recipe: setting to default value.\n"); + options->xBin1 = 4; + } + options->yBin1 = psMetadataLookupS32(&status, recipe, "BIN1.YBIN"); + if (!status) { + psWarning("BIN1.YBIN not found in recipe: setting to default value.\n"); + options->yBin1 = 4; + } + + options->xBin2 = psMetadataLookupS32(&status, recipe, "BIN2.XBIN"); + if (!status) { + psWarning("BIN2.XBIN not found in recipe: setting to default value.\n"); + options->xBin1 = 16; + } + options->yBin2 = psMetadataLookupS32(&status, recipe, "BIN2.YBIN"); + if (!status) { + psWarning("BIN2.YBIN not found in recipe: setting to default value.\n"); + options->yBin1 = 16; + } + + options->BaseFITS = psMetadataLookupBool(NULL, recipe, "BASE.FITS"); + options->BaseMaskFITS = psMetadataLookupBool(NULL, recipe, "BASE.MASK.FITS"); + options->BaseVarianceFITS = psMetadataLookupBool(NULL, recipe, "BASE.VARIANCE.FITS"); + + options->ChipFITS = psMetadataLookupBool(NULL, recipe, "CHIP.FITS"); + options->ChipMaskFITS = psMetadataLookupBool(NULL, recipe, "CHIP.MASK.FITS"); + options->ChipVarianceFITS = psMetadataLookupBool(NULL, recipe, "CHIP.VARIANCE.FITS"); + + options->FPA1FITS = psMetadataLookupBool(NULL, recipe, "FPA1.FITS"); + options->FPA2FITS = psMetadataLookupBool(NULL, recipe, "FPA2.FITS"); + + options->Bin1FITS = psMetadataLookupBool(NULL, recipe, "BIN1.FITS"); + options->Bin1JPEG = psMetadataLookupBool(NULL, recipe, "BIN1.JPEG"); + options->Bin2FITS = psMetadataLookupBool(NULL, recipe, "BIN2.FITS"); + options->Bin2JPEG = psMetadataLookupBool(NULL, recipe, "BIN2.JPEG"); + + options->doPhotom = psMetadataLookupBool(NULL, recipe, "PHOTOM"); + options->doAstromChip = psMetadataLookupBool(NULL, recipe, "ASTROM.CHIP"); + options->doAstromMosaic = psMetadataLookupBool(NULL, recipe, "ASTROM.MOSAIC"); + options->doBG = psMetadataLookupBool(NULL, recipe, "BACKGROUND"); + + options->checkCTE = psMetadataLookupBool(NULL, recipe, "CHECK.CTE"); + options->checkNoise = psMetadataLookupBool(NULL, recipe, "CHECK.NOISE"); + options->squashNANs = psMetadataLookupBool(NULL, recipe, "SQUASH.NANS"); + + /* doMaskBuild : there are some cases where we require a mask, so we force doMaskBuild to be set even if the user specified 'FALSE' + * + * doPhotom : a mask is required because it is used to mark the locations of stars + * + * doNoiseMap : no reason this needs to trigger a mask? + * doBias : no reason this needs to trigger a mask? + * doOverscan : no reason this needs to trigger a mask? + * doDark : no reason this needs to trigger a mask? + * doShutter : no reason this needs to trigger a mask? + * doFlat : no reason this needs to trigger a mask? + */ + + // if the variance image is requested, build it (if not supplied) + if (options->BaseVarianceFITS || options->ChipVarianceFITS) { + options->doVarianceBuild = true; + } + // photometry and noisemap both require a variance image + if (options->doNoiseMap || options->doPhotom) { + options->doVarianceBuild = true; + } + + // we need a mask if we are going to apply these things: + if (options->doMaskSat || options->doMaskLow || options->doMaskBurntool || options->doMaskStats) { + options->doMaskBuild = true; + } + // photometry, mask, and background all require a mask image + if (options->doMask || options->doBG || options->doPhotom) { + options->doMaskBuild = true; + } + + if ((options->doAstromChip || options->doAstromMosaic) && !options->doPhotom) { + psLogMsg(__func__, PS_LOG_ERROR, "Invalid PPIMAGE options: cannot do ASTROMetry without PHOTOMetry"); + exit(EXIT_FAILURE); + } + + // Fringe options + options->fringeRej = psMetadataLookupF32(NULL, recipe, "FRINGE.REJ"); + options->fringeIter = psMetadataLookupS32(NULL, recipe, "FRINGE.ITER"); + options->fringeKeep = psMetadataLookupF32(NULL, recipe, "FRINGE.KEEP"); + + // Video cell options + if (psMetadataLookup(recipe, "USE.VIDEO.DARK")) { + options->useVideoDark = psMetadataLookupBool(NULL,recipe,"USE.VIDEO.DARK"); + } + if (psMetadataLookup(recipe, "USE.VIDEO.MASK")) { + options->useVideoMask = psMetadataLookupBool(NULL,recipe,"USE.VIDEO.MASK"); + } + + // Pattern correction + if (psMetadataLookup(format, "PATTERN.ROW.ORDER")) { + options->patternRowOrder = psMetadataLookupS32(NULL, format, "PATTERN.ROW.ORDER"); + } + else { + options->patternRowOrder = psMetadataLookupS32(NULL, recipe, "PATTERN.ROW.ORDER"); + } + if (psMetadataLookup(format, "PATTERN.ROW.ITER")) { + options->patternRowIter = psMetadataLookupS32(NULL, format, "PATTERN.ROW.ITER"); + } + else { + options->patternRowIter = psMetadataLookupS32(NULL, recipe, "PATTERN.ROW.ITER"); + } + if (psMetadataLookup(format, "PATTERN.ROW.REJ")) { + options->patternRowRej = psMetadataLookupF32(NULL, format, "PATTERN.ROW.REJ"); + } + else { + options->patternRowRej = psMetadataLookupF32(NULL, recipe, "PATTERN.ROW.REJ"); + } + if (psMetadataLookup(format, "PATTERN.ROW.THRESH")) { + options->patternRowThresh = psMetadataLookupF32(NULL, format, "PATTERN.ROW.THRESH"); + } + else { + options->patternRowThresh = psMetadataLookupF32(NULL, recipe, "PATTERN.ROW.THRESH"); + } + if (psMetadataLookup(format, "PATTERN.ROW.MEAN")) { + options->patternRowMean = psStatsOptionFromString(psMetadataLookupStr(NULL, format, "PATTERN.ROW.MEAN")); + } + else { + options->patternRowMean = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "PATTERN.ROW.MEAN")); + } + if (psMetadataLookup(format, "PATTERN.ROW.STDEV")) { + options->patternRowStdev = psStatsOptionFromString(psMetadataLookupStr(NULL, format, "PATTERN.ROW.STDEV")); + } + else { + options->patternRowStdev = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "PATTERN.ROW.STDEV")); + } + if (psMetadataLookup(format, "PATTERN.CELL.BG")) { + options->patternCellBG = psStatsOptionFromString(psMetadataLookupStr(NULL, format, "PATTERN.CELL.BG")); + } + else { + options->patternCellBG = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "PATTERN.CELL.BG")); + } + if (psMetadataLookup(format, "PATTERN.CELL.MEAN")) { + options->patternCellMean = psStatsOptionFromString(psMetadataLookupStr(NULL, format, "PATTERN.CELL.MEAN")); + } + else { + options->patternCellMean = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "PATTERN.CELL.MEAN")); + } + + if (psMetadataLookup(format, "PATTERN.CONTINUITY.WIDTH")) { + options->patternContinuityEdgeWidth = psMetadataLookupS32(NULL, format, "PATTERN.CONTINUITY.WIDTH"); + } + else { + options->patternContinuityEdgeWidth = psMetadataLookupS32(NULL, recipe, "PATTERN.CONTINUITY.WIDTH"); + } + + // Option to enable the background continuity + options->doBackgroundContinuity = psMetadataLookupBool(NULL, recipe, "BACKGROUND.CONTINUITY"); + + + // Remnance options + options->remnanceSize = psMetadataLookupF32(NULL, recipe, "REMNANCE.SIZE"); + options->remnanceThresh = psMetadataLookupS32(NULL, recipe, "REMNANCE.THRESH"); + + // per-class normalization source (just a reference; don't free) + options->normClass = psMetadataLookupStr(NULL, recipe, "NORM.CLASS"); + + options->maskstat_static = psMetadataLookupU16(NULL, recipe, "MASKSTAT.STATIC"); + options->maskstat_dynamic = psMetadataLookupU16(NULL, recipe, "MASKSTAT.DYNAMIC"); + options->maskstat_magic = psMetadataLookupU16(NULL, recipe, "MASKSTAT.MAGIC"); + options->maskstat_advisory = psMetadataLookupU16(NULL, recipe, "MASKSTAT.ADVISORY"); + + + return options; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageParityFlip.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageParityFlip.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageParityFlip.c (revision 42655) @@ -0,0 +1,165 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// Update a concept to the assumed value +#define FIX_CONCEPT(SOURCE, NAME, TYPE, VALUE) { \ +psMetadataItem *item = psMetadataLookup(SOURCE, NAME); \ +item->data.TYPE = VALUE; } + +// flip the image to have 'native' (detector) or 'raw' (readout) orientation +bool ppImageParityFlip (pmConfig *config, const ppImageOptions *options, const pmFPAview *view, bool native) { + + bool status; + int xParity, yParity; + int xParityRaw, yParityRaw; + int xParityTarget, yParityTarget; + + if (!options->applyParity) return true; + + // psTimerStart("parity.flip"); + + // find the currently selected readout + pmReadout *readout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + pmCell *cell = readout->parent; + + if (native) { + // find the current (raw) parity + xParity = psMetadataLookupS32(&status, cell->concepts, "CELL.XPARITY"); + if (!status || (xParity != -1 && xParity != 1)) { + psWarning("CELL.XPARITY is not set for the input cell; assuming 1.\n"); + FIX_CONCEPT(cell->concepts, "CELL.XPARITY", S32, 1); + xParity = 1; + } + yParity = psMetadataLookupS32(&status, cell->concepts, "CELL.YPARITY"); + if (!status || (yParity != -1 && yParity != 1)) { + psWarning("CELL.YPARITY is not set for the input cell; assuming 1.\n"); + FIX_CONCEPT(cell->concepts, "CELL.YPARITY", S32, 1); + yParity = 1; + } + + // save the raw parity + psMetadataAddS32 (cell->concepts, PS_LIST_TAIL, "CELL.XPARITY.RAW", PS_META_REPLACE, "original parity", xParity); + psMetadataAddS32 (cell->concepts, PS_LIST_TAIL, "CELL.YPARITY.RAW", PS_META_REPLACE, "original parity", yParity); + + xParityTarget = 1; + yParityTarget = 1; + } else { + // find the current (native) parity + xParity = psMetadataLookupS32(&status, cell->concepts, "CELL.XPARITY"); + psAssert (status, "CELL.XPARITY not set : cannot call ppImageParityFlip (native = false) without first calling it with native = true"); + + yParity = psMetadataLookupS32(&status, cell->concepts, "CELL.YPARITY"); + psAssert (status, "CELL.YPARITY not set : cannot call ppImageParityFlip (native = false) without first calling it with native = true"); + + // find the raw parity + xParityRaw = psMetadataLookupS32(&status, cell->concepts, "CELL.XPARITY.RAW"); + psAssert (status, "CELL.XPARITY not set : cannot call ppImageParityFlip (native = false) without first calling it with native = true"); + + yParityRaw = psMetadataLookupS32(&status, cell->concepts, "CELL.YPARITY.RAW"); + psAssert (status, "CELL.YPARITY not set : cannot call ppImageParityFlip (native = false) without first calling it with native = true"); + + xParityTarget = xParityRaw; + yParityTarget = yParityRaw; + } + + // for this case, nothing to be done: + if ((xParity == xParityTarget) && (yParity == yParityTarget)) return true; + + psImage *image = readout->image; + psImage *var = readout->variance; + psImage *mask = readout->mask; + + if (var) psAssert (image->numCols == var->numCols, "image and variance sizes are mismatched"); + if (var) psAssert (image->numRows == var->numRows, "image and variance sizes are mismatched"); + if (mask) psAssert (image->numCols == mask->numCols, "image and mask sizes are mismatched"); + if (mask) psAssert (image->numRows == mask->numRows, "image and mask sizes are mismatched"); + + int Nx = image->numCols; + int Ny = image->numRows; + + // we need working buffers for the image, variance, and mask: + psF32 *imrow = (psF32 *) psAlloc (Nx*sizeof(psF32)); + psF32 *wtrow = (psF32 *) psAlloc (Nx*sizeof(psF32)); + psImageMaskType *mkrow = (psImageMaskType *) psAlloc (Nx*sizeof(psImageMaskType)); + + // the three cases (+1,-1), (-1,+1), (-1,-1) should be handled independently + + // flip only in x-direction + if ((xParity != xParityTarget) && (yParity == yParityTarget)) { + for (int iy = 0; iy < Ny; iy++) { + for (int ix = 0; ix < Nx; ix++) { + imrow[Nx-1-ix] = image->data.F32[iy][ix]; + } + for (int ix = 0; (var != NULL) && (ix < Nx); ix++) { + wtrow[Nx-1-ix] = var->data.F32[iy][ix]; + } + for (int ix = 0; (mask != NULL) && (ix < Nx); ix++) { + mkrow[Nx-1-ix] = mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]; + } + memcpy (image->data.F32[iy], imrow, Nx*sizeof(psF32)); + if (var) memcpy (var->data.F32[iy], wtrow, Nx*sizeof(psF32)); + if (mask) memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy], mkrow, Nx*sizeof(psImageMaskType)); + } + } + // flip only in y-direction + if ((xParity == xParityTarget) && (yParity != yParityTarget)) { + for (int iy = 0; iy < Ny/2; iy++) { + memcpy (imrow, image->data.F32[iy], Nx*sizeof(psF32)); + memcpy (image->data.F32[iy], image->data.F32[Ny-1-iy], Nx*sizeof(psF32)); + memcpy (image->data.F32[Ny-1-iy], imrow, Nx*sizeof(psF32)); + + if (var) { + memcpy (wtrow, var->data.F32[iy], Nx*sizeof(psF32)); + memcpy (var->data.F32[iy], var->data.F32[Ny-1-iy], Nx*sizeof(psF32)); + memcpy (var->data.F32[Ny-1-iy], wtrow, Nx*sizeof(psF32)); + } + + if (mask) { + memcpy (mkrow, mask->data.PS_TYPE_IMAGE_MASK_DATA[iy], Nx*sizeof(psImageMaskType)); + memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy], mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy], Nx*sizeof(psImageMaskType)); + memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy], mkrow, Nx*sizeof(psImageMaskType)); + } + } + } + // flip in both directions + if ((xParity != xParityTarget) && (yParity != yParityTarget)) { + for (int iy = 0; iy < Ny/2; iy++) { + for (int ix = 0; ix < Nx; ix++) { + imrow[Nx-1-ix] = image->data.F32[iy][ix]; + } + for (int ix = 0; var && (ix < Nx); ix++) { + wtrow[Nx-1-ix] = var->data.F32[iy][ix]; + } + for (int ix = 0; mask && (ix < Nx); ix++) { + mkrow[Nx-1-ix] = mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]; + } + + for (int ix = 0; ix < Nx; ix++) { + image->data.F32[iy][Nx-1-ix] = image->data.F32[Ny-1-iy][ix]; + } + for (int ix = 0; var && (ix < Nx); ix++) { + var->data.F32[iy][Nx-1-ix] = var->data.F32[Ny-1-iy][ix]; + } + for (int ix = 0; mask && (ix < Nx); ix++) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][Nx-1-ix] = mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy][ix]; + } + + memcpy (image->data.F32[Ny-1-iy], imrow, Nx*sizeof(psF32)); + if (var) memcpy (var->data.F32[Ny-1-iy], wtrow, Nx*sizeof(psF32)); + if (mask) memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy], mkrow, Nx*sizeof(psImageMaskType)); + } + } + // psLogMsg ("ppImage", 5, "parity flip: %f sec\n", psTimerMark ("parity.flip")); + + FIX_CONCEPT(cell->concepts, "CELL.XPARITY", S32, xParityTarget); + FIX_CONCEPT(cell->concepts, "CELL.YPARITY", S32, yParityTarget); + + psFree (imrow); + psFree (wtrow); + psFree (mkrow); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageParseCamera.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageParseCamera.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageParseCamera.c (revision 42655) @@ -0,0 +1,573 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +ppImageOptions *ppImageParseCamera(pmConfig *config) +{ + bool status = false; + + if (!ppImageDefineFile(config, NULL, "PPIMAGE.INPUT", "INPUT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NONE)) { + psError(PS_ERR_IO, false, "Can't find an input image source"); + return NULL; + } + pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPIMAGE.INPUT"); // Input file + psAssert(input, "We just put it there!"); + + // add recipe options supplied on command line + psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); + + // parse the options from the metadata format to the ppImageOptions structure + ppImageOptions *options = ppImageOptionsParse(config); + + // the following are defined from the argument list, if given, + // otherwise they revert to the config information + // not all input or output images are used in a given recipe + if (options->doNoiseMap) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.NOISEMAP", "NOISEMAP", + PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NOISEMAP)) { + psError(PS_ERR_IO, false, "Can't find a noise map image source"); + psFree(options); + return NULL; + } + } + if (options->doBias) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.BIAS", "BIAS", + PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_BIAS)) { + psError(PS_ERR_IO, false, "Can't find a bias image source"); + psFree(options); + return NULL; + } + } + if (options->doDark) { + // Always load the regular Dark + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.DARK", "DARK", + PM_FPA_FILE_DARK, PM_DETREND_TYPE_DARK)) { + psError(PS_ERR_IO, false, "Can't find a dark image source"); + psFree(options); + return NULL; + } + // Sometimes load the video dark if we need it. + if (options->useVideoDark) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.VIDEODARK", "DARK", + PM_FPA_FILE_DARK, PM_DETREND_TYPE_VIDEODARK)) { + psError(PS_ERR_IO, false, "Can't find a dark image source"); + psFree(options); + return NULL; + } + } + } + if (options->doMask) { + // Always load the regular mask + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.MASK", "MASK", + PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) { + psError(PS_ERR_IO, false, "Can't find a mask image source"); + psFree(options); + return NULL; + } + + if (options->useVideoMask) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.VIDEOMASK", "MASK", + PM_FPA_FILE_MASK, PM_DETREND_TYPE_VIDEOMASK)) { + psError(PS_ERR_IO, false, "Can't find a mask image source"); + psFree(options); + return NULL; + } + } + if (options->doAuxMask) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.AUXMASK", "MASK", + PM_FPA_FILE_MASK, PM_DETREND_TYPE_AUXMASK)) { + psError(PS_ERR_IO, false, "Can't find a auxillary mask image source"); + psFree(options); + return NULL; + } + } + } + if (options->doShutter) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.SHUTTER", "SHUTTER", + PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_SHUTTER)) { + psError(PS_ERR_IO, false, "Can't find a shutter image source"); + psFree(options); + return NULL; + } + } + + if (options->doFlat) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.FLAT", "FLAT", + PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_FLAT)) { + psError(PS_ERR_IO, false, "Can't find a flat image source"); + psFree(options); + return NULL; + } + } + + if (options->doNonLin) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.LINEARITY", "LINEARITY", + PM_FPA_FILE_LINEARITY, PM_DETREND_TYPE_LINEARITY)) { + psError(PS_ERR_IO, false, "Can't find a non-linearity correction source"); + psFree(options); + return NULL; + } + } + if (options->doNewNonLin) { + // if the file has been specified on the command-line (-newnonlin file), then the file + // is identified by the NEWNONLIN entry in config->arguments. otherwise, load from the + // detrend system as type NEWNONLIN + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.NEWNONLIN", "NEWNONLIN", + PM_FPA_FILE_NEWNONLIN, PM_DETREND_TYPE_NEWNONLIN)) { + psError(PS_ERR_IO, false, "Can't find a new non-linearity correction source"); + psFree(options); + return NULL; + } + } + + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); + if (nThreads > 0) { + int nScanRows = psMetadataLookupS32(&status, recipe, "SCAN.ROWS"); + pmDetrendSetThreadTasks(nScanRows); + } + ppImageSetThreads (); // even if threading is off, we need to identify the ppImageDetrendPattern thread function + + if (options->doPatternRow || options->doPatternCell) { + pmFPAfile *outPattern = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.PATTERN"); + if (!outPattern) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPIMAGE.PATTERN")); + psFree(options); + return NULL; + } + outPattern->save = true; + } + + // fringe frame are only applied for a subset of the filters. + // if the filter is one of those identified by a FRINGE.FILTERS metadata entry + // in ppImage.config, apply the fringe frame + if (options->doFringe) { + // determine filter from the concepts + const char *filter = psMetadataLookupStr(&status, input->fpa->concepts, "FPA.FILTER"); + if (!status || !filter || strlen(filter) == 0) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find FPA.FILTER.\n"); + psFree(options); + return NULL; + } + + // select FRINGE.FILTERS from the recipe and test if the filter matches + // is the mdi saved with psList or string? + psMetadataItem *mdi = psMetadataLookup (recipe, "FRINGE.FILTERS"); + if (mdi == NULL) { + // no valid filters for fringe data for this camera + options->doFringe = false; + goto skip_fringe; + } + // place entry on a list regardless of type + psList *filters = NULL; + if (mdi->type == PS_DATA_STRING) { + filters = psListAlloc(NULL); + psListAdd (filters, PS_LIST_HEAD, mdi); + } else if (mdi->type == PS_DATA_METADATA_MULTI) { + filters = psMemIncrRefCounter(mdi->data.list); + } else { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, + "FRINGE.FILTERS in recipe %s is not of type METADATA", RECIPE_NAME); + psFree(options); + return NULL; + } + + // search through list to find the current filter + psListIterator *iter = psListIteratorAlloc (filters, PS_LIST_HEAD, FALSE); + options->doFringe = false; + for (int i = 0; !options->doFringe && (i < filters->n); i++) { + psMetadataItem *item = psListGetAndIncrement (iter); + char *validFilter = item->data.V; + if (strcmp (validFilter, filter)) continue; + options->doFringe = true; + } + psFree(iter); + psFree(filters); + } +skip_fringe: + if (options->doFringe) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.FRINGE", "FRINGE", + PM_FPA_FILE_FRINGE, PM_DETREND_TYPE_FRINGE)) { + psError (PS_ERR_IO, false, "Can't find a fringe image source"); + return NULL; + } + } + + if (options->doPatternRow) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.PATTERN.ROW.AMP", "PATTERN.ROW.AMP", + PM_FPA_FILE_PATTERN_ROW_AMP, PM_DETREND_TYPE_PATTERN_ROW_AMP)) { + psWarning ("Can't find a pattern row amplitude source, will apply to all cells or defined subset"); + // an empty or invalid file may have been generated -- we want to skip, not raise an error + pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP"); + if (PRAfile) { + PRAfile->state |= PM_FPA_STATE_INACTIVE; + } + } + } + if (options->doPatternDeadCells) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.PATTERN.DEAD.CELLS", "PATTERN.DEAD.CELLS", + PM_FPA_FILE_PATTERN_DEAD_CELLS, PM_DETREND_TYPE_PATTERN_DEAD_CELLS)) { + psWarning ("Can't find a pattern dead cells source"); + // an empty or invalid file may have been generated -- we want to skip, not raise an error + pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.DEAD.CELLS"); + if (PRAfile) { + PRAfile->state |= PM_FPA_STATE_INACTIVE; + } + } + } + + if (options->checkCTE && false) { + int DX = psMetadataLookupS32 (&status, recipe, "CTE.XBIN"); + int DY = psMetadataLookupS32 (&status, recipe, "CTE.YBIN"); + pmFPAfile *outCTE = pmFPAfileDefineFromFile (config, input, DX, DY, "PPIMAGE.CTEMAP"); + if (!outCTE) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PPIMAGE.CTEMAP"); + return false; + } + outCTE->save = true; + } + if (options->doApplyBurntool) { + // If no burntool file was supplied do not fail. (camera_exp.pl does not supply it + // for example. + if (!psMetadataLookupStr(NULL, config->arguments, "BURNTOOL.TABLE")) { + psWarning("BURNTOOL.TABLE not supplied setting doApplyBurntool to false"); + options->doApplyBurntool = false; + } + } + + // the following files are output targets + pmFPAfile *outImage = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.OUTPUT"); + if (!outImage) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPIMAGE.OUTPUT")); + psFree(options); + return NULL; + } + if (outImage->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT is not of type IMAGE"); + psFree(options); + return NULL; + } + pmFPAfile *outMask = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.OUTPUT.MASK"); + if (!outMask) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPIMAGE.OUTPUT.MASK")); + psFree(options); + return NULL; + } + if (outMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.MASK is not of type MASK"); + psFree(options); + return NULL; + } + pmFPAfile *outVariance = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.OUTPUT.VARIANCE"); + if (!outVariance) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPIMAGE.OUTPUT.VARIANCE")); + psFree(options); + return NULL; + } + if (outVariance->type != PM_FPA_FILE_VARIANCE) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.VARIANCE is not of type VARIANCE"); + psFree(options); + return NULL; + } + + // XXX should these be bound explicitly to the outImage->fpa rather than the input->fpa? + pmFPAfile *chipImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PPIMAGE.CHIP"); + if (!chipImage) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.CHIP")); + psFree(options); + return NULL; + } + if (chipImage->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.CHIP is not of type IMAGE"); + psFree(options); + return NULL; + } + pmFPAfile *chipMask = pmFPAfileDefineOutput(config, chipImage->fpa, "PPIMAGE.CHIP.MASK"); + if (!chipMask) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPIMAGE.CHIP.MASK")); + psFree(options); + return NULL; + } + if (chipMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPIMAGE.CHIP.MASK is not of type MASK"); + psFree(options); + return NULL; + } + pmFPAfile *chipVariance = pmFPAfileDefineOutput(config, chipImage->fpa, "PPIMAGE.CHIP.VARIANCE"); + if (!chipVariance) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPIMAGE.CHIP.VARIANCE")); + psFree(options); + return NULL; + } + if (chipVariance->type != PM_FPA_FILE_VARIANCE) { + psError(PS_ERR_IO, true, "PPIMAGE.CHIP.VARIANCE is not of type VARIANCE"); + psFree(options); + return NULL; + } + + pmFPAfile *byFPA1 = pmFPAfileDefineFPAMosaic(config, input->fpa, "PPIMAGE.OUTPUT.FPA1"); + if (!byFPA1) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.OUTPUT.FPA1")); + psFree(options); + return NULL; + } + if (byFPA1->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.FPA1 is not of type IMAGE"); + psFree(options); + return NULL; + } + pmFPAfile *byFPA2 = pmFPAfileDefineFPAMosaic(config, input->fpa, "PPIMAGE.OUTPUT.FPA2"); + if (!byFPA2) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.OUTPUT.FPA2")); + psFree(options); + return NULL; + } + if (byFPA2->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.FPA2 is not of type IMAGE"); + psFree(options); + return NULL; + } + + + // chipImage -> psphotInput (pmFPAfileDefineFromFile) : fpa is constructed + // psphotInput -> psphotOutput (pmFPAfileDefineOutputFromFile) : fpa is equated + // psphotOutput -> psastroInput (pmFPAfileDefineInput) : fpa is ref-copied + // psastroInput -> psastroModel (pmFPAfileDefineFromArgs : fpa is ref-copied + // psastroInput -> psastroModel (pmFPAfileDefineFromConf : fpa is constructed + // psastroInput -> psastroModel (pmFPAfileDefineFromDetDB : fpa is constructed (pmDetrendSelect uses input concepts ) + + // For photometry, we operate on the chip-mosaicked image + // we create a copy of the mosaicked image for psphot so we can write out a clean image + if (options->doPhotom || options->doBG) { + + // this file is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by + // psphotDefineFiles + pmFPAfile *psphotInput = pmFPAfileDefineFromFile (config, chipImage, 1, 1, "PSPHOT.INPUT"); + PS_ASSERT (psphotInput, false); + + // specify the number of psphot input images + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", 1); + + // define associated psphot input/output files + if (!psphotDefineFiles (config, psphotInput)) { + psError(PSPHOT_ERR_CONFIG, false, + "Trouble defining the additional input/output files for psphot"); + return false; + } + } + + // For photometry, we operate on the chip-mosaicked image + if (options->doAstromChip || options->doAstromMosaic) { + if (!options->doPhotom) { + psError(PSASTRO_ERR_CONFIG, false, + "Photometry mode is not selected; it is required for astrometry"); + return false; + } + + pmFPAfile *psphotOutput = psMetadataLookupPtr(&status, config->files, "PSPHOT.OUTPUT"); + PS_ASSERT(psphotOutput, false); + + pmFPAfile *psastroInput = pmFPAfileDefineInput(config, psphotOutput->fpa, NULL, "PSASTRO.INPUT"); + PS_ASSERT(psastroInput, false); + psastroInput->mode = PM_FPA_MODE_REFERENCE; + + // define associated psphot input/output files + if (!psastroDefineFiles(config, psastroInput)) { + psError(PSPHOT_ERR_CONFIG, false, + "Trouble defining the additional input/output files for psastro"); + return false; + } + + // deactivate the psastro files, reactive when needed + pmFPAfileActivate(config->files, false, "PSASTRO.OUTPUT"); + } + + // save any of these files? + outImage->save = options->BaseFITS; + outMask->save = options->BaseMaskFITS; + outVariance->save = options->BaseVarianceFITS; + + chipImage->save = options->ChipFITS; + chipMask->save = options->ChipMaskFITS; + chipVariance->save = options->ChipVarianceFITS; + + byFPA1->save = options->FPA1FITS; + byFPA2->save = options->FPA2FITS; + + // outImage is used as a carrier: input to chipImage -> require the data to remain at the CHIP level + outImage->freeLevel = PS_MIN(outImage->freeLevel, PM_FPA_LEVEL_CHIP); + outImage->dataLevel = outImage->freeLevel; + outImage->fileLevel = PS_MIN(outImage->fileLevel, outImage->dataLevel); + + // outMask and outVariance must be freed at the same level as outImage (all freed by pmFPAFreeData) + outMask->freeLevel = outImage->freeLevel; + outVariance->freeLevel = outImage->freeLevel; + outMask->dataLevel = outImage->dataLevel; + outVariance->dataLevel = outImage->dataLevel; + + // Ditto for the chip-mosaicked version + chipMask->freeLevel = chipImage->freeLevel; + chipVariance->freeLevel = chipImage->freeLevel; + chipMask->dataLevel = chipImage->dataLevel; + chipVariance->dataLevel = chipImage->dataLevel; + + // the input data is the same as the outImage data : force the free levels to match + input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel); + + // define the continuity corrected background model files + if (options->doBackgroundContinuity) { + pmFPAfile *bkgMosaicModel = pmFPAfileDefineFromFPA(config,input->fpa, 1, 1, "PPIMAGE.BACKMDL"); + if (!bkgMosaicModel) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BACKMDL")); + psFree(options); + return NULL; + } + if (bkgMosaicModel->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.BACKMDL is not of type IMAGE"); + psFree(options); + return NULL; + + } + bkgMosaicModel->save = options->doBackgroundContinuity; + // bkgMosaicModel->freeLevel = PS_MIN(bkgMosaicModel->freeLevel, PM_FPA_LEVEL_FPA); + bkgMosaicModel->freeLevel = PM_FPA_LEVEL_FPA; + bkgMosaicModel->dataLevel = bkgMosaicModel->dataLevel; + bkgMosaicModel->fileLevel = PS_MIN(bkgMosaicModel->fileLevel, bkgMosaicModel->dataLevel); + } + + // define the binned target files (which may just be carriers for some camera configurations) + pmFPAfile *bin1 = pmFPAfileDefineFromFPA(config, chipImage->fpa, options->xBin1, options->yBin1, + "PPIMAGE.BIN1"); + if (!bin1) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BIN1")); + psFree(options); + return NULL; + } + if (bin1->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.BIN1 is not of type IMAGE"); + psFree(options); + return NULL; + } + + pmFPAfile *bin2 = pmFPAfileDefineFromFPA(config, chipImage->fpa, options->xBin2, options->yBin2, + "PPIMAGE.BIN2"); + if (!bin2) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BIN2")); + psFree(options); + return NULL; + } + if (bin2->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.BIN2 is not of type IMAGE"); + psFree(options); + return NULL; + } + + // bin1 and bin2 are used as carriers: input for byFPA1, byFPA2 + bin1->freeLevel = PM_FPA_LEVEL_FPA; + bin2->freeLevel = PM_FPA_LEVEL_FPA; + + pmFPAfile *jpg1 = pmFPAfileDefineOutput(config, byFPA1->fpa, "PPIMAGE.JPEG1"); + if (!jpg1) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.JPEG1")); + psFree(options); + return NULL; + } + if (jpg1->type != PM_FPA_FILE_JPEG) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.JPEG1 is not of type JPEG"); + psFree(options); + return NULL; + } + pmFPAfile *jpg2 = pmFPAfileDefineOutput(config, byFPA2->fpa, "PPIMAGE.JPEG2"); + if (!jpg2) { + psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.JPEG2")); + psFree(options); + return NULL; + } + if (jpg2->type != PM_FPA_FILE_JPEG) { + psError(PS_ERR_IO, true, "PPIMAGE.OUTPUT.JPEG2 is not of type JPEG"); + psFree(options); + return NULL; + } + + // XXX we could potentially not define these pmFPAfiles if no output is requested... + bin1->save = options->Bin1FITS; + bin2->save = options->Bin2FITS; + jpg1->save = options->Bin1JPEG; + jpg2->save = options->Bin2JPEG; + + // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg) + char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS"); + psArray *chips = psStringSplitArray(chipLine, ",", false); + if (chips->n > 0) { + pmFPASelectChip (input->fpa, -1, true); // deselect all chips + for (int i = 0; i < chips->n; i++) { + int chipNum = atoi(chips->data[i]); + if (! pmFPASelectChip(input->fpa, chipNum, false)) { + psError(PS_ERR_IO, false, "Chip number %d doesn't exist in camera.\n", chipNum); + psFree(options); + return false; + } + } + } + psFree (chips); + + if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_FRINGE")) { + // It's a fringe file, so change the file type + input->type = PM_FPA_FILE_FRINGE; + outImage->type = PM_FPA_FILE_FRINGE; + } + if (psMetadataLookupBool(NULL, config->arguments, "INPUT_IS_DARK")) { + // It's a dark file, so change the file type + input->type = PM_FPA_FILE_DARK; + outImage->type = PM_FPA_FILE_DARK; + // Turn off compression --- there are just too many nasties that can happen + psFree(outImage->compression); + outImage->compression = NULL; + psFree(outImage->options); + outImage->options = NULL; + } + + // Turn off mask and variance output if we're not doing anything interesting + if (!options->doMaskBuild && outMask->save) { + psWarning("output mask image (BASE.MASK.FITS) requested, but not generated: skipping.\n"); + outMask->save = false; + } + if (!options->doVarianceBuild && outVariance->save) { + psWarning("output variance image (BASE.VARIANCE.FITS) requested, but not generated: skipping.\n"); + outVariance->save = false; + } + if (!options->doMaskBuild && chipMask->save) { + psWarning("output mask image (CHIP.MASK.FITS) requested, but not generated: skipping.\n"); + chipMask->save = false; + } + if (!options->doVarianceBuild && chipVariance->save) { + psWarning("output variance image (CHIP.VARIANCE.FITS) requested, but not generated: skipping.\n"); + chipVariance->save = false; + } + + if (psTraceGetLevel("ppImage.config") > 0) { + // Get a look inside all the files. + psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL); + psMetadataItem *item; // Item from iteration + fprintf(stderr, "Files:\n"); + while ((item = psMetadataGetAndIncrement(filesIter))) { + pmFPAfile *file = item->data.V; // File of interest + fprintf(stderr, "%s: %p %p %p (%p) %p\n", file->name, + file->src, file->fpa, + file->camera, file->fpa->camera, file->format); + } + psFree(filesIter); + } + + // Change the input dark type between the old (IMAGE) and new (multi-DARK). + // Hopefully this is a temporary change until we all move over to using the new dark types + bool mdok; // Status of MD lookup + if (options->doDark && psMetadataLookupBool(&mdok, recipe, "OLDDARK")) { + pmFPAfile *dark = psMetadataLookupPtr(NULL, config->files, "PPIMAGE.DARK"); + assert(dark); + dark->type = PM_FPA_FILE_IMAGE; + } + + return (options); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImagePhotom.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImagePhotom.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImagePhotom.c (revision 42655) @@ -0,0 +1,63 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +# include "ppImage.h" + +// In this function, we perform the psphot analysis routine for the chip-mosaicked images +bool ppImagePhotom(psMetadata *stats, pmConfig *config, pmFPAview *view) { + + bool status; + pmCell *cell; + pmReadout *readout; + + psphotInit(); + + // find or define a pmFPAfile PSPHOT.INPUT + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT"); + if (!status) { + psError(PSPHOT_ERR_CONFIG, false, "PSPHOT.INPUT I/O file is not defined"); + return false; + } + + // we make a new copy of the output chip to keep psphot from modifying the output image + pmChip *oldChip = pmFPAviewThisChip (view, input->src); + pmChip *newChip = pmFPAviewThisChip (view, input->fpa); + pmChipCopy (newChip, oldChip); + + // iterate over the cells and readout for this chip + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("ppImagePhotom", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! cell->process || ! cell->file_exists) { continue; } + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + if (! readout->data_exists) { continue; } + + // run the actual photometry analysis + if (!psphotReadout (config, view, "PSPHOT.INPUT")) { + // This is likely a data quality issue + // XXX Split into multiple cases using error codes? + psErrorStackPrint(stderr, "Unable to perform photometry on image"); + psWarning("Unable to perform photometry on image --- suspect bad data quality."); + if (stats && psMetadataLookupS32(NULL, stats, "QUALITY") == 0) { + psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, + "Unable to perform photometry on image", psErrorCodeLast()); + } + psErrorClear(); + // psphotFilesActivate(config, false); + } + + // we want to save the MASK as modified by psphot, but not the data or weight + // free the old mask and replace with a memory copy of the new mask + pmReadout *oldReadout = pmFPAviewThisReadout(view, input->src); + pmReadout *newReadout = pmFPAviewThisReadout(view, input->fpa); + psFree (oldReadout->mask); + oldReadout->mask = psMemIncrRefCounter(newReadout->mask); + } + } + + ppImageMemoryDump("photom"); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImagePixelStats.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImagePixelStats.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImagePixelStats.c (revision 42655) @@ -0,0 +1,89 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// calculate stats, including MD5 +bool ppImagePixelStats(pmConfig *config, psMetadata *stats, const ppImageOptions *options, + const pmFPAview *inputView) +{ + bool mdok; // Status of MD lookup + + // psTimerStart("pixel.stats"); + + // loop over the cells/readouts for this chip + pmFPAview *view = pmFPAviewAlloc(0); // View for local processing + *view = *inputView; + + // perform the analysis of for this FPA + pmFPAfile *output = psMetadataLookupPtr(&mdok, config->files, "PPIMAGE.OUTPUT"); + if (!output) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find output file (PPIMAGE.OUTPUT)."); + psFree(view); + return false; + } + + // select the corresponding chip + pmChip *chip = pmFPAviewThisChip(view, output->fpa); + if (!chip) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find requested chip."); + psFree(view); + return false; + } + + // Perform statistics for this chip + if (options->doStats) { + if (!ppStatsPixels(stats, output->fpa, view, options->maskValue, config)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate stats for image."); + psFree(view); + return false; + } + + // extract the fringe amplitude from the analysis + if (options->doFringe && !ppStatsFringe(stats, chip, "FRINGE", "FRINGE.SOLUTION")) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to extract fringe solution for image."); + psFree(view); + return false; + } + + // extract the fringe residual amplitude from the analysis + if (options->doFringe && !ppStatsFringe(stats, chip, "FRINGE_RESID", "FRINGE.RESIDUAL.SOLUTION")) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to extract fringe solution for image."); + psFree(view); + return false; + } + } + + + view->cell = view->readout = -1; + pmCell *cell = NULL; // Cell from iteration + while ((cell = pmFPAviewNextCell(view, output->fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists) { + continue; + } + + // Add MD5 information for cell + pmHDU *hdu = pmHDUFromCell(cell); // HDU that owns the cell + pmReadout *readout = NULL; // Readout from iteration + while ((readout = pmFPAviewNextReadout(view, output->fpa, 1)) != NULL) { + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); + + psString headerName = NULL; // Header name for MD5 + psStringAppend(&headerName, "MD5_%s_%s_%d", chipName, cellName, view->readout); + + psVector *md5 = psImageMD5(readout->image); // md5 hash + psString md5string = psMD5toString(md5); // String + psFree(md5); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, headerName, PS_META_REPLACE, "Image MD5", md5string); + psFree(md5string); + psFree(headerName); + } + } + + // psLogMsg ("ppImage", 5, "measure pixel stats: %f sec\n", psTimerMark ("pixel.stats")); + + psFree (view); + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageRandomGaussian.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageRandomGaussian.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageRandomGaussian.c (revision 42655) @@ -0,0 +1,102 @@ +# include "ppImage.h" + +static int Ngaussint = 0; +static double *gaussint = NULL; + +extern double drand48(); + +double p_ppImageGaussian (double x, double mean, double sigma) { + + double f; + + f = exp (-0.5 * PS_SQR(x - mean) / PS_SQR(sigma)) / sqrt(2 * M_PI * PS_SQR(sigma)); + + return (f); + +} + +void ppImageRandomGaussianFree() +{ + psFree (gaussint); + return; +} + +void ppImageRandomGaussianAlloc (int Nbin) { + + gaussint = (double *) psAlloc(Nbin*sizeof(double)); + return; +} + +/* integrate a gaussian from -5 sigma to +5 sigma */ +void p_ppImageRandomGaussianInit (void) { + + int i; + long A, B; + double val, x, dx, dx1, dx2, dx3, df; + double mean, sigma; + + /* no need to generate this if it already exists */ + if (gaussint) return; + + A = time(NULL); + for (B = 0; A == time(NULL); B++); + srand48(B); + + Ngaussint = 0x1000; + ppImageRandomGaussianAlloc (Ngaussint + 1); + + val = 0; + dx = 1.0 / Ngaussint; + dx1 = dx / 3.0; + dx2 = 2.0*dx/3.0; + dx3 = dx; + mean = 0.0; + sigma = 1.0; + + for (i = 0, x = -7.0; (i < Ngaussint) && (x < 7.0); x += dx) { + df = (3.0*p_ppImageGaussian(x , mean, sigma) + + 9.0*p_ppImageGaussian(x+dx1, mean, sigma) + + 9.0*p_ppImageGaussian(x+dx2, mean, sigma) + + 3.0*p_ppImageGaussian(x+dx3, mean, sigma)) * (dx1/8.0); + val += df; + if (val > (i + 0.5) / (double) Ngaussint) { + gaussint[i] = x + dx / 2.0; + i++; + } + } +} + +// XXX we are using drand48() rather than the random var supplied by rnd +double ppImageRandomGaussian (const psRandom *rnd, double mean, double sigma) { + + int i; + double y; + + if (gaussint == NULL) { + p_ppImageRandomGaussianInit (); + } + + y = drand48(); + i = Ngaussint*y; + y = gaussint[i]*sigma + mean; + + return (y); + +} + +// XXX we are using drand48() rather than the random var supplied by rnd +double ppImageRandomGaussianNorm (const psRandom *rnd) { + + int i; + double y; + + if (gaussint == NULL) { + p_ppImageRandomGaussianInit (); + } + + y = drand48(); + i = Ngaussint*y; + y = gaussint[i]; + + return (y); +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageRebinReadout.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageRebinReadout.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageRebinReadout.c (revision 42655) @@ -0,0 +1,52 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageRebinChip (pmConfig *config, pmFPAview *view, ppImageOptions *options, char *outName) { + + pmCell *cell; + pmReadout *inReadout, *outReadout; + + pmFPAfile *outFile = psMetadataLookupPtr(NULL, config->files, outName); + if (outFile == NULL) return false; + + // psTimerStart("rebin.chip"); + + // XXX double check that chip != -1? + + pmChip *inChip = pmFPAviewThisChip (view, outFile->src); + pmChip *outChip = pmFPAviewThisChip (view, outFile->fpa); + if (!pmChipCopyStructure (outChip, inChip, outFile->xBin, outFile->yBin)) { + psError(PS_ERR_UNKNOWN, false, "Unable to copy chip structure."); + return false; + } + + while ((cell = pmFPAviewNextCell (view, outFile->src, 1)) != NULL) { + psLogMsg ("ppImageRebinChip", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! cell->process || ! cell->file_exists) { continue; } + + // process each of the readouts + while ((inReadout = pmFPAviewNextReadout (view, outFile->src, 1)) != NULL) { + if (! inReadout->data_exists) { continue; } + + outReadout = pmFPAviewThisReadout (view, outFile->fpa); + + // run the rebin code + // XXX EAM 2022.04.21 : this function rebins the signal image and makes an attempt to + // generate a mask only with bits raised in masked pixels that have > 50% of input pixels masked. + // this step is very expensive because it must count the input mask bits for each pixel. + // What is particularly silly is that the mask is not even used in the jpeg image. + if (!pmReadoutRebin(outReadout, inReadout, options->maskValue, outFile->xBin, outFile->yBin)) { + psError(PS_ERR_UNKNOWN, false, "Unable to rebin readout."); + return false; + } + } + } + + // psLogMsg ("ppImage", 5, "rebin chip for %s: %f sec\n", outName, psTimerMark ("rebin.chip")); + + return true; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImageReplaceBackground.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageReplaceBackground.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageReplaceBackground.c (revision 42655) @@ -0,0 +1,212 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// In this function, we perform the psphot analysis routine for the chip-mosaicked images +bool ppImageSubtractBackground(pmConfig *config, const pmFPAview *view, const ppImageOptions *options) +{ + psAssert(config, "Need configuration"); + psAssert(view, "Need view to chip"); + psAssert(options, "Need options"); + + if (!options->doBG) { + return true; + } + + bool status; // Status of MD lookup + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.CHIP"); // File to correct + if (!status) { + psError(PS_ERR_UNEXPECTED_NULL, true, "PPIMAGE.CHIP file is not defined"); + return false; + } + + psMetadata *ppImageRecipe = psMetadataLookupPtr(NULL, config->recipes, RECIPE_NAME); + psAssert(ppImageRecipe, "Need PPIMAGE recipe"); + psMetadata *psphotRecipe = psMetadataLookupPtr(NULL, config->recipes, PSPHOT_RECIPE); + psAssert(psphotRecipe, "Need PSPHOT recipe"); + + // XXX Should this be options->maskValue or options->maskValue & ~options->satMask? + // The latter will leave saturated pixels high + psImageMaskType maskVal = options->maskValue; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psMetadataAddImageMask(psphotRecipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "user-defined mask", maskVal); + + // Since we are working on a chip-mosaicked image, there should only be a single cell and readout + pmChip *chip = pmFPAviewThisChip(view, input->fpa); // Chip of interest + if (!chip) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find chip"); + return false; + } + if (chip->cells->n == 0) { + psWarning("Chip has no cells"); + return true; + } + if (chip->cells->n > 1) { + psWarning("Chip has %ld cells; only the first will be processed", chip->cells->n); + } + pmCell *cell = chip->cells->data[0]; // Cell of interest + if (!cell || !cell->process || !cell->file_exists) { + // Nothing to process + return true; + } + if (cell->readouts->n == 0) { + psWarning("Cell has no readouts"); + return true; + } + if (cell->readouts->n > 1) { + psWarning("Cell has %ld readouts; only the first will be processed", cell->readouts->n); + } + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + if (!ro || !ro->data_exists) { + // Nothing to process + return true; + } + psImage *image = ro->image, *mask = ro->mask; // Image and mask of interest + + // View corresponding to this readout + pmFPAview roView = *view; + roView.cell = roView.readout = 0; + + // If the background model file has not been defined, psphotModelBackground will generate it + pmReadout *modelRO = NULL; + pmFPAfile *modelFile = psMetadataLookupPtr(&status, config->files, "PSPHOT.BACKMDL"); // Background model + if (modelFile && modelFile->fpa) { + modelRO = pmFPAviewThisReadout(&roView, modelFile->fpa); // Background model + } + + // the background model has not been defined, or at least not generated + if (!modelFile || !modelRO) { + if (!psphotModelBackgroundReadoutFileIndex(config, &roView, "PPIMAGE.CHIP", 0)) { + int lastError = psErrorCodeLast(); + if (lastError == PSPHOT_ERR_DATA) { + // a data error in psphotModelBackground* means an empty or bad image: skip background subtraction + psErrorStackPrint(stderr, "Unable to model background"); + psErrorClear(); + return true; + } else { + psError(PS_ERR_UNKNOWN, false, "Unable to model background"); + return false; + } + } + // the model file should now at least be defined + modelFile = psMetadataLookupPtr(&status, config->files, "PSPHOT.BACKMDL"); // Background model + if (!modelFile) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to define background model I/O file"); + return false; + } + // now grab the readout from the correct location: + if (modelFile->mode == PM_FPA_MODE_INTERNAL) { + modelRO = modelFile->readout; + } else { + modelRO = pmFPAviewThisReadout(&roView, modelFile->fpa); + } + if (!modelRO) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background model"); + return false; + } + } + psImageBinning *binning = psMetadataLookupPtr(&status, modelRO->analysis, + "PSPHOT.BACKGROUND.BINNING"); // Binning for model + if (!binning) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background binning"); + return false; + } + +# define USE_UNBIN 1 +# if (USE_UNBIN) + // select background pixels, from output background file, or create + pmReadout *background = NULL; + pmFPAfile *backfile = psMetadataLookupPtr (&status, config->files, "PSPHOT.BACKGND"); + if (backfile) { + // we are using PSPHOT.BACKGND as an I/O file: select readout or create + if (backfile->mode == PM_FPA_MODE_INTERNAL) { + background = backfile->readout; + } else { + background = pmFPAviewThisReadout (&roView, backfile->fpa); + } + if (background == NULL) { + // readout does not yet exist: create from input + pmFPAfileCopyStructureView (backfile->fpa, input->fpa, 1, 1, &roView); + background = pmFPAviewThisReadout (&roView, backfile->fpa); + if ((image->numCols != background->image->numCols) || (image->numRows != background->image->numRows)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for background dimensions"); + return false; + } + } + } else { + background = pmFPAfileDefineInternal (config->files, "PSPHOT.BACKGND", image->numCols, image->numRows, PS_TYPE_F32); + } + psF32 **backData = background->image->data.F32; + + // linear interpolation to full-scale + if (!psImageUnbin (background->image, modelRO->image, binning)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning"); + return false; + } + + // Do the background subtraction + int numCols = image->numCols, numRows = image->numRows; // Size of image + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if(options->doApplyPixelZero) { + if (mask && mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal) { + image->data.F32[y][x] = 0.0; + } else { + float value = backData[y][x]; + if (!isfinite(value)) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->lowMask; + } else { + image->data.F32[y][x] -= value; + } + } + } else { + float value = backData[y][x]; + if (!isfinite(value)) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->lowMask; + } else { + image->data.F32[y][x] -= value; + } + } + } + } +# else + // Do the background subtraction + int numCols = image->numCols, numRows = image->numRows; // Size of image + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if(options->doApplyPixelZero) { + if (mask && mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal) { + image->data.F32[y][x] = 0.0; + } else { + float value = psImageUnbinPixel(x + 0.5, y + 0.5, modelRO->image, binning); // Background value + if (!isfinite(value)) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->lowMask; + } else { + image->data.F32[y][x] -= value; + } + } + } else { + float value = psImageUnbinPixel(x + 0.5, y + 0.5, modelRO->image, binning); // Background value + if (!isfinite(value)) { + image->data.F32[y][x] = NAN; + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->lowMask; + } else { + image->data.F32[y][x] -= value; + } + } + } + } +# endif + + // XXX should these really be here?? (probably not...) + // pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL"); + // pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL.STDEV"); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageSetMaskBits.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageSetMaskBits.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageSetMaskBits.c (revision 42655) @@ -0,0 +1,187 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// Structure to hold the properties of a mask value +typedef struct { + char *badMaskName; // name for "bad" (i.e., mask me please) pixels + char *fallbackName; // Fallback name in case a bad mask name is not defined + psImageMaskType defaultMaskValue; // Default value in case a bad mask name and its fallback are not defined + bool isBad; // include this value as part of the MASK.VALUE entry (generically bad) +} pmConfigMaskInfo; + +static pmConfigMaskInfo ppimagemasks[] = { + // Features of the detector + { "DETECTOR", NULL, 0x01, true }, // Something is wrong with the detector + { "FLAT", "DETECTOR", 0x01, true }, // Pixel doesn't flat-field properly + { "DARK", "DETECTOR", 0x01, true }, // Pixel doesn't dark-subtract properly + { "BLANK", "DETECTOR", 0x01, true }, // Pixel doesn't contain valid data + { "CTE", "DETECTOR", 0x01, false }, // Pixel has poor CTE + { "BURNTOOL", NULL, 0x04, false }, // Pixel has been touched by burntool + // Invalid signal ranges + { "SAT", NULL, 0x02, true }, // Pixel is saturated or non-linear + { "LOW", "SAT", 0x02, true }, // Pixel is low + { "SUSPECT", NULL, 0x04, false }, // Pixel is suspected of being bad + // Non-astronomical structures + { "CR", NULL, 0x08, true }, // Pixel contains a cosmic ray + { "SPIKE", NULL, 0x08, false }, // Pixel contains a diffraction spike + { "GHOST", NULL, 0x08, false }, // Pixel contains an optical ghost + { "STREAK", NULL, 0x08, false }, // Pixel contains a streak + { "CROSSTALK", NULL, 0x08, false }, // Pixel contains crosstalk data + { "STARCORE", NULL, 0x08, false }, // Pixel contains a bright star core + // Effects of convolution and interpolation + { "CONV.BAD", NULL, 0x02, true }, // Pixel is bad after convolution with a bad pixel + { "CONV.POOR", NULL, 0x04, false }, // Pixel is poor after convolution with a bad pixel +}; + +bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options) { + + + // Look up recipe values + psMetadata *pprecipe = psMetadataLookupMetadata(NULL, config->recipes, RECIPE_NAME); + + psAssert(pprecipe, "We checked this earlier, so it should be here."); + bool doDetectCTE = psMetadataLookupBool(NULL, pprecipe, "DETECT.CTE"); // Do detections on pixels underneath CTE masks + + // this function sets the required single-image mask bits + if(!doDetectCTE) { + if (!pmConfigMaskSetBits (&options->maskValue, &options->markValue, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + } else { + psMetadata *maskrecipe = psMetadataLookupMetadata(NULL, config->recipes, "MASKS"); // The recipe + if (!maskrecipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find MASKS recipe."); + return false; + } + if (!ppImageMaskSetInMetadata(&options->maskValue, &options->markValue, maskrecipe)) { + psError (PS_ERR_UNKNOWN, true, "Unable to determine the mask value"); + return false; + } + } + + // psTimerStart("set.mask.bits"); + + // at this point we know we have valid values for required entries SAT, BAD, FLAT, BLANK: + + // mask for bad flat corrections + options->flatMask = pmConfigMaskGet("FLAT", config); + psAssert (options->flatMask, "flat mask not set"); + + // mask for bad dark corrections + options->darkMask = pmConfigMaskGet("DARK", config); + psAssert (options->darkMask, "dark mask not set"); + + // mask for non-existent data (default to DETECTOR if not defined) + options->blankMask = pmConfigMaskGet("BLANK", config); + psAssert (options->blankMask, "blank mask not set"); + if (options->overscan) { + options->overscan->maskVal = options->blankMask; + } + + // mask for saturated data (default to RANGE if not defined) + options->satMask = pmConfigMaskGet("SAT", config); + psAssert (options->satMask, "sat mask not set"); + + // mask for below-range data (default to RANGE if not defined) + options->lowMask = pmConfigMaskGet("LOW", config); + if (!options->lowMask) { + // look up old name for backward compatability + options->lowMask = pmConfigMaskGet("BAD", config); + } + psAssert (options->lowMask, "low mask not set"); + + // mask for suspect regions due to burntool if we need to. + if (options->doMaskBurntool) { + options->burntoolMask = pmConfigMaskGet("BURNTOOL",config); + psAssert (options->burntoolMask, "burntool mask not set"); + } + + // save MASK and MARK on the PSPHOT recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + // set maskValue and markValue in the psphot recipe + psMetadataAddImageMask(recipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "user-defined mask", + options->maskValue); + psMetadataAddImageMask(recipe, PS_LIST_TAIL, "MARK.PSPHOT", PS_META_REPLACE, "user-defined mask", + options->markValue); + + // psLogMsg ("ppImage", 5, "set mask bits: %f sec\n", psTimerMark ("set.mask.bits")); + + return true; +} + + +bool ppImageMaskSetInMetadata(psImageMaskType *outMaskValue, // Value of MASK.VALUE, returned + psImageMaskType *outMarkValue, // Value of MARK.VALUE, returned + psMetadata *source // Source of mask bits + ) +{ + PS_ASSERT_METADATA_NON_NULL(source, false); + + // Ensure all the bad mask names exist, and set the value to catch all bad pixels + psImageMaskType maskValue = 0; // Value to mask to catch all the bad pixels + psImageMaskType allMasks = 0; // Value to mask to catch all masked bits (to set MARK) + + int nMasks = sizeof (ppimagemasks) / sizeof (pmConfigMaskInfo); + + for (int i = 0; i < nMasks; i++) { + bool mdok; // Status of MD lookup + psImageMaskType value = psMetadataLookupImageMaskFromGeneric(&mdok, source, ppimagemasks[i].badMaskName); // Value of mask + if (!mdok) { + psWarning ("problem with mask value %s\n", ppimagemasks[i].badMaskName); + } + + if (!value) { + if (ppimagemasks[i].fallbackName) { + value = psMetadataLookupImageMaskFromGeneric(&mdok, source, ppimagemasks[i].fallbackName); + } + if (!value) { + value = ppimagemasks[i].defaultMaskValue; + } + psMetadataAddImageMask(source, PS_LIST_TAIL, ppimagemasks[i].badMaskName, PS_META_REPLACE, NULL, value); + } + if (ppimagemasks[i].isBad) { + maskValue |= value; + } + allMasks |= value; + } + + // search for an unset bit to use for MARK: + psImageMaskType markValue = 0x00; + psImageMaskType markTrial = 0x01; + + int nBits = sizeof(psImageMaskType) * 8; + for (int i = 0; !markValue && (i < nBits); i++) { + if (allMasks & markTrial) { + markTrial <<= 1; + } else { + markValue = markTrial; + } + } + if (!markValue) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the MARK bit mask: all bits taken!"); + return false; + } + + // update the list with the results + psMetadataAddImageMask(source, PS_LIST_TAIL, "MASK.VALUE", PS_META_REPLACE, NULL, maskValue); + psMetadataAddImageMask(source, PS_LIST_TAIL, "MARK.VALUE", PS_META_REPLACE, NULL, markValue); + + if (outMaskValue) { + *outMaskValue = maskValue; + } + if (outMarkValue) { + *outMarkValue = markValue; + } + + return true; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImageSetThreads.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageSetThreads.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageSetThreads.c (revision 42655) @@ -0,0 +1,49 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +bool ppImageThread_ppImageDetrendReadout (psThreadJob *job) { + + pmConfig *config = job->args->data[0]; + ppImageOptions *options = job->args->data[1]; + pmFPAview *view = job->args->data[2]; + + bool status = ppImageDetrendReadout(config, options, view); + return status; +} + +bool ppImageThread_ppImageDetrendPatternApplyCell (psThreadJob *job) { + + pmConfig *config = job->args->data[0]; + pmFPA *fpa = job->args->data[1]; + pmChip *chip = job->args->data[2]; + pmCell *cell = job->args->data[3]; + pmFPAview *view = job->args->data[4]; + ppImageOptions *options = job->args->data[5]; + + bool status = ppImageDetrendPatternApplyCell (config, fpa, chip, cell, view, options); + return status; +} + +bool ppImageSetThreads (void) { + +# if (0) + // *** Detrend Readout (now unused : threading is done at the readout level in pmDetrend) + { + psThreadTask *task = psThreadTaskAlloc ("PPIMAGE_DETREND_READOUT", 3); + task->function = &ppImageThread_ppImageDetrendReadout; + psThreadTaskAdd (task); + } +# endif + + // *** Detrend Pattern Row + { + psThreadTask *task = psThreadTaskAlloc("PPIMAGE_PATTERN_ROW_CELL", 6); + task->function = &ppImageThread_ppImageDetrendPatternApplyCell; + psThreadTaskAdd(task); + psFree(task); + } + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageSquashNANs.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageSquashNANs.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageSquashNANs.c (revision 42655) @@ -0,0 +1,67 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +# define LO_16BIT -32768 +# define HI_16BIT 32768 +# define MX_16BIT 65536 + +bool ppImageSquashNANsImage (psImage *image, float minValue, float maxValue, float offset); + +bool ppImageSquashNANs(pmConfig *config, ppImageOptions *options, pmFPAview *view) +{ + // this step is completely optional. + if (!options->squashNANs) { + return true; + } + + psTimerStart("squash.NANs"); + + // find the currently selected readout + pmReadout *readout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT"); + + psImage *image = readout->image; // Readout's image + + // We are going to force the range to be (-999 to 2^16 - 1002, ie, 1 less on each end than 2^16. this allows for float round errors + int bzero = LO_16BIT - 0; + float minValue = HI_16BIT + bzero + 1; + float maxValue = MX_16BIT + minValue - 3; + + if (image->parent) { + psImage *parent = (psImage *) image->parent; + ppImageSquashNANsImage (parent, minValue, maxValue, 500.0); + } else { + ppImageSquashNANsImage (image, minValue, maxValue, 500.0); + + // squash NANs in overscan regions + psList *overscans = readout->bias; // List of the overscan images + psListIterator *iter = psListIteratorAlloc(overscans, PS_LIST_HEAD, false); // Iterator + psImage *overscan = NULL; // Overscan image from iterator + while ((overscan = psListGetAndIncrement(iter))) { + ppImageSquashNANsImage (overscan, minValue, maxValue, 500.0); + } + psFree(iter); + } + + psLogMsg ("ppImage", 5, "squash NANs: %f sec\n", psTimerMark ("squash.NANs")); + return true; +} + +bool ppImageSquashNANsImage (psImage *image, float minValue, float maxValue, float offset) { + + int numCols = image->numCols, numRows = image->numRows; // Size of image + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (!isfinite(image->data.F32[y][x])) { + image->data.F32[y][x] = 1.0; + } else { + image->data.F32[y][x] = PS_MAX (minValue, PS_MIN (maxValue, image->data.F32[y][x] + 500.0)); + } + } + } + return true; +} + Index: /branches/eam_branches/ppImage.20240412/src/ppImageStatsOutput.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageStatsOutput.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageStatsOutput.c (revision 42655) @@ -0,0 +1,49 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" + +// write stats to output file +bool ppImageStatsOutput(pmConfig *config, psMetadata *stats, const ppImageOptions *options) +{ + bool mdok; + + // measure statistics, or ignore? + if (!options->doStats) { + return true; + } + + // get the output stats filename + const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics + if (!statsName || !strlen(statsName)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "missing STATS entry in arguments list."); + return false; + } + + // Write out + psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename + if (!resolved) { + psError(psErrorCodeLast(), false, "Unable to resolve statistics file name"); + return false; + } + + // check for Metadata compression options: + char *compressMode = NULL; + bool status = false; + if (config->camera) { + // XXX use a different config variable for this output? + compressMode = psMetadataLookupStr(&status, config->camera, "METADATA.COMPRESSION"); + } + + if (!psMetadataConfigWrite(stats, resolved, compressMode)) { + psError(psErrorCodeLast(), false, "Unable to serialize stats metadata.\n"); + psFree(resolved); + return false; + } + psFree(resolved); + + pmConfigRunFilenameAddWrite(config, "STATS", statsName); + + return true; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageVersion.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageVersion.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageVersion.c (revision 42655) @@ -0,0 +1,115 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "ppImage.h" +#include "ppImageVersionDefinitions.h" + +#ifndef PPIMAGE_VERSION +#error "PPIMAGE_VERSION is not set" +#endif +#ifndef PPIMAGE_BRANCH +#error "PPIMAGE_BRANCH is not set" +#endif +#ifndef PPIMAGE_SOURCE +#error "PPIMAGE_SOURCE is not set" +#endif + +psString ppImageVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PPIMAGE_BRANCH, PPIMAGE_VERSION); + return value; +} + +psString ppImageSource(void) +{ + return psStringCopy(PPIMAGE_SOURCE); +} + +psString ppImageVersionLong(void) +{ + psString version = ppImageVersion(); // Version, to return + psString source = ppImageSource(); // Source + + psStringPrepend(&version, "ppImage "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + + return version; +}; + +bool ppImageVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "ppImage at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + psModulesVersionHeader(header); + psphotVersionHeader(header); + psastroVersionHeader(header); + ppStatsVersionHeader(header); + + psString version = ppImageVersion(); // ppImage software version + psString source = ppImageSource(); // ppImage software source + + psMetadataAddStr(header, PS_LIST_TAIL, "IMAGE_V", PS_META_REPLACE, NULL, PPIMAGE_VERSION); + + psStringPrepend(&version, "ppImage version: "); + psStringPrepend(&source, "ppImage source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + + return true; +} + + +void ppImageVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("ppImage", PS_LOG_INFO, "ppImage at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString psphot = psphotVersionLong(); // psphot version + psString psastro = psastroVersionLong(); // psastro version + psString ppStats = ppStatsVersionLong(); // ppStats version + psString ppImage = ppImageVersionLong(); // ppImage version + + psLogMsg("ppImage", PS_LOG_INFO, "%s", pslib); + psLogMsg("ppImage", PS_LOG_INFO, "%s", psmodules); + psLogMsg("ppImage", PS_LOG_INFO, "%s", psphot); + psLogMsg("ppImage", PS_LOG_INFO, "%s", psastro); + psLogMsg("ppImage", PS_LOG_INFO, "%s", ppStats); + psLogMsg("ppImage", PS_LOG_INFO, "%s", ppImage); + + psFree(pslib); + psFree(psmodules); + psFree(psphot); + psFree(psastro); + psFree(ppStats); + psFree(ppImage); + + return; +} Index: /branches/eam_branches/ppImage.20240412/src/ppImageVersionDefinitions.h.in =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppImageVersionDefinitions.h.in (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppImageVersionDefinitions.h.in (revision 42655) @@ -0,0 +1,8 @@ +#ifndef PPIMAGE_VERSION_DEFINITIONS_H +#define PPIMAGE_VERSION_DEFINITIONS_H + +#define PPIMAGE_VERSION @PPIMAGE_VERSION@ // SVN version +#define PPIMAGE_BRANCH @PPIMAGE_BRANCH@ // SVN branch +#define PPIMAGE_SOURCE @PPIMAGE_SOURCE@ // SVN source + +#endif Index: /branches/eam_branches/ppImage.20240412/src/ppTest.c =================================================================== --- /branches/eam_branches/ppImage.20240412/src/ppTest.c (revision 42655) +++ /branches/eam_branches/ppImage.20240412/src/ppTest.c (revision 42655) @@ -0,0 +1,190 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pslib.h" +#include "psmodules.h" + +int main(int argc, char *argv[]) +{ + psLibInit(NULL); + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ppImageConfig.c +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + pmConfig *config = pmConfigRead(&argc, argv, "PPIMAGE"); + if (! config) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + // Parse other command-line arguments + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-key", 0, "exposure ID", ""); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-bias", 0, "Name of the bias image", ""); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-dark", 0, "Name of the dark image", ""); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-flat", 0, "Name of the flat-field image", ""); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-mask", 0, "Name of the mask image", ""); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-fringe", 0, "Name of the fringe image", ""); + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "-chip", 0, "Chip number to process (if positive)", -1); + + if (! psArgumentParse(config->arguments, &argc, argv) || argc != 3) { + printf("\nPan-STARRS Phase 2 processing\n\n"); + printf("Usage: %s INPUT.fits OUTPUT.fits\n\n", argv[0]); + psArgumentHelp(config->arguments); + psFree(config->arguments); + exit(EXIT_FAILURE); + } + + // Add the input and output images (which remain on the command-line) to the arguments list + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-input", 0, "Name of the input image", argv[1]); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "-output", 0, "Name of the output image", argv[2]); + + // Define database handle, if used +#if 0 + config->database = pmConfigDB(config->site); +#endif + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ppImageParseCamera.c extract with some alterations +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + const char *inName = psMetadataLookupStr(NULL, config->arguments, "-input"); + psLogMsg("ppImage", PS_LOG_INFO, "Opening input image: %s\n", inName); + psFits *inFile = psFitsOpen(inName, "r"); // File handle for FITS file + if (! inFile) { + // There's no point in continuing if we can't open the input + psErrorStackPrint(stderr, "Can't open input image: %s\n", inName); + exit(EXIT_FAILURE); + } + psMetadata *phu = psFitsReadHeader(NULL, inFile); // FITS primary header + + psMetadata *cameraFormat = pmConfigCameraFormatFromHeader(NULL, NULL, config, phu, true); + if (! config->camera) { + cameraFormat = pmConfigCameraFormatFromHeader(NULL, NULL, config, phu, true); + if (! config->camera) { + // There's no point in continuing if we can't recognise what we've got + psErrorStackPrint(stderr, "Can't find camera configuration!\n"); + exit(EXIT_FAILURE); + } + } + // Determine the correct recipe to use + if (! config->recipes && !pmConfigReadRecipes(config, PM_RECIPE_SOURCE_CAMERA | PM_RECIPE_SOURCE_CL)) { + // There's no point in continuing if we can't work out what recipes to use + psErrorStackPrint(stderr, "Can't find recipe configuration!\n"); + exit(EXIT_FAILURE); + } + +#if 1 + const char *outName = psMetadataLookupStr(NULL, config->arguments, "-output"); + psLogMsg("ppImage", PS_LOG_INFO, "Opening output image: %s\n", outName); + psFits *outFile = psFitsOpen(outName, "w"); + if (!outFile) { + // There's no point in continuing if we can't open the output + psErrorStackPrint(stderr, "Can't open output image: %s\n", outName); + exit(EXIT_FAILURE); + } +#endif + + // Construct camera in preparation for reading + pmFPA *fpa = pmFPAConstruct(config->camera, config->cameraName); + pmFPAview *view = pmFPAAddSourceFromHeader(fpa, phu, cameraFormat); + printf("View chip: %d\n", view->chip); + printf("View cell: %d\n", view->cell); + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// The action happens here +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#if 0 + pmFPAPrint(stdout, fpa, false, false); + exit(0); +#endif + +#if 0 + // A mosaic + psMetadata *mosaicCamera = psMetadataConfigRead(NULL, NULL, "/home/mithrandir/price/ipp/config/mcshort_mosaic/camera.config", true); + pmFPA *mosaicFPA = pmFPAConstruct(mosaicCamera); + psMetadata *mosaicFormat = psMetadataConfigRead(NULL, NULL, "/home/mithrandir/price/ipp/config/mcshort_mosaic/format_mosaic.config", true); + pmFPAview *mosaicView = pmFPAviewAlloc(0); + pmFPAAddSourceFromView(mosaicFPA, mosaicView, mosaicFormat); + psFree(mosaicView); +#endif + + // Read the FPA + pmFPARead(fpa, inFile, NULL); + +#if 1 + psArray *chips = fpa->chips; + pmFPAWrite(fpa, outFile, NULL, true, false); + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; +#if 1 + //pmChipRead(chip, inFile, NULL); + pmChipWrite(chip, outFile, NULL, true, false); + psArray *cells = chip->cells; + for (int j = 0; j < cells->n; j++) { + pmCell *cell = cells->data[j]; + pmCellWrite(cell, outFile, NULL, true); +#if 0 // Read bit by bit + pmReadout *readout = pmReadoutAlloc(cell); + for (int z = 0; pmReadoutReadNext(readout, inFile, z, 512); z++) { + do { + printf("Chip %d, Cell %d, Plane %d, row0 = %d\n", i, j, z, readout->row0); + pmReadoutWriteNext(readout, outFile, z); + //pmFPAPrint(stdout, fpa, false, true); + } while (pmReadoutReadNext(readout, inFile, z, 512)); + } + psFree(readout); +#else // Read the whole cell at once + // pmCellRead(cell, inFile, NULL); + // pmCellWrite(cell, outFile, NULL, false); + // pmCellFreeData(cell); +#endif + } + pmChipWrite(chip, outFile, NULL, false, false); +#endif + +#if 0 + pmChipMosaic(mosaicFPA->chips->data[i], chip); + pmChipWrite(mosaicFPA->chips->data[i], outFile, NULL, true, true); + pmChipFreeData(chip); + pmChipFreeData(mosaicFPA->chips->data[i]); +#endif + } + pmFPAWrite(fpa, outFile, NULL, false, false); +#endif + +#if 0 + pmFPAMosaic(mosaicFPA, fpa); + pmFPAWrite(mosaicFPA, outFile, NULL, true, true); + //pmFPAPrint(stdout, mosaicFPA, true, true); + psFree(mosaicCamera); + psFree(mosaicFormat); + psFree(mosaicView); + //psFree(mosaicFPA); +#endif + +#if 1 + pmFPAPrint(stdout, fpa, false, false); +#endif + +#if 1 + psFitsClose(outFile); +#endif + + psFree(view); + psFree(phu); + psFree(fpa); + psFree(config); + psFitsClose(inFile); + psFree(cameraFormat); + + pmConceptsDone(); + pmConfigDone(); + psLibFinalize(); + + // Pau. +}