Index: /branches/ccl_branches/ipponly-20191108/psphot/Doxyfile.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/Doxyfile.in (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/Doxyfile.in (revision 41083) @@ -0,0 +1,1310 @@ +# Doxyfile 1.5.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file that +# follow. The default is UTF-8 which is also the encoding used for all text before +# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into +# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of +# possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = psphot + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = ipp-2.7.dev + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, +# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, +# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, +# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is +# documented as struct with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code where the coding convention is that all structs are +# typedef'ed and only the typedef is referenced never the struct's name. + +TYPEDEF_HIDES_STRUCT = NO + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be extracted +# and appear in the documentation as a namespace called 'anonymous_namespace{file}', +# where file will be replaced with the base name of the file that contains the anonymous +# namespace. By default anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text " + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ./src + +# This tag can be used to specify the character encoding of the source files that +# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default +# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. +# See http://www.gnu.org/software/libiconv for the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = *.h + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the output. +# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, +# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH +# then you must also enable this option. If you don't then doxygen will produce +# a warning and turn it on anyway + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to +# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to +# specify the directory where the mscgen tool resides. If left empty the tool is assumed to +# be found in the default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the number +# of direct children of the root node in a graph is already larger than +# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO Index: /branches/ccl_branches/ipponly-20191108/psphot/Makefile.am =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/Makefile.am (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/Makefile.am (revision 41083) @@ -0,0 +1,12 @@ +SUBDIRS = src + +CLEANFILES = *.pyc *~ core core.* + +bin_SCRIPTS = psphot-config +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA= psphot.pc + +EXTRA_DIST = \ + psphot-config.in \ + psphot.pc.in \ + autogen.sh Index: /branches/ccl_branches/ipponly-20191108/psphot/autogen.sh =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/autogen.sh (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/autogen.sh (revision 41083) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=psphot +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOLIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/ccl_branches/ipponly-20191108/psphot/configure.ac =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/configure.ac (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/configure.ac (revision 41083) @@ -0,0 +1,221 @@ +dnl Process this file with autoconf to produce a configure script. +AC_PREREQ(2.61) + +AC_INIT([psphot], [0.9.0], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDLDFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +dnl ------------------------------------------------------------ + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +dnl ------------------ kapa,libkapa options ------------------------- +dnl -- libkapa implies the requirement for libpng, libjpeg as well -- + +dnl save LIBS/CFLAGS/LDFLAGS +TMP_LIBS=${LIBS} +TMP_CFLAGS=${CFLAGS} +TMP_LDFLAGS=${LDFLAGS} +TMP_CPPFLAGS=${CPPFLAGS} + +dnl test for command-line options: use ohana-config if not supplied +KAPA_CFLAGS_CONFIG="true" +KAPA_LIBS_CONFIG="true" +AC_ARG_WITH(kapa, +[AS_HELP_STRING(--with-kapa=DIR,Specify location of libkapa)], +[KAPA_CFLAGS="-I$withval/include" KAPA_LIBS="-L$withval/lib" + KAPA_CFLAGS_CONFIG="false" KAPA_LIBS_CONFIG="false"]) +AC_ARG_WITH(kapa-include, +[AS_HELP_STRING(--with-kapa-include=DIR,Specify libkapa include directory.)], +[KAPA_CFLAGS="-I$withval" KAPA_CFLAGS_CONFIG="false"]) +AC_ARG_WITH(kapa-lib, +[AS_HELP_STRING(--with-kapa-lib=DIR,Specify libkapa library directory.)], +[KAPA_LIBS="-L$withval" KAPA_LIBS_CONFIG="false"]) + +echo "KAPA_CFLAGS_CONFIG: $KAPA_CFLAGS_CONFIG" +echo "KAPA_LIBS_CONFIG: $KAPA_LIBS_CONFIG" +echo "KAPA_CFLAGS: $KAPA_CFLAGS" +echo "KAPA_LIBS: $KAPA_LIBS" + +dnl HAVE_KAPA is set to false if any of the tests fail +HAVE_KAPA="true" +AC_MSG_NOTICE([checking for libkapa]) +if test "$KAPA_CFLAGS_CONFIG" = "true" -o "$KAPA_LIBS_CONFIG" = "true"; then + AC_MSG_NOTICE([kapa info supplied by ohana-config]) + KAPA_CONFIG=`which ohana-config` + AC_CHECK_FILE($KAPA_CONFIG,[], + [HAVE_KAPA="false"; AC_MSG_WARN([libkapa is not found: output plots disabled. Obtain libkapa at http://kiawe.ifa.hawaii.edu/Elixir/Ohana or use --with-kapa to specify location])]) + + echo "HAVE_KAPA: $HAVE_KAPA" + echo "KAPA_CFLAGS_CONFIG: $KAPA_CFLAGS_CONFIG" + + if test "$HAVE_KAPA" = "true" -a "$KAPA_CFLAGS_CONFIG" = "true" ; then + AC_MSG_NOTICE([libkapa cflags info supplied by ohana-config]) + AC_MSG_CHECKING([libkapa cflags]) + KAPA_CFLAGS="`${KAPA_CONFIG} --cflags`" + AC_MSG_RESULT([${KAPA_CFLAGS}]) + fi + + if test "$HAVE_KAPA" = "true" -a "$KAPA_LIBS_CONFIG" = "true" ; then + AC_MSG_NOTICE([libkapa ldflags info supplied by ohana-config]) + AC_MSG_CHECKING([libkapa ldflags]) + KAPA_LIBS="`${KAPA_CONFIG} --libs` -lX11" + AC_MSG_RESULT([${KAPA_LIBS}]) + fi +fi + +if test "$HAVE_KAPA" = "true" ; then + AC_MSG_NOTICE([libkapa supplied]) + PSPHOT_CFLAGS="${PSPHOT_CFLAGS} ${KAPA_CFLAGS}" + PSPHOT_LIBS="${PSPHOT_LIBS} ${KAPA_LIBS}" +else + AC_MSG_NOTICE([libkapa ignored]) +fi + +dnl HAVE_KAPA is set to false if any of the tests fail +dnl HAVE_KAPA=true +dnl AC_CHECK_HEADERS([kapa.h], +dnl [PSPHOT_CFLAGS="$PSPHOT_CFLAGS $KAPA_CFLAGS" AC_SUBST(KAPA_CFLAGS)], +dnl [HAVE_KAPA=false; AC_MSG_WARN([libkapa headers not found: output plots disabled. Obtain libkapa at http://kiawe.ifa.hawaii.edu/Elixir/Ohana or use --with-kapa to specify location.])] +dnl ) +dnl AC_CHECK_LIB(kapa,KapaInitGraph, +dnl [PSPHOT_LIBS="$PSPHOT_LIBS $JPEG_LDFLAGS -ljpeg"], +dnl [HAVE_KAPA=false; AC_MSG_WARN([libkapa headers not found: output plots disabled. Obtain libkapa at http://kiawe.ifa.hawaii.edu/Elixir/Ohana or use --with-kapa to specify location.])],[-lm] +dnl ) + +dnl restore the CFLAGS/LDFLAGS +LIBS=${TMP_LIBS} +CFLAGS=${TMP_CFLAGS} +LDFLAGS=${TMP_LDFLAGS} +CPPFLAGS=${TMP_CPPFLAGS} + +dnl ------------------ libjpeg options --------------------- + +dnl save LIBS/CFLAGS/LDFLAGS +TMP_LIBS=${LIBS} +TMP_CFLAGS=${CFLAGS} +TMP_LDFLAGS=${LDFLAGS} +TMP_CPPFLAGS=${CPPFLAGS} + +AC_ARG_WITH(jpeg, +[AS_HELP_STRING(--with-jpeg=DIR,Specify location of libjpeg.)], +[JPEG_CFLAGS="-I$withval/include" + JPEG_LDFLAGS="-L$withval/lib"]) +AC_ARG_WITH(jpeg-include, +[AS_HELP_STRING(--with-jpeg-include=DIR,Specify libjpeg include directory.)], +[JPEG_CFLAGS="-I$withval"]) +AC_ARG_WITH(jpeg-lib, +[AS_HELP_STRING(--with-jpeg-lib=DIR,Specify libjpeg library directory.)], +[JPEG_LDFLAGS="-L$withval"]) + +CFLAGS="${CFLAGS} ${JPEG_CFLAGS}" +CPPFLAGS=${CFLAGS} +LDFLAGS="${LDFLAGS} ${JPEG_LDFLAGS}" + +AC_CHECK_HEADERS([jpeglib.h], + [PSPHOT_CFLAGS="$PSPHOT_CFLAGS $JPEG_CFLAGS" AC_SUBST(JPEG_CFLAGS)], + [HAVE_KAPA=false; AC_MSG_WARN([libjpeg headers not found: output plots disabled. Obtain libjpeg from http://www.ijg.org/ or use --with-jpeg to specify location.])] +) + +AC_CHECK_LIB(jpeg,jpeg_CreateCompress, + [PSPHOT_LIBS="$PSPHOT_LIBS $JPEG_LDFLAGS -ljpeg"], + [HAVE_KAPA=false; AC_MSG_WARN([libjpeg library not found: output plots disabled. Obtain libjpeg from http://www.ijg.org/ or use --with-jpeg to specify location.])] +) + +dnl restore the CFLAGS/LDFLAGS +LIBS=${TMP_LIBS} +CFLAGS=${TMP_CFLAGS} +LDFLAGS=${TMP_LDFLAGS} +CPPFLAGS=${TMP_CPPFLAGS} + +dnl ------------------ libpng options --------------------- + +dnl save LIBS/CFLAGS/LDFLAGS +TMP_LIBS=${LIBS} +TMP_CFLAGS=${CFLAGS} +TMP_LDFLAGS=${LDFLAGS} +TMP_CPPFLAGS=${CPPFLAGS} + +AC_ARG_WITH(png, +[AS_HELP_STRING(--with-png=DIR,Specify location of libpng.)], +[PNG_CFLAGS="-I$withval/include" + PNG_LDFLAGS="-L$withval/lib"]) +AC_ARG_WITH(png-include, +[AS_HELP_STRING(--with-png-include=DIR,Specify libpng include directory.)], +[PNG_CFLAGS="-I$withval"]) +AC_ARG_WITH(png-lib, +[AS_HELP_STRING(--with-png-lib=DIR,Specify libpng library directory.)], +[PNG_LDFLAGS="-L$withval"]) + +CFLAGS="${CFLAGS} ${PNG_CFLAGS}" +CPPFLAGS=${CFLAGS} +LDFLAGS="${LDFLAGS} ${PNG_LDFLAGS}" + +AC_CHECK_HEADERS([png.h], + [PSPHOT_CFLAGS="$PSPHOT_CFLAGS $PNG_CFLAGS" AC_SUBST(PNG_CFLAGS)], + [HAVE_KAPA=false; AC_MSG_WARN([libpng headers not found: output plots disabled. Obtain libpng from http://www.ijg.org/ or use --with-png to specify location.])] +) + +AC_CHECK_LIB(png,png_init_io, + [PSPHOT_LIBS="$PSPHOT_LIBS $PNG_LDFLAGS -lpng"], + [HAVE_KAPA=false; AC_MSG_WARN([libpng library not found: output plots disabled. Obtain libpng from http://www.ijg.org/ or use --with-png to specify location.])] +) + +dnl restore the CFLAGS/LDFLAGS +LIBS=${TMP_LIBS} +CFLAGS=${TMP_CFLAGS} +LDFLAGS=${TMP_LDFLAGS} +CPPFLAGS=${TMP_CPPFLAGS} + +dnl ------------------ use kapa or not? --------------------- + +if test "$HAVE_KAPA" == "true" ; then + AC_MSG_RESULT([including plotting functions]) + AC_DEFINE([HAVE_KAPA],[1],[enable use of libkapa]) +else + AC_MSG_RESULT([skipping plotting functions]) + AC_DEFINE([HAVE_KAPA],[0],[disable use of libkapa]) +fi + +dnl ------------- psLib, psModules --------------- +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) + +dnl Set CFLAGS for build +IPP_STDOPTS +IPP_STDCFLAGS + +echo "PSPHOT_CFLAGS: $PSPHOT_CFLAGS" +echo "PSPHOT_LIBS: $PSPHOT_LIBS" + +IPP_VERSION + +dnl note: the PSPHOT_ entries below do not include PSLIB_ and PSMODULE_ entires +dnl those are passed down to lower Makefiles by the PKG_CHECK_MODULES call above +AC_SUBST([PSPHOT_CFLAGS]) +AC_SUBST([PSPHOT_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile + psphot-config + psphot.pc + Doxyfile +]) + +AC_OUTPUT Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/2Dmodels.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/2Dmodels.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/2Dmodels.txt (revision 41083) @@ -0,0 +1,29 @@ + +I am testing a new concept for the 2D representation of various values +measured by psphot. I have been using 2D polynomials to fit the +variations of, eg, the psf parameters as a function of position in the +images. this has been somewhat successful, but it has also been +fraught with a number of problems. the worst issue has been the fact +that the polynomial form is not physically motivated. in some cases, +eg the variation of the psf shape parameters, the true variations for +real distributions are not well represented by a polynomial. I have +done some ad hoc tricks to get the functional form to look roughly +like a polynomial, but this is really quite a hack. the other problem +is that it is difficult to decide if the polynomial does an acceptable +job representing the variations. + +My new concept for modeling 2D variations in some parameter is to +define a low-resolution 2D image to represent the variation. The +advantage is that the image may be define with whatever resolution may +be sampled by the input data, and the functional form need not be +constrained by the polynomial basis function. + +** is this a poor substitute for using a chebychev polynomial as a + better basis function? + +psphot parameters which vary by position: + +pmPSF.params +pmPSF.ApTrend +pmPSF.FluxScale (flux given normalization of 1.0) + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/config.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/config.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/config.txt (revision 41083) @@ -0,0 +1,56 @@ + +# input data options +# IMAGE STR file.fits +# MASK_IMAGE STR mask.fits +# WEIGHT_IMAGE STR weight.fits + +# output data options +OUTPUT_FILE STR output.smp # output object file +OUTPUT_MODE STR TEXT # output mode: TEXT, OBJ, SX, CMP, CMF +# RESID_IMAGE STR resid.fits # output residual image + +# image noise parameters +RDNOISE STR HD:RDNOISE # read-noise in electrons +GAIN STR HD:GAIN # electrons / DN + +# masking parameters +XMIN F32 32 # minimum valid x-coord +XMAX F32 0 # maximum valid x-coord +YMIN F32 1 # minimum valid y-coord +YMAX F32 0 # maximum valid y-coord +SATURATION F32 50000 # saturation level on this chip + +# image statistics parameters +IMSTATS_NPIX S32 100000 # number of pixels to use for sky estimate: + +# peak finding +PEAKS_SMOOTH_SIGMA F32 1.0 # smoothing kernel sigma in pixels +PEAKS_SMOOTH_NSIGMA F32 3.0 # smoothing kernel width in sigmas +PEAKS_NSIGMA_LIMIT F32 10.0 # peak significance threshold + +# basic object statistics +SKY_INNER_RADIUS F32 15 # square annulus for local sky measurement +SKY_OUTER_RADIUS F32 25 # square annulus for local sky measurement +PSF_MOMENTS_RADIUS F32 5 +PSF_SN_LIM F32 100 + +# PSF model parameters : choose the PSF model +# list as many PSF_MODEL options as desired +PSF_MODEL STR PS_MODEL_QGAUSS +#PSF_MODEL STR PS_MODEL_PGAUSS +#PSF_MODEL STR PS_MODEL_GAUSS +PSF_FIT_RADIUS F32 25 # fitting radius for test PSF model + +# PSF model parameters : apply the PSF model +PSF_FIT_NSIGMA F32 1 # significance for pixel included in fit +PSF_FIT_PADDING F32 5 # extra annulus to use for fit +PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation +PSF_MIN_SN F32 2.0 # reject objects below this significance +PSF_MAX_CHI F32 10.0 # reject objects worse that this + +# Galaxy model parameters +GAL_MODEL STR PS_MODEL_SGAUSS +GAL_MIN_SN F32 3 +GAL_FIT_NSIGMA F32 1 # significance for pixel included in fit +GAL_FIT_PADDING F32 5 # extra annulus to use for fit +GAL_MOMENTS_RADIUS F32 9 Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/efficiency.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/efficiency.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/efficiency.txt (revision 41083) @@ -0,0 +1,51 @@ +Strategy for efficiency analysis (a.k.a. "fake" stage) + +Want: +* Detection efficiency as a function of instrumental magnitude +* Masked fraction + +Given: +* Image +* Mask +* Variance +* PSF +* Sources on image +* Recipe: + + Magnitude bins (relative to guessed detection limit) for fake sources + + Number of fake sources for each bin + + + +Algorithm: +* Determine mean instrumental magnitude detection limit + + Have: + - Mean variance + - Smoothing size + - Covariance + - Threshold + + Calculate mean peak flux of source at threshold + + Integrate PSF to determine magnitude +* Remove all real sources from image +* Add fake sources into image +* Smooth image, variance (maybe only at positions of fake sources?) +* Count number of sources masked +* Count number of fake sources above significance level as a function of fake source magnitude + + +If the required density of fake sources is so high that they begin to +overlap, we will need to do add sources, smooth, and count +independently for each magnitude bin. This could be optimised +using the distributive property of convolutions (f*[g+h]=f*g+f*h) +if we smooth the image first, and then add in smoothed PSFs. + + +For a 5% statistical measurement of the detection efficiency per bin, we want +dN/N = 0.05 ==> N ~ 400 for Poisson statistics. +Assume a 80% masked fraction, so want 500 sources per bin. +For 5 magnitude bins, want 2500 sources +Assume worst case PSF is 15 pix FWHM --> ~ 51 pixels for 4-sigma either side +Total area is 6502500 pixels. +Chip images are 4846x4868 = 23590328 pixels = 3.6 times the worst total area. +Not too bad (especially since this is pretty much worst case, and we only +care about the central pixel, which is much smaller) +==> Let's just throw all sources on the same image. Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/footprints.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/footprints.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/footprints.txt (revision 41083) @@ -0,0 +1,61 @@ + +2010.01.18 + + * we are having some problems with footprints being freed to early in the second pass + + * footprint memory history: + + * entering psphotFindFootprints, on the first pass, we have an + array of detected peak in detections->peaks + + * in pmFootprintsFind, footprints array owns allocated footprints + footprints each own their spans + + * from pmFootprintAssignPeaks: + * the footprints have an array of peaks with valid memory references + * the peaks have just the pointer to the corresponding footprint (not owned) + * all peaks get assigned to a footprint + * at end of pmFootprintAssignPeaks, only a single copy of each peak is/should be available from the collection of footprints + +pmFootprintCullPeaks is very expensive (15 - 30 msec per object) and +must be performed on every peak. We need to improve the speed (even +at the cost of memory) and we need to thread it. + +*** pmFootprints makes use of the mask image, but it does not use the + master mask values. this must be fixed! + +-- here is the algorithm + +** note: this is run once for each footprint... + +* the incoming footprint peak list is already sorted by brightness + +* loop over all peaks + + * choose a threshold which is the peak flux - nsigma_delta*stdev + ? actually, the threshold is using the edge of the containing + subimage? I'm a little confused by this. + + * if the threshold is below a minimum threshold, we drop it + - the code is calling psArrayRemoveIndex for each of these, then decrementing i. + this is inefficient since psArray is not a psList. two possible options here: + + * do not remove the dropped peaks from the array until the full + processing is done, then re-build the array in a single pass. + flag peaks to be dropped (either in the structure or with a + parallel vector) + + * have the code build a separate 'bright peaks' array on the fly + then swap the new peaks out under the pmFootprint. + + * psArrayRemoveIndex is used 3x in pmFootprints.c : review each case + + * the subImage is divided into footprints above threshold about the + current peak. + + - EAM : Need to disect pmFindFootprintsAtPoint. Is it expensive + in this case? Should we be calling the normal pmFootprints + function? + + + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/galaxy.models.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/galaxy.models.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/galaxy.models.txt (revision 41083) @@ -0,0 +1,49 @@ + +psphot uses analytical models to represent stars (PSF model), +galaxies, and trailed asteroids. At certain points in the code, the +analytical models are fitted to the source flux distributions using +chi-square minimization (Levenberg Marquardt Minimization). + +The analytical models have some common features, but there are some +subtleties, especially wrt the way some of the galaxy parameters are +modeled. + +Within the code, the models are represented by the pmModel structure. +The fitted values of these parameters are carried on this structure in +a psVector element. The assignment of the different vector elements +to the different parameters is hard-wired in the code, with the values +of PM_PAR* used to abstract the values (eg, PM_PAR_SKY, etc). + +When the results of fits are written out to the CMF format fits files, +the table field names may differ from the internal parameter names. +In some cases, the model values themselves are modified. + +In the discussion below, I refer to both the PM_PAR_* values and the + +All models have 4 parameters in common: + +* XPOS, YPOS: the centroid position of the object + * for PSF models in the PSF table, these are written as X_PSF, Y_PSF + * for EXT models in the extended model table, these are written as + X_EXT, Y_EXT + +* SKY : the local background. Note, this parameter is in general NOT + fitted in psphot. The local value is subtracted during the sky + background step and the residual is assumed to be zero. The output + tables store the value which was subtracted before the fit took + place. + +* I0 : the model flux normalization. This field is normally not + written out to the FITS table. Instead, the integrated flux or + magnitude (or both) is saved. + +All models, except for the trailed asteroid model, use a common +elliptical contour. The functional form of the model can then be +written as I(x,y) = Io * f(z) where: + +z(x,y) = 0.5(x/\sigma_x)^2 + 0.5(y/\sigma_y)^2 + 0.5 x y \sigma_xy + +The elliptical profile is actually fitted using the following +parameters: + +* SXX, SYY, SXY : For Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20090523.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20090523.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20090523.txt (revision 41083) @@ -0,0 +1,123 @@ + +20090809 : more on extended sources: + + my algorithm for getting the petrosian radii and fluxes is: + + * measure radial profile by interpolation to specific locations along the radial line + (at low radii, this gives much more accurate results; at high radii, we may need to + average a group of pixels -- say, for 15 deg separations, choose a box that is + < 5 degree in width -- this is > 1 pixel at a distance of 12 pixels. + + psphotRadialProfilesByAngle + + * find intersection of profile with isophot + + psphotRadiusFromProfile -> r50(theta) + + * generate r50x, r50y and fit to ellipse + + psphotEllipticalContour + + * generate elliptical profile + + psphotEllipticalProfile + + * convert to \alpha r_i < r < \beta r_i bins + + psphotPetrosianRadialBins + +20090725 : extended source radial profiles + + I need to fix the Petrosian and similar code. There are several issues: + + 1) correct implementation of the actual Petrosian radius / Petrosian flux measurement: + + R_p : Given F(R) : F(0) / F(R_p) = 0.9 ?? (something like this...) + + Actually implemented by calculating the average surface brightness in annullii: + + R_p : Given F(R) -> define f(R) = \sum_{R-dR}^{R+dR}F(R) / 4\pi R dR + R_p : f(dR) / F(R_p) = P_ratio + + 2) need to construct a clean, normalized radial profile to make this measurement + + * generate radial profiles for each of N annular wedges + * find R_50 : f(0) / f(R_50) = 0.5 + $ fit the ellipse of R_50 to Rmaj sin(T) + Rmin cos(T) + + + cos(a+b) = cos(a) cos(b) - sin(a) sin(b) + sin(a+b) = cos(a) sin(b) + cos(b) sin(a) + + cos(theta+To) = cos(theta) cos(To) - sin(theta) sin(To) + sin(theta+To) = cos(theta) sin(To) + sin(theta) cos(To) + + R_x = R_maj cos(theta + theta_o) = R_maj cos(theta) cos(theta_o) - Rmaj sin(theta) sin(theta_o) + R_y = R_min sin(theta + theta_o) = R_min cos(theta) sin(theta_o) + Rmin sin(theta) cos(theta_o) + + R_X = R_x cos(phi) - R_y sin(phi) + R_Y = R_x sin(phi) + R_y cos(phi) + + R_X = R_maj cos(phi) cos(theta + To) - R_min sin(phi) sin(theta + To) + R_Y = R_maj sin(phi) cos(theta + To) + R_min cos(phi) sin(theta + To) + + R_X = (R_maj cos(phi) cos (To) - R_min sin(phi) sin(To)) cos(theta) - (Rmaj cos(phi) sin(To) + Rmin sin(phi) cos(To)) sin(theta) + R_Y = (R_maj sin(phi) cos (To) + R_min cos(phi) sin(To)) cos(theta) - (Rmaj cos(phi) sin(To) + Rmin sin(phi) cos(To)) sin(theta) + + + R_X = Ax cos(theta) + Bx sin(theta) + R_Y = Ay cos(theta) + By sin(theta) + + Ax = \sum (R_X * cos(theta)) + Bx = \sum (R_X * sin(theta)) + Ay = \sum (R_Y * cos(theta)) + By = \sum (R_Y * sin(theta)) + + R_maj = sqrt(Ax^2 + Ay^2) + R_min = sqrt(Bx^2 + By^2) + + phi = atan2(Bx + By, Ax + Ay) + + *** get rest of math from my notebook... + +20090525 : some clarity of issues: + + I've made some progress understanding why the moments analysis was + giving poor results. I've added Gaussian (pseudo-gaussian) + windowing and the ability to set the per-pixel S/N limit. The first + important conclusion was: + + * the per-pixel S/N limit of 1.0 was driving the Mxx,Myy values low + for faint sources. This should be completely disabled. + + The next point is about the Gaussian vs tophat windowing. One + interesting realization was that the Gaussian windowing (or tophat + for that matter) have a stronger impact on the measured moments for + sources with profiles that are non-Gaussian. + + * Gauss with seeing (FWHM) = 1.0 arcsec = 4pix -> sigma = 1.702 + + Gauss window | measured Mxx + 2.0 1.75 + 4.0 2.58 + 6.0 2.85 + + * QGAUSS with see + +20090523 : extended source analysis -- further development + + issues to address: + + * quality of moments used for initial psf selection + - what is the behavior of moments with Gaussian windowing as function of S/N & size? + - can we choose a sigma value that will give reliable moments for an expected range of seeing? + + * moments for galaxy model guesses + - trade-off between Gaussian windowing, tophat windowing, and S/N windowing in footprint area + + initial dev work: + + * add Gaussian windowing, S/N windowing, (tophat windowing) and options to select + + * check visualization of moments + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20091026.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20091026.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20091026.txt (revision 41083) @@ -0,0 +1,44 @@ + +2009.10.26 + + I'm looking into psphot as applied to diff images. For now, I am + running a comparison of a simtest output warp with the convolved + version of the same image as generated by ppSub. + + * we are losing the cores of bright stars, probably to the choices + in the FITS output 16-bit conversion (pixel-value -> NAN) + + * errors for the convolved image seem to be smaller for the same + instrumental mags than for the warp (check chip). + + * the psf model seems to be quite poor -- a lot of structure in the + psf core. + + * (note: not a very good match with the warp image, though better) + + * psfsub images seem to be slightly displaced in X-dir. perhaps + because the convolution kernel has a bit of X-dir tilt in the + peak. Try with PPSUB ISIS orders set to 0.0 + +2009.10.27 + + I need some psphot options / variants: + + * forced photometry: + + I should be able to do: + + psphotForced -file input.fits -psf psf.fits -src src.list output + + and have it only perform the forced photometry on the sources listed + in src.list. + + * psf model creation + + I should be able to do: + + psphotMakePSF -file input.fits -src src.list output + + and have it generate a PSF model using the sources listed in + src.list + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20100131.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20100131.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20100131.txt (revision 41083) @@ -0,0 +1,66 @@ + +2010.03.24 + + I have finished another iteration on source size analysis. I feel + fairly confident of the extended source and CR identifications. They + are not perfect, but they are in pretty good shape. The CR IDs use + the moments to find things smaller in one dimension than a PSF, while + the EXT is set for things larger than a PSF in one dimention. + + There still seems to be some trouble with SATSTARS : these are being + set for sources which are fainter than the SAT limit -- I'm not sure + if this is due to the quality of the PSF fit or if they really are + SAT, but have integrated fluxes lower than expected (underfitted in + PSF). + + Additional Parameters needed from psphot for distinguishing dipoles: + + -- note regarding CfA parameters -- + + Here are our settings. + + Note that Fpos is the flux which is 3-sigma above background, and + Fneg is the flux which is 3-sigma below background. + + Npos is the # of pixels with flux which is 3-sigma above background. + + etc. + + # for positive objects: + # Ngood = Npos, Nbad = Nneg, Fgood = Fpos, Fbad = Fneg + + # for negative objects: + # Ngood = Nneg, Nbad = Npos, Fgood = Fneg, Fbad = Fpos + # FRATIO=Fgood/(Fgood+Fbad) + # NRATIO_BAD=Ngood/(Ngood+Nbad) + # NRATIO_MASK=Ngood/(Ngood+Nmask) + # NRATIO_ALL=Ngood/(Ngood+Nbad+Nmask) + DC_FRATIO_MIN 0.75 + DC_NRATIO_BAD_MIN 0.70 + DC_NRATIO_MASK_MIN 0.60 + DC_NRATIO_ALL_MIN 0.50 + DC_NGOOD_MIN 3 + + *** double-check on the use of fluxScale in pmSourcePhotometry.c:106 + +20100131 + + We have several stand-alone psphot versions, and a few different + psphotReadout versions called by other programs. Here is a complete + list: + + Stand-alone psphot versions | associated psphotReadout version + + o psphot | psphotReadout + o psphotForced | psphotForcedReadout + o psphotMakePSF | psphotMakePSFReadout + x psphotDetect | psphotDetectReadout + x psphotStack | psphotStackReadout + + External programs | associated psphotReadout version + + o ppImage | psphotReadout + o pswarp | psphotReadoutFindPSF + o ppSub | psphotReadoutFindPSF, psphotReadoutMinimal + o ppStack | psphotReadoutKnownSources + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20100715.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20100715.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20100715.txt (revision 41083) @@ -0,0 +1,49 @@ + +2010.08.24 + + Remaining work to be done: + + * test and turn on CR masking + * double-check source size analysis: + * are we doing the right thing with sources flagged as bad in 'rough'? + * are we doing the right thing for SAT, CR, EXT, PSF after SourceSize? + * convert CR_SIGMA and EXT_SIGMA to probabilities + * example results of model fits vs fake inputs + * what is the right solution for the stack PSF photometry? + * Nigel sees 4-6% + * we see 1.9% at the worst + * non-poisson errors are clearly better -- check on the chisq values + * why is QGAUSS so good for the one example (better for poisson, if trendy) + +2010.08.12 + + speeding up the PCM fitting. + + * first, I need to convert the convolutions to FFTs (double check that this will actually be faster!) + * second, I need to thread the PCM fitting process + + I have three options: + + 1) use the FFT threading in fftw -- each convolution would run threaded, but separate ones would be run in series + * con: the images are generally small, so the threading overhead may be large compared to the operation (test?) + 2) each pass of the minimization has 1 + N convolutions to perform: run these in separate threads + * can only use 1+N threads, where N is the number of free parameters (Io, Sxx, Syy, Sxy, but maybe also Xo, Yo) + 3) thread by source, using the system defined for the other fitting. + * since these source are large, this is a bit trickier to guarantee non-collision. + + infrastructure is easy -- copy psphotBlendFit. + +2010.07.15 + +various psphot issues that need to be addressed: + +* update extNsigma based on kron mags +* convert EXT sigmas to a probability (watch that extNsigma is correctly defined) +* psphot forced fails for negative flux + - do a run, plot a histogram of fluxes as various steps + FIXED: this was caused by minimum limit of 0.0 in linear fitting + +* capability to choose extended source fits based on |b| + +* update psf-convolved fitting to work with sersic (iteration) + DONE: but, needs to be sped up + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20101013.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20101013.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20101013.txt (revision 41083) @@ -0,0 +1,80 @@ + +20101019 + + We can identify CRs in the (M_minor, KronMag) plane as a fairly + tight clump (M_minor ~< 1.0, KronMag ~< -8.0). The best bounds of + the clump vary somewhat from image to image. I've added a dynamic + CR limit function which looks for the peak in M_minor and in KronMag + in those regions, then finds the valley floor. It seems to work + fairly well. Outstanding issues: + + * only adjust the limits once + * record limits in the headers + * record CR counts in the headers + * mask the CRs. + +20101017 + + pmSourceMoments: + * FWHM(src) vs FWHM(win) & 1st pass centroid accuracy (no noise) + * 4.0 : 6.0 -> 30% + * 4.0 : 8.0 -> 20% + * 4.0 : 15.0 -> 6.7% + * FWHM(src) vs Tophat(radius) + * 4.0 : 5.0 -> ~30% + * 4.0 : 10.0 -> 0.2% + * 4.0 : 8.0 -> 3% + * 4.0 : 6.0 -> 18% + * 4.0 : 7.0 -> 9% + + pmSourceMomentsGetCentroid(source, 8.0, 0.0, 0.0, 0); + pmSourceMomentsGetCentroid(source, 10.0, 8.0/2.35, 0.0, 0); + + Io = 10000 : (S/N ~ 400) + +dX stats: -0.000708 +/- 0.004586 +dY stats: -0.000296 +/- 0.004678 + + Io = 1000 : (S/N ~ 100) + +dX stats: -0.000131 +/- 0.020428 +dY stats: 0.001038 +/- 0.020556 + + Io = 100 : (S/N ~ 10) + +dX stats: -0.025900 +/- 0.166111 +dY stats: 0.025336 +/- 0.169681 + +raising bits : these functions may raise: + +psphotSourceStats : BELOW_MOMENTS_SN | SKY_FAILURE | SKYVAR_FAILURE | BIG_RADIUS | MOMENTS_FAILURE + +psphotBasicDeblend : BLEND + +psphotDeblendSatstars : BLEND + +psphotRoughClass : SATSTAR | SATURATED | DEFECT | PSFSTAR + +psphotChoosePSF : can remove PSFSTAR from failed candidates + +psphotGuessModels : BADPSF + +psphotMergeSources : PSFSTAR | EXTERNAL + * note : this filters out 'bad' sources -- maybe too restrictive? + +psphotFitSourcesLinearStack : LINEAR_FIT + +psphotSourceSize : SIZE_SKIPPED | DEFECT | EXT_LIMIT | CR_LIMIT + +psphotApResid : AP_MAGS + +psphotRadialAperturesByObject RADIAL_FLUX | EXTENDED_STATS + +psphotExtendedSourceFits : EXTENDED_FIT + +**** + +flags from psastro: + + OFF_CHIP | ON_GHOST | ON_GLINT | ON_SPIKE + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20101222.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20101222.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.20101222.txt (revision 41083) @@ -0,0 +1,18 @@ + +20101222 : EAM + + continuing work on psphot stack. Some notes: + + * visualization now has duplicate image display calls + * visual needs to do a better jobs scaling the smoothed, subtracted images. + * the fit radius for the convolved images should be a bit bigger than the unconvolved one + * *** fitRadius vs apRadius *** + I have been using 'markVal' to define the valid pixels for different operations + I think this has been inconsistent. + + ** Have pmSourceMagnitudes apply the defined apRadius on the fly + when measuring aperture photometry + + ** Set the fitRadius and keep it until redefined? + + ** RadialApertures does not use the marked region? Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/notes.txt (revision 41083) @@ -0,0 +1,971 @@ + +2009.09.26 + + Remaining PSPHOT Issues: + + o soften errors a bit for bright stars? (essentially allow a + systematic error on the flat of some percentage). this may help + the very saturated stars (peaks > 10k) which seem to be + excessively driven by the cores. + ** did not seem to be a big success --> makes things worse for f = + 0.01, about the same for f = 0.005 + + o clean up and remove test bits of code. + + o clean up psphot.config files : drop unused, fix some conflicting + names, reduce camera-specific options, make S/N limits + consistent... + + * PSF Eval & ChiSq test?? + + * EXT fail fit? lack of iterations? + + * extended source output + + * still some concerns with the source size analysis: the parameters + space seems fine (Mxx, Myy, nSigma), but we need to define more + sophisticated boundaries (perhaps include dMag or Mag). + + * some optimization issues: + * too many re-calculations of all magnitude / aperture data + * is FluxScale really faster? do we need a finer mapping? (5x5 + clearly yields too much scatter) + * is psphotFitSourcesLinear.c (build matrix) super-slow when optimized? + + * very saturated stars are somewhat segmented. + + * footprint coverage in dense simtest image 005.001? + + * footprint S/N limit vs peaks S/N limit? + + * warning about burntool table? + + * is the residual map failing to be masked at the radius? + + * PSF 2D convergence : how to choose between marginally different + options? (note that the GPC1 images may go to 2x2 instead of 3x3) + + * -visual needs more fine-grained control + + * visualization graphs should reset the sections in general. + + * some excess verbosity (my log level is 9, lower?) + + ----- + + Apparently I'm getting bad psf vs ap mags because of the difference + between linear and non-linear fits. I think this is due to the + weighting schemes: + + psphotChoosePSF & psphotBlendFit depend on POISSON.ERRORS.PHOT.LMM, + which is set to TRUE => use Poisson errors + + psphotFitSourcesLinear depends on CONSTANT_PHOTOMETRIC_WEIGHTS, + which is set to TRUE => do not use Poisson errors. + + nothing seems to use the value POISSON.ERRORS.PHOT.LIN + + ** Poisson errors seeem to be working much better than Constant + errors (as judged by consistency between PSF and APERTURE + magnitdues). Not sure I understand this. It could be that the + implementation of Constant errors in the linear fitting analysis + is busted. + + ** Also, I was getting some additional scatter from the FluxScale + (lookup-table for Io -> mags). + + * surprisingly, it is not clear that the FluxScale method is + faster than the more accurate integration technique... + +2009.09.25 + + Notes on pmSourceMagnitudes: + + * in psphotChoosePSF, in pmPSFtry: raw PSF & raw AP + * in psphotChoosePSF, psphotMakeGrowthCurve: uses direct all to pmSourcePhotometryAper + * in psphotSourceSize, in psphotSourceSizePSF, raw PSF & raw AP (but uses moments->Sum) for PSF stars + NOTE: on the initial call, the PSF stars may have had multiple models tried and the psfMag + may not be the value for the chosen model. + * in psphotSourceSize, in psphotSourceClassRegion, raw PSF & raw AP for all sources for which size is not yet measured. + * in psphotApResid: raw PSF & raw AP for STARS only + * in psphotMagnitudes; corrected PSF & AP mags + +2009.01.24 + + Some results from the threading work below. these are all tests on + pikake using a sample GPC1 image. This set uses an optimized + build of psLib, psModules, and psphot. + +pikake: grep --color "built models" o4771g0244o.t?.opt.log +o4771g0244o.t0.opt.log: built models for 3156 objects: 0.696490 sec +o4771g0244o.t0.opt.log: built models for 1738 objects: 0.367092 sec +o4771g0244o.t1.opt.log: built models for 3150 objects: 0.746428 sec +o4771g0244o.t1.opt.log: built models for 1745 objects: 0.411406 sec +o4771g0244o.t2.opt.log: built models for 3152 objects: 0.412824 sec +o4771g0244o.t2.opt.log: built models for 1756 objects: 0.241859 sec +o4771g0244o.t3.opt.log: built models for 3151 objects: 0.341083 sec +o4771g0244o.t3.opt.log: built models for 1746 objects: 0.176361 sec +o4771g0244o.t4.opt.log: built models for 3159 objects: 0.235669 sec +o4771g0244o.t4.opt.log: built models for 1741 objects: 0.164474 sec +o4771g0244o.t5.opt.log: built models for 3155 objects: 0.286719 sec +o4771g0244o.t5.opt.log: built models for 1747 objects: 0.165341 sec + +pikake: grep --color "measure magnitudes" o4771g0244o.t?.opt.log +o4771g0244o.t0.opt.log: measure magnitudes : 4.080628 sec for 4895 objects (2191 with apertures) +o4771g0244o.t1.opt.log: measure magnitudes : 4.155192 sec for 4896 objects (2185 with apertures) +o4771g0244o.t2.opt.log: measure magnitudes : 2.174322 sec for 4909 objects (2190 with apertures) +o4771g0244o.t3.opt.log: measure magnitudes : 1.507431 sec for 4898 objects (2189 with apertures) +o4771g0244o.t4.opt.log: measure magnitudes : 1.174447 sec for 4901 objects (2191 with apertures) +o4771g0244o.t5.opt.log: measure magnitudes : 1.149875 sec for 4903 objects (2194 with apertures) + +pikake: grep --color "full-frame aperture" o4771g0244o.t?.opt.log +o4771g0244o.t0.opt.log: measure full-frame aperture residuals for 1977 sources: 1.681260 sec +o4771g0244o.t1.opt.log: measure full-frame aperture residuals for 1972 sources: 1.728578 sec +o4771g0244o.t2.opt.log: measure full-frame aperture residuals for 1980 sources: 0.929584 sec +o4771g0244o.t3.opt.log: measure full-frame aperture residuals for 1978 sources: 0.656556 sec +o4771g0244o.t4.opt.log: measure full-frame aperture residuals for 1977 sources: 0.507413 sec +o4771g0244o.t5.opt.log: measure full-frame aperture residuals for 1981 sources: 0.512259 sec + +pikake: grep --color "fit models" o4771g0244o.t?.opt.log +o4771g0244o.t0.opt.log: fit models: 4.623715 sec for 2011 objects (1904 psf, 45 ext, 62 failed, 1146 skipped) +o4771g0244o.t1.opt.log: fit models: 5.036680 sec for 2006 objects (1901 psf, 45 ext, 60 failed, 1145 skipped) +o4771g0244o.t2.opt.log: fit models: 3.507495 sec for 2014 objects (1912 psf, 44 ext, 58 failed, 1139 skipped) +o4771g0244o.t3.opt.log: fit models: 3.236495 sec for 2012 objects (1906 psf, 44 ext, 62 failed, 1140 skipped) +o4771g0244o.t4.opt.log: fit models: 3.929518 sec for 2013 objects (1906 psf, 44 ext, 63 failed, 1147 skipped) +o4771g0244o.t5.opt.log: fit models: 4.152286 sec for 2012 objects (1903 psf, 45 ext, 64 failed, 1144 skipped) + +pikake: grep failed o4771g0244o.t?.opt.log | grep --color "moments" +o4771g0244o.t0.opt.log: 3161 sources, 3147 moments, 14 failed: 1.288588 sec +o4771g0244o.t0.opt.log: 1738 sources, 1698 moments, 40 failed: 0.700303 sec +o4771g0244o.t1.opt.log: 3155 sources, 3142 moments, 13 failed: 1.329107 sec +o4771g0244o.t1.opt.log: 1745 sources, 1702 moments, 43 failed: 0.739017 sec +o4771g0244o.t2.opt.log: 3157 sources, 3144 moments, 13 failed: 0.717423 sec +o4771g0244o.t2.opt.log: 1756 sources, 1712 moments, 44 failed: 0.409145 sec +o4771g0244o.t3.opt.log: 3156 sources, 3142 moments, 14 failed: 0.510468 sec +o4771g0244o.t3.opt.log: 1746 sources, 1702 moments, 44 failed: 0.311198 sec +o4771g0244o.t4.opt.log: 3164 sources, 3150 moments, 14 failed: 0.412192 sec +o4771g0244o.t4.opt.log: 1741 sources, 1698 moments, 43 failed: 0.240172 sec +o4771g0244o.t5.opt.log: 3160 sources, 3144 moments, 16 failed: 0.418781 sec +o4771g0244o.t5.opt.log: 1747 sources, 1701 moments, 46 failed: 0.244808 sec + +pikake: grep --color "complete psphot" o4771g0244o.t?.opt.log +o4771g0244o.t0.opt.log: complete psphot run: 38.044034 sec +o4771g0244o.t1.opt.log: complete psphot run: 39.039324 sec +o4771g0244o.t2.opt.log: complete psphot run: 28.565228 sec +o4771g0244o.t3.opt.log: complete psphot run: 25.380312 sec +o4771g0244o.t4.opt.log: complete psphot run: 24.606107 sec +o4771g0244o.t5.opt.log: complete psphot run: 24.796387 sec + + Some results from the threading work below. these are all tests on + pikake using a sample GPC1 image. This set uses an unoptimized + build of psphot (though psLib may be optmized). + +pikake: grep --color "built models" o4771g0244o.t?.no-opt.log +o4771g0244o.t0.no-opt.log: built models for 3160 objects: 0.997456 sec +o4771g0244o.t0.no-opt.log: built models for 1725 objects: 0.523033 sec +o4771g0244o.t1.no-opt.log: built models for 3154 objects: 1.031943 sec +o4771g0244o.t1.no-opt.log: built models for 1725 objects: 0.562166 sec +o4771g0244o.t2.no-opt.log: built models for 3156 objects: 0.563504 sec +o4771g0244o.t2.no-opt.log: built models for 1729 objects: 0.311835 sec +o4771g0244o.t3.no-opt.log: built models for 3155 objects: 0.399958 sec +o4771g0244o.t3.no-opt.log: built models for 1728 objects: 0.235203 sec +o4771g0244o.t4.no-opt.log: built models for 3153 objects: 0.329366 sec +o4771g0244o.t4.no-opt.log: built models for 1726 objects: 0.192076 sec +o4771g0244o.t5.no-opt.log: built models for 3155 objects: 0.408434 sec +o4771g0244o.t5.no-opt.log: built models for 1734 objects: 0.198095 sec + +pikake: grep --color "measure magnitudes" o4771g0244o.t?.no-opt.log +o4771g0244o.t0.no-opt.log: measure magnitudes : 5.939663 sec for 4886 objects (2187 with apertures) +o4771g0244o.t1.no-opt.log: measure magnitudes : 6.041825 sec for 4880 objects (2189 with apertures) +o4771g0244o.t2.no-opt.log: measure magnitudes : 3.126765 sec for 4886 objects (2192 with apertures) +o4771g0244o.t3.no-opt.log: measure magnitudes : 2.158470 sec for 4884 objects (2188 with apertures) +o4771g0244o.t4.no-opt.log: measure magnitudes : 1.756856 sec for 4880 objects (2192 with apertures) +o4771g0244o.t5.no-opt.log: measure magnitudes : 1.697331 sec for 4890 objects (2188 with apertures) + +pikake: grep --color "full-frame aperture" o4771g0244o.t?.no-opt.log +o4771g0244o.t0.no-opt.log: measure full-frame aperture residuals for 1976 sources: 2.869795 sec +o4771g0244o.t1.no-opt.log: measure full-frame aperture residuals for 1973 sources: 2.908194 sec +o4771g0244o.t2.no-opt.log: measure full-frame aperture residuals for 1977 sources: 1.545140 sec +o4771g0244o.t3.no-opt.log: measure full-frame aperture residuals for 1977 sources: 1.091451 sec +o4771g0244o.t4.no-opt.log: measure full-frame aperture residuals for 1976 sources: 0.857477 sec +o4771g0244o.t5.no-opt.log: measure full-frame aperture residuals for 1979 sources: 0.860652 sec + +pikake: grep --color "fit models" o4771g0244o.t?.no-opt.log +o4771g0244o.t0.no-opt.log: fit models: 7.771925 sec for 2011 objects (1902 psf, 45 ext, 64 failed, 1150 skipped) +o4771g0244o.t1.no-opt.log: fit models: 8.053570 sec for 2010 objects (1904 psf, 45 ext, 61 failed, 1145 skipped) +o4771g0244o.t2.no-opt.log: fit models: 5.063026 sec for 2009 objects (1904 psf, 44 ext, 61 failed, 1148 skipped) +o4771g0244o.t3.no-opt.log: fit models: 4.314687 sec for 2012 objects (1905 psf, 44 ext, 63 failed, 1144 skipped) +o4771g0244o.t4.no-opt.log: fit models: 4.147600 sec for 2009 objects (1904 psf, 44 ext, 61 failed, 1145 skipped) +o4771g0244o.t5.no-opt.log: fit models: 4.207860 sec for 2012 objects (1906 psf, 44 ext, 62 failed, 1144 skipped) + +pikake: grep failed o4771g0244o.t?.no-opt.log | grep --color "moments" +o4771g0244o.t0.no-opt.log: 3165 sources, 3151 moments, 14 failed: 2.867193 sec +o4771g0244o.t0.no-opt.log: 1725 sources, 1681 moments, 44 failed: 1.513526 sec +o4771g0244o.t1.no-opt.log: 3159 sources, 3146 moments, 13 failed: 2.966456 sec +o4771g0244o.t1.no-opt.log: 1725 sources, 1685 moments, 40 failed: 1.568208 sec +o4771g0244o.t2.no-opt.log: 3161 sources, 3148 moments, 13 failed: 1.590746 sec +o4771g0244o.t2.no-opt.log: 1729 sources, 1687 moments, 42 failed: 0.823426 sec +o4771g0244o.t3.no-opt.log: 3160 sources, 3146 moments, 14 failed: 1.068936 sec +o4771g0244o.t3.no-opt.log: 1728 sources, 1684 moments, 44 failed: 0.603041 sec +o4771g0244o.t4.no-opt.log: 3158 sources, 3144 moments, 14 failed: 0.869741 sec +o4771g0244o.t4.no-opt.log: 1726 sources, 1685 moments, 41 failed: 0.505545 sec +o4771g0244o.t5.no-opt.log: 3160 sources, 3146 moments, 14 failed: 0.830094 sec +o4771g0244o.t5.no-opt.log: 1734 sources, 1695 moments, 39 failed: 0.469828 sec + +pikake: grep --color "complete psphot" o4771g0244o.t?.no-opt.log +o4771g0244o.t0.no-opt.log: complete psphot run: 70.375518 sec +o4771g0244o.t1.no-opt.log: complete psphot run: 71.167205 sec +o4771g0244o.t2.no-opt.log: complete psphot run: 51.237604 sec +o4771g0244o.t3.no-opt.log: complete psphot run: 44.195015 sec +o4771g0244o.t4.no-opt.log: complete psphot run: 41.782608 sec +o4771g0244o.t5.no-opt.log: complete psphot run: 42.483942 sec + +Not bad: almost a factor of 2 overall. psphotMagnitudes, +psphotApResid, psphotSourceStats all get nearly the max improvement. +psphotBlendFit seems to be saturating. There are a lot of allocs in +the main functions: pmSourceFitModel and pmSourceFitSet, which are +probably blocking each other. + +2009.01.22 + + working on more psphot threading. So far, I have threaded code for: + + * psphotGuessModel (DONE) : not working so well just yet (see + below). Late Update: in recent tests, the result below may have + been caused by the unoptimized psLib. + + here are functions for which a modest gain can be realized and which + should be easy to thread: + + * psphotMagnitudes (DONE) + + ** this function uses psImageShiftMask to interpolate the source + flux to compensate for fractional pixel offsets. This function + requires a call to psImageAlloc and a subsequent call to + psFree. This probably will make the threading + inefficient. However, it is not clear we need to perform this + interpolation for all sources: the correction is probably only + signficant compared to the errors for the brighter sources. + + ** psImageShiftMask gratuitously calls psImageInterpolationAlloc; + perhaps it should accept this as an argument, or perhaps this + function should be re-examined. + + * psphotApResid (DONE) + + * this function is basically calling pmSourceMagnitudes, so + suffers the same issues as psphotMagnitudes + + * psphotFindDetections (footprints) (fairly tricky, actually) + + ** need to examine the steps in the function. it is not obvious + yet what needs to be improved. + + * psphotBlendFit (DONE) + + ** this one will be a little tricky: this function currently + extends the source array to add new sources detected as part of + psphotFitBlob. I would need to have a way of saving these from + each thread, then joining them back together at the end. + + * psphotSourceStats (DONE) + + ** this one includes the allocs to generate the source pixels, but + could be split into two steps: 1) generate the complete array + of sources; 2) measure the sky, sky variance, and moments. + + Otherwise, this function should be easy: no other allocs or + frees, i think. + + * psphotChoosePSF : + + ** need to use the grid technique for the pmPSFtryModel which does + the source fitting + +2008.10.26 + + (note: this is unoptimized) + 0 threads: built models for 3921 objects: 15.223455 sec + built models for 2868 objects: 10.698732 sec + + 2 threads: built models for 3921 objects: 37.529655 sec + built models for 2862 objects: 28.142142 sec + + (clearly, the threading is doing bad things, at least without optimization..) + + old code (no threading): + built models for 3923 objects: 15.265046 sec + built models for 2842 objects: 10.840331 sec + + (so, the threaded version without threading is OK) + +2008.06.25 + + timing : footprints + + on GPC1, with FFT: + pass 1 : convolved with grow disc: 17.558344 sec + pass 2 : convolved with grow disc: 8.915036 sec + +2008.01.31 + + Most of the fixes listed below have been finished except for: + + * careful handling of the r ~ 0 fluxes / effective positions (for eg sersic) + * the EXT output parameters have not been tested + + In addition, some further work to be done: + + * multiple image detection + * multiple image fitting (including projection) + * optimization + * multi-threading + + * possibly try the 'active deblending' concept that Ken from WISE is using. + * instead of fitting a double star with the positions floating at a + slightly random location, search for significant additions within + a grid about the central peak (say ~ 1sigma / 0.2 pix?). + (note that this is a linear fit) + + * for the blend groups, freeze the positions for the fainter sources + (consistent with the non-linear S/N threshold) + + + +2007.08.17 + + I am working on a number of cleanup / fixes. I have made an overhaul + of the pmModel APIs, adding function pointers in the pmModel + structure to the class-specific utility functions (eg, + pmModel->modelFunc is the actual function which is evaluated). + + TO DO: + + * update pmSourceFitSet to be able to include more than one model + type (currently it assumes the sources are all, eg, PSFs). This is + now needed because the old implementation used the function lookups + which have been dropped. + + * define a generic API set to handle 2D modelling of a scalar using + either polynomials (as the pmPSF code currently does) or an + image-based representation (as the psphot sky model currently + does). + + The image-based representation can automatically step down from NxM + super pixels to a smaller number based on the density of + measurements. + + Include ways to smooth and regularize the output result. + + This mechanism can be applied to: psf parameters, aperture + residual, psf peak-to-flux variations, the psphot background + representation. + + * generate and store the output radial profile for objects + + * finish testing and incorportate the CR / EXT measurements + + * adjustments to pixel center based on second derivatives : needed + for the sersic models. + + * adjustments to pixel flux for extreme values (r ~ 0) : needed for + the sersic models. + + * on psf stars : fall back on a Gaussian. + + * OPTIMIZATIONS !! + + * drop the model sky element : should only be in source->sky,dsky + +2006.11.16 + +ensemble: + + * first block: + * select sources of interest / skip + * generate a guess model + * re-measure moments for saturated stars (MOVE) + * update peak position (MOVE to peaks) + * set the object radius + * build temporary source copies (including pixels, mask, weight) + + * second block: + * measure the cross-products and weights + * add to the sparse matrix + + * build full-image mask + * measure coord weights + * build order elements + + * solve for source amplitudes + * update models + + * create psSparseBorder to solve matrix equations which have a large + sparse region, with a completely filled border of a smaller number + of rows. + + * trying to modify psphotEnsemble to (1) be more general (not just + PSFs) and (2) fit for the sky offset. + + - how is the radius for each source set? + + +2006.11.11 + + I am trying to define a complete set of tests to validate the + measurement of the photometry compared with the simulated images. + Part of my confusion is controlling exactly which corrections are + applied to the output magnitudes. Here is the complete list: + + psfMag_out = psfMag_meas + ApTrend(x,y,0.0,0.0) + apMag_out = apMag_meas + growth(radius,refRadius) + f(skyBias,skySat) + + these corrections are modified by the following config settings: + + IGNORE_GROWTH (sets all growth magnitudes to the fitMag value, + effectively turning off the growth correction) + + APTREND : set the level of correction + NONE : no correction is fitted + CONSTANT : just a single constant (applied to get psfMag_out) + SKYBIAS : the skybias term is fitted (constant applied to psfMag, + slope applied apMag) + +2006.11.08 + + I have been testing the pmSourceMagnitudes interpolation of the + source position when measuing the aperture magnitudes. I built a + test suite for pmSourceMagnitudes and pmGrowthCurve. even with + bicubic interpolation, there are noticable errors for small + apertures (ie, small compared to the PSF sigma). I used a gaussian + PSF to make the tests. it seems that, if the radius is less than 2 + sigma or so, the interpolated aperture mag is in error by as much as + 12mmag. The size of this effect must depend on the smoothing + algorithm, the shape of the PSF, the size of the PSF relative to the + pixels, and the size of the subpixel interpolation. I believe the + effect is a result of the smoothing introduced by the interpolation: + the smoothing pushes the flux in the inner portions of the PSF down + and enhances the flux in the outer portions. thus, there is + actually a turn over in the error: at very large values, the error + is very small because the aperture contains all of the flux, and the + interpolation has little effect. as the radius shrinks, the error + grows (in the sense of M_raw - M_interpolated; ie, the flux in the + interpolated source is too large). around 2-3sigma, the error turns + over and becomes negative (too little flux in the interpolated + source). + + The sigmas I used for these tests (1.0, 1.5, 2.0 pix) are + appropriate for the range we expect to see from PS1, or any + well-sampled detector. Thus, even though the effect is probably + less important for poorer seeing, we have to be careful to avoid it + in our Pan-STARRS analysis. + + Currently, the minimum aperture is defined only by the config + variable PSF_FIT_PAD, which really implies a padding. My proposal + is to require the minimum aperture to be a number of sigma (an + additional parameter). Then, the formula for the radius would be + something like: + + MAX (modelRadiusPSF (PSF_FIT_NSIGMA), PSF_FIT_RADIUS_NSIG) + + PSF_FIT_PAD + + with PSF_FIT_RADIUS_NSIG (choose a better name!) set to something + like 2 or 3. + +2006.11.06 + + I have added the aperture interpolation as a component of the + pmSourceMagnitudes function. + + I am trying to make things a bit more organized, with more + consistent APIs. + + * changed pmPSFtry to use the pmSourceMagnitudes API + * changed pmPSFtry to use new pmSource entries for each try (saving + * the pmModels generated) + + other notes: + * moved psphotGrowthCurve -> pmGrowthCurveGenerate + + things I know need to be added / fixed: + * measure bicube fit to peak from smoothed image + * use psphotEnsemble fit to re-measure normalization? + * extended object statistical measures + +2006.11.02 + + Looking into the growth curve problem. RHL sent his recipe for + correcting to true circular apertures. I've done a test fix of + interpolating the stellar image to the desired center. it looks + like bilinear interpolation is insufficient (about 8% error for 2 + pixel radius aperture). Unless I've got the formula wrong, the + effective smoothing of the gaussian is moving flux from the center + out and making small apertures in error that depends on the offset. + Bicubic could be enough to clean this up? + + I think the current code may need to be careful modified to include + the interpolation for every aperture measurement. there are paired + 'pmModelAdd' and pmModelSub steps to add and remove the model flux. + I need to be careful about how the mask affects these steps. + + I also need to be careful in interpreting the test results: since + the simulated images create objects by filling in a pixel and then + smoothing, all objects have centroids which are exactly 0.5 pixel + offsets. These all then have a fixed growth correction error, which + fortunately happens to be substantial (because of a 0.5 pixel offset + in the growth curve reference object!) + + I need to double check the 0.5 pixel error issues. There are + problems with the pslib version of psImageShift and + psImageInterpolate (or at least inconsistencies). the pmModelAdd + and Sub functions and the centroid functions are probably also + inconsistent. + +2006.11.01 + + Trying to understand low-level errors in psphot using simulated + data. the main issue I am concerned with is the bias RHL is seeing + in SDSS-psphot comparisons. But there are other issues that are + creeping in and probably point at logical errors in the code. + + * in pmPSFtry, I measure the first pass on the aperture residual (to + choose between PSF model options, if that is being tested). This + analysis currently uses a polynomial fit (1st order) for the offset + and the sky bias (using r^2/flux as an independent variable). + + Notes: + + - is the psf-fit error distribution reasonable? perhaps a little + on the high side: at 10000 DN, noise is 0.0138, should be + 0.0107. + + - scatter of ap-fit is surprisingly high. M_input - M_fit has a + median chisq of 2 (mean of 5), while M_input - M_ap has a median + chisq of 125 (mean of 380). This is using a radius of 15. It + cleans up hugely when I go to a radius of 7 and even more so + with radius = 4. (In this case, the PSF sigma is 1.5, so 4 is + well matched and 7 is only slightly large). + + * make the initial PSF fitting radius and aperture radius + different? + * make the initial PSF fitting radius a function of the measured + PSF sigma? + + - I have been fitting ap-fit mag without weights; the resulting + ap-fit errors are highly over-estimated since the ap-fit scatter + is too high. I have added code to carry the weights (error from + the fit only at the moment, ie, not the ap error as well). This + is good, but not as important if the radius is chosen well above. + + * a bias in the psphotEnsemble results depending on the sky level + but not in the PSF model fits? turning on/off CONST_PHOT_WT has + an equivalent effect, but is quite small. why is one sensitive to + the sky and the other is not? + + * the error measured by psphotEnsemble was wrong (and inconsistent + for a given weighting situation): fixed the def of the error. + + * still trying to understand the corrections being made. running a + few tests: + + keeping PSF_FIT_RADIUS at 4.0 + + t1 : no growth correction, constant weights + t2 : growth correction, constant weights + t3 : no growth correction, no constant weights + t4 : growth correction, no constant weights + t5 : growth correction, constant weights, constant ap (4) + t6 : growth correction, constant weights, integer radius + t7 : growth correction, constant weights, integer radius + (for t7, I've fixed the bug that the growth correction include the + min radius). + + * psphotGrowthCurve is messed up : the measurement does not correct + for the actual object center, and at small radii this is an + important source of noise. + +2006.06.28 + + I am adding a few minor features. First up is the ability to break + the processing at interesting stages. These will be: + + - PEAKS: after sources are generated, but before moments are + measured + - MOMENTS: after moments are measured, before the PSF is measured + - PSFMODEL: after the psf model is measured, before sources are + fitted in bulk + - ENSEMBLE: after the linear ensemble fitting + - DEFAULT: complete processing + +2006.04.22 + + I have updated psphot to work with the cycle 11 release of psLib and + psModules. At this point, the IfA group is now working off of the + CVS head rather than from a separate IfA branch of psLib and + psModules. The cycle 11 release of psLib needed a few updates to + fix minor problems. The psModules code needed significant work to + fix discrepancies. The new version of psphot has also been leak + checked, at least for basic configurations. This new version of + psphot should be used with the new camera configuration system, + which can be found in the ipp/config directory. Here are the + necessary tags: + + psphot: psphot_dev_11_0 + psModules: rel11_ifa_pre0 + psLib: rel11_ifa_0 + +2006.04.16 + + Examining some of the throughput issues. the flux measurement speed + is strongly dependent on the interation of the function step size: + + dz = 0.01 : 3.24s for 4400 objects + dz = 0.1 : 0.44s for 4400 objects + + there is an increased error in the result with the larger step size, + but it is smaller than the phot error. Still, we could do better + with a smarter integration function. + + Including the aperture photometry on all sources, but not the + pixWeight, the source magnitudes takes 4.8 sec for 4400 objects. + + Adding the pixWeight increases the time to 7.5 sec for 4400 objects + +2006.02.12 + + Last week, I made a new tag (psphot_dev_08) after finishing the work + to clean memory leaks and to modify psphot to work with Paul's + ppImage infrastructure. + + I have added a new background model function which generates a + median map based on Robert's suggestion of a) half-step windows and + b) linear interpolation. + + Now, I am working to remove the constant sky level concepts and make + every relevant test use the signal/noise ratio as the trigger. This + has impacts on the peak detections, the threshold for blends, and + the threshold for choosing pixels in the fitting routines. + +2005.12.23 + + Improvements in psphot as of psphot_dev_07 + + - updated to work with psModule release 9.0 + - moved model test functions into psphot as optional mode + - added pmSourceFitSet : simultaneous model group fitting + - added bicubic interpolation to tweak up the peak coordinates + - added aperture residual analysis after complete object model & subtraction + - added 3D (x,y,rflux) aperture residual fitting + - added blend vs blob fitting (BlendFit) + - added ensemble PSF fitting, with sparse matrix inversion + - improved the flagging / marking of model results + - better abstraction for setting fit parameters + - various low-level fixes + - added PSF output option + - added break points + - added more optional output formats + - fixed FITS table output + + Things still to be added to psphot: + + - better initial background model (NxM median grid?) + - include local background model in EnsemblePSF Fit (use sparse matrix) + - 2nd pass (after object subtraction) to smooth and detect faint objects + - re-add background to object-subtracted image + - cleanup function names + - push more functions into psLib / psModule + - fix extended / galaxy models to improve stability + - add mode with input PSF + - add mode with input fixed sources (PS_SOURCE_FIXED) + - cleanup memory leaks + +2005.11.25 + + I've updated psphot to work with the current psLib v8, though a + number of psLib fixes were needed. These are pushed under + psLib:eam_rel8_b2. The psphot code which works with that version of + psLib is tagged psphot_dev_04. + + I'm working on converting psphot to work with the current release of + psModules, which basically includes all of the object functions. + + - change comments are relative to psModules (vs psphot versions) + + psEllipse.c : OK + psEllipse.h : OK + pmModelGroup.c : OK (missing TGAUSS entry) + pmModelGroup.h : OK (adds many comments) + + pmPSFtry.c : a bunch of formatting weirdness (try-> got line breaks + added pmPSFtryMetric_Alt + fixed up usage of psVectorClipFitPolynomial1D + fixed up usage of psPolynomial..Alloc + + pmPSFtry.h : OK (Added extensive comments) + + pmPSF.c : fixed up usage of psPolynomial..Alloc + + pmPSF.h : OK (Added extensive comments) + + pmObjects.c : cleaned up some formatting, + fixed usage of psImageCountPixelMask + + pmObjects.h : added apMag and fitMag to pmSource + dropped PS_MODEL_name defines + + psLibUtils.[ch] should be dropped from psModules + psModulesUtils.[ch] should be dropped from psModules + + Makefile.am : dropped psModulesUtils.[ch], psLibUtils.[ch] + + - I have successfully tested psphot with the psModules.v8 code + (eam_rel8_b1 tag). I only needed to make a few modes in the + psModules code. + +2005.11.16 + + I have made some fixes to make psphot work with pslib rel8_0. I + need to merge psphot with the rel8_0 version of the psMinimize and + pmObjects code. + +2005.09.06 + + I have built a working version of PSPhot using my own copy of the + pmObjects.[ch] files. I need to minimize the difference between the + v0.7.0, v0.5.0, and my own version of pmObjects.c + + additions to psLib based on the new version: + - extended the minimization to include parameter limits + + changes v0.5.0 / v0.7.0 / EAM + - changed all ps* to pm* + - fixed naming for local static functions (eg modelFree not p_psModelFree) + - dropped all hard-coded model names (vs v0.7.0) + - using current psMemSetDellocator name + - added noise image to pmSource + - converted old asserts to new asserts (PS_.._CHECK..) + - add return from error in pmFindImagePeaks + - allow isItInThisRegion to have NULL region (true) + - fixed subImages to correspond to region definition (end is + exclusive) in LocalSky and SetPixelCircle + - dropped old, redundant code in pmSourceLocalSky + - fixed pmSourceMoments use of mask (0 is valid, not 1) + - added pmSourceMoments validity tests (numPixels, Sum, centroid shift) + - added correction for Sxy (XY/Sum - x*y) + - added pmSourcePSFClump function (and pmPSFClump structure) + - allow missing sources in pmSourcePSFClump (not all peaks yield + sources) + - moved pmSourceRoughClass metadata lookups out of inner loop + - added saturated pixel test using pmImageCountPixelMask + - various redefinitions of initial classes + - dropped old, redundant code in pmSourceSetPixelsCircle + - mask image set to type psU8 (was psF32) + - abstract object model functions for pmSourceModelGuess + - converted evalModel to a public function psModelEval, modified params + - pmSourceFitModel operates on a specified model + - pmSourceFitModel had option to apply/ignore PSF information + - pmSourceFitModel calculates yErr + - pmSourceFitModel calculates the covariance matrix + - added paramMask to pmSourceFitModel + - pmSourceFitModel_EAM uses covar to carry in beta and param + limits... + - pmSourceFitModel requires solution to stay within image + - pmSourceFitModel calculates and saves chisq, nIter, nDOF + - pmSourceFitModel uses GaussNewtonDelta for frozen params + - added mask to pmSourceAdd, pmSourceSub + - model abstractions in pmSourceAdd, pmSourceSub + - fixed 'center' option in pmSourceAdd/Sub + + - pmPSF params should be F64 + + lingering concerns: + - Polynomials and related functions are STILL messed up. this is a + major priority!!! + - pmSourceFitModel calculates sqrt(var), psMinimize calculates sq(sqrt(var)) + - does psImageAlloc zero the image? if so, drop init routine + from pmSourcePSFClump + - pmSourceRoughClass uses RDNOISE, GAIN to calculate SN: should use + the source->noise image? part of pmSourceMoments? + - pmSourceMoments_EAM uses macro for checkRegion + - pmObjects_EAM.c uses some hardcoded mask values (LocalSky) + - why does LocalSky_EAM not need subImageMask->col0 = subImage->col0?? + - psGetRowVectorFromImage should be in psLib (in psLibUtils) + - pmSourceSetPixelsCircle and pmSourceLocalSky probably could use + the psImageMaskRegion, etc functions. + - pmSourceSetPixelsCircle uses a hard-coded mask value + - define a mask registration process?? + - pmSourceContour is using the model flux, not the image flux + +2005.09.05 + + I have a working version of PSPhot which handles PSF and galaxy + models. I am tagging the module with an alpha version number. this + version compiles with psLib-0.5.0, at least my version of it. My + current goal is to make it work with the current psLib head (nearly + 0.7.0) with a mimimum of _EAM versions of functions. + + the tag for the working version is dev_01 + +2005.06.04 + + progress on psphot has moved along well. At this point, the process + loads the image, finds the peaks, determines a best PSF model, + identifies objects which are consistent with that model, and + attempts to fit a galaxy model to the objects which are not + consistent with the model. + + a variety of minor issues remain, as well as some major issues. + + - use a proper noise image to keep the fits honest after other + objects have been subtracted. + + - define the masks in terms of a single mask image. this image + could be provided initially by the user. + + - add noise enhancement (couple with mask marking subtraction) + + - + +2005.04.12 + +- psPeak, psSource, etc: should be pmPeak, pmSource, etc +- pmCullPeaks: should be pmCullImagePeaks + +- does pmSourceMoments need an image argument? + +- psLibInit / p_psTimeInit + + we should not have to know about astronomy time concepts (ie, have a + timeConfig file) in order to use the time functions for basic work. + This means having a PS_TIME_UNIX which just works with the UNIX + clock and avoids all initialization issues. + +- psImageStats is much too slow for basic median + - this function does a complete sort on all pixel values. we need + to use the histogram method (spec a different type of median) + +- psImageStats, ROBUST_MEDIAN fails +- there are errors in psVectorStats for ROBUST MEAN, STDEV (& median?) + * this is ill-defined. revise the ADD: too sensitive to the + binning, smoothing scales + +- pmFindImagePeaks is not finding any peaks: + typo in interior row section: see bug 359 + after this fix: PASS + +* p_psGetRowVectorFromImage : PASS +* psGetRowVectorFromImage : PASS +- both have about the same time (~15 us +/- 5us) for 2048 pix on alala + +* pmFindVectorPeaks : PASS (returned row data in data.U32) + +- # include : needed in psMetadata.c + +* MyListAddPeak / pmPeakAlloc are inconsistent wrt col,row vs x,y + +- why does psMetadataItemAllocV allocate a string of length + MAX_STRING_LENGTH for the comment, then use strncpy? + + should be: + + // set metadata item comment + if (comment == NULL) { + // Per SDRS, null isn't allowed, must use "" instead + metadataItem->comment = psStringCopy (""); + } else { + metadataItem->comment = psStringCopy (comment); + } + +psMetadataAdd -> psMetadataAddItem is all confused: + - we end up with two psMetadataItemAlloc calls, when only one should + be used + - psMetadataItemAlloc was doing the wrong thing if the incoming data + was NULL + - I think the logic in this block is wrong as well. + + * I fixed this in psMetadata.c + +2005.04.14 : + + - psImageStats is still much too slow + + - pmObjects : changed to using psArray to carry peaks / sources, + rather than psList + + - added function pmPeaksSubset to replace pmCullPeaks + + - pmSourceLocalSky: added min/max fncs to force subimage to stay in + image + + - Sx, Sy can go negative: have forced limit to 0,0 + + - pmSourceRoughClass: implemented sigmaX, sigmaY search for stellar + clump (uses pmComparePeakDescend) + + - the search ignores the 0,0 pixel + + - pmSourceRoughClass uses all detected peaks. allow an exclusion + for peak > max, peak < min? + + - need to push clump stats on the metadata + + - the source classes are assigned so they are mutually exclusive + + - change PS_SOURCE_OTHER to STAR? + + - pmSourceSetPixelCircle: + + - name should be pmSourceSetPixelsCircle (geometry could be an + option...) + + - I cleaned the defined region to match the convensions of + pmSourceLocalSky + + - truncate and force subimage to lie on image + + - CheckRadius2 seems like an inefficient coding + + - valid pixels have mask value of 1 (inverse of other funcs) + + - pmSourceFitModel: + + - num iterations is much too large + + - + +mrq: + + - allocate alpha, beta arrays + - mrq2dcof (pars, alpha, beta) + - make guess pars', alpha', beta' + - mrq2dcof (pars', alpha', beta') + - if (chisq < old chisq), keep new guess + + +2005.04.19: + +- psMinimizeLMChi2 has some serious problems. I re-wrote it starting + from the mrqmin example in ohana. after a variety little bugs, it + seems to work quite well. the program 'modeltest.c' runs two + different gaussians through psMinimizeLMChi2, and gets the right + answer quite quickly. The first pass at the implementation had a + variety of problems. to avoid any worry about errors in the + psMatrix code, I implemented psGaussJordan, basing the code on the + elixir gaussj.c code. this worked fine, and let me find the errors + elsewhere in the code. the LUD version now works just the same as + the GaussJ version. + + +2005.04.21 + +- psMinimizeLMChi2 does an excellent job now in most cases. I am able + to run pmSourceFitModel on the PSF stars quite well. I added in the + (nearly) correct errors (actually, just using sqrt(N), limiting to + N>=1). I see one problem object, which is quite faint, and does not + converge well: I get very large values for x,y, Sx, Sy, etc. I need + to think about constraining these fits from running way off scale. + + fit: 0.123879 sec for 277 stars (gauss) + fit: 0.155940 sec for 277 stars (pgauss) (!) + fit: 0.153292 sec for 277 stars + (0.5 msec per star) Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/outline.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/outline.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/outline.txt (revision 41083) @@ -0,0 +1,56 @@ + +Here is a summary outline of the steps taken by psphotReadout: + +* setup (choose recipe, readout, etc) + +* create mask and weight images if needed + +* construct a background model image and subtract it + +* generate the significance image (smoothed by small Gaussian) + +* find peaks and associated footprints + +* create sources for each peak and measure moments + - scan over several window sigma values and choose an optimal window size + - at this stage, the moments are aimed at identifying psf-like objects + +* identify blended sources by proximity and valley depth + +* crude source classification guess based on moments & saturated pixels + - major goal is to identify the psf-stars + - allows for 2D variations in the psf + +* use the selected PSF candidate stars to generate a PSF model + - fits PSF model parameters as a function of position + - uses psfMag - apMag to choose order of 2D (minimize systematic error in this value) + - generates a PSF residual image & optional 2D variations + +* fit all detected sources to the psf model (linear fit for normalization only) + +* high-quality source classification + - uses Mxx, Myy, psfMag - moment->Sum (equiv to apMags) + * should the extended / psf cut be a function of galactic latitude? + * should the cosmic ray / psf cut be a function of galactic latitude? + +* non-linear fit for all brighter sources to single psf, double psf, or extended source model + - sources identified as extended above are fitted with extended source model + * are group / clumps of sources being fitted in the best way? + * is the psf-model ap mag being measured in an appropriate-size aperture? + * is the criterion for choosing between 2 psfs and extended source OK? + * is the test for double psf OK? (uses moments to guess starting positions) + +* subtract, re-detect, measure faint sources (PSF-only) + +* optional extended source aperture-like measurements (petrosian, etc) + +* optional extended source non-linear fits (Sersic, etc) + +* measure psfMag-apMag correcion (2D) + * convert analysis to use systematic error measurement + +* generate magnitudes + +* measure detection efficiency + +* output Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/output.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/output.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/output.txt (revision 41083) @@ -0,0 +1,15 @@ + +we have several output formats: + - TEXT : separate ASCII tables for PSF, EXT, MNT, and NULL + - SX : sextractor-style ASCII file + - OBJ : dophot-style ASCII file + - CMP : old-style elixir format (header + text) + - CMF : fits-table + +we have several options for organization: + - single chip/readout : outroot.ext + - split chips : outroot.NN.ext + - mef chips : outroot.ext (extensions with names) + + * only CMF will work with MEF model + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/psf.load.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/psf.load.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/psf.load.txt (revision 41083) @@ -0,0 +1,75 @@ + +We need to have consistent behavior when we determine the PSF from the image or load it externally. + +Here is a list of the metadata info generated during the psphot analysis (these are mostly stored on the 'analysis' metadata, sometimes in chip and sometimes in readout). + + ANGLE : PSF Model position angle + APLOSS : flux lost out of reference aperture + APMIFIT : aperture magnitude - psf fit magnitude (aperture correction + DAPMIFIT : error on APMIFIT + *EFFECTIVE_AREA : effective area of the PSF model + FLIMIT : limiting flux (not currently set correctly) + FSATUR : saturation flux (not currently set correctly) + FWHM_MAJ : PSF Model FWHM (Major-Axis) + FWHM_MIN : PSF Model FWHM (Minor-Axis) + FW_MJ_LQ : PSF Model FWHM (Major-Axis lower quartile) + FW_MJ_SG : PSF Model FWHM (Major-Axis sigma) + FW_MJ_UQ : PSF Model FWHM (Major-Axis upper quartile) + FW_MN_LQ : PSF Model FWHM (Minor-Axis lower quartile) + FW_MN_SG : PSF Model FWHM (Minor-Axis sigma) + FW_MN_UQ : PSF Model FWHM (Minor-Axis upper quartile) + IQ_FW1 : Moments-based FWHM (Major-Axis) + IQ_FW1_E : Moments-based FWHM (Major-Axis Stdev) + IQ_FW2 : Moments-based FWHM (Minor-Axis) + IQ_FW2_E : Moments-based FWHM (Minor-Axis Stdev) + IQ_M2 : Moments-based shape parameter + IQ_M2C : Moments-based shape parameter + IQ_M2C_E : Moments-based shape parameter + IQ_M2C_L : Moments-based shape parameter + IQ_M2C_U : Moments-based shape parameter + IQ_M2S : Moments-based shape parameter + IQ_M2S_E : Moments-based shape parameter + IQ_M2S_L : Moments-based shape parameter + IQ_M2S_U : Moments-based shape parameter + IQ_M2_ER : Moments-based shape parameter + IQ_M2_LQ : Moments-based shape parameter + IQ_M2_UQ : Moments-based shape parameter + IQ_M3 : Moments-based shape parameter + IQ_M3_ER : Moments-based shape parameter + IQ_M3_LQ : Moments-based shape parameter + IQ_M3_UQ : Moments-based shape parameter + IQ_M4 : Moments-based shape parameter + IQ_M4_ER : Moments-based shape parameter + IQ_M4_LQ : Moments-based shape parameter + IQ_M4_UQ : Moments-based shape parameter + IQ_NSTAR : Number of sources used to measure moments-based parameters + MOMENTS_GAUSS_SIGMA : Sigma of Gaussian window used for moments analysis + MOMENTS_SX_MAX : max allowed Mxx (second moment in x) allowed for PSF clump + MOMENTS_SY_MAX : max allowed Myy (second moment in y) allowed for PSF clump + MSKY_MAX : Sky Model max value + MSKY_MIN : Sky Model min value + MSKY_MN : Sky Model mean value + MSKY_NX : Sky Model number of cells (x) + MSKY_NY : Sky Model number of cells (y) + MSKY_SIG : Sky Model sigma + NAPMIFIT : Number of stars used for APMIFIT analysis + NDET_CR : Number of sources classified as 'cosmic ray' + NDET_EXT : Number of sources classified as 'extended' + NPSFSTAR : Number of sources used for PSF model + NSTARS : Total number of sources + PEAK_THRESHOLD : threshold of peak detection analysis (saved but not re-used) + PSF.CLUMP.DX : size of clump in X + PSF.CLUMP.DY : size of clump in Y + PSF.CLUMP.NREGIONS : number of PSF clump regions measured for this image + PSF.CLUMP.X : center of clump in X + PSF.CLUMP.Y : center of clump in Y + PSFMODEL : do we have a valid PSF model? + PSF_APERTURE : aperture used to measure the aperture magnitude of PSF sources + PSF_CLUMP_GRID_SCALE: pixelization size used for PSF clump search + PSF_FIT_RADIUS : size of window used for PSF model fits + PSF_MAX_NSTARS : maximum number of sources used to find the PSF model + PSF_MOMENTS_RADIUS : size of window used for moments analysis (Gaussian window is applied within this radius) + SIGMA_SMOOTH : sigma of kernel used for smoothing to generate the significance + SIGNIFICANCE_SCALE_FACTOR : factor needed to adjust the level of the significance image + SKY_MEAN : mean of background + SKY_STDEV : stdev of full background image Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/psfmodel.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/psfmodel.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/psfmodel.txt (revision 41083) @@ -0,0 +1,62 @@ + +2007.09.25 + + The PSF model has parameters which vary across the image. This 2D + is can be represented as either a polynomial (ordinary or chebychev) + or as an image map. The user requests the maximum X & Y scale + (PSF.TREND.NX & PSF.TREND.NY) for the variations, and + psphotChoosePSF / pmPSFtry attempt to find the best choice for the + scale. For the polynomial representations, NX and NY are the order + of the variation. For the image map, NX and NY are the size of the + image map. + +2007.09.21 + + there are three places where we can choose to use errors in the fits or not: + + * non-linear fitting of the models to the pixel flux distribution (poissonErrorsPhotLMM) + * linear fitting of the models to the pixel flux distribution (poissonErrorsPhotLin) + * fitting of the 2D variations in the psf parameters (poissonErrorsParams) + * fitting of the 2D variations in the aperture residuals + +2007.09.20 + + I am upgrading the PSF model to allow the parameter variation to be + modeled with pmTrend2D instead of just polynomials. I am making a + list of places to modify the code: + +pmPSFAlloc : need a method beyong psfTrendMask to carry in the psf +options + +pmPSF_ModelToFit : no need to change these + +update pmPSFBuildSimple to set the parameters of the pmTrend, which +ever is used. + +pmPSFtry.c: some significant re-work! + + + +pmPSF_IO : need new functions to save / load the trend (psImages) + +2006.10.27 + + I have been working to fix the PSF modeling in psphot. The PSF + model consists of a flux model for an individual object using an + analytical model with a number of parameters. For a collection of + PSF objects, the variation of the parameters as a function of + position are then themselves fitted with a model. The PSF model for + a single object consists of a radial profile with a functional form + f(z) where a given value of z defines an elliptical coutour of the + form z = \frac{x^2}{2\sigma_x^2} + \frac{y^2}{2\sigma_y^2} + + \sigma_{xy}xy. + + + + The term \sigma_{xy} is difficult to model as a simple function of x + and y (eg, a low-order polynomial). + + A better model can be constructed for + \frac{\sigma_{xy}}{(\sigma_x^{-2} + \sigma_y^{-2})^2}, which varies + like a^2 \sin 2\theta + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/psphot.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/psphot.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/psphot.txt (revision 41083) @@ -0,0 +1,160 @@ + +TBD priorities: + +- test psImageSmooth / remove psImageSmooth_EAM +- unify pmSourceLocalSky / pmSourceLocalSky_EAM +- unify pmSourceMoments_EAM / pmSourceMoments_EAM +- merge with image data hierarchy stuff +- better name for psPSF_Test + +Defined APIs: + +top-level psphot functions : these do not need to be in psModules + psMetadata *psphotArguments (); + psImageData *psphotSetup (); + psStats *psphotImageStats (); + psArray *pmPeaksSigmaLimit (); + psArray *psphotSourceStats (); + pmPSF *psphotChoosePSF (); + bool psphotApplyPSF (); + bool psphotFitGalaxies (); + void psphotOutput (); + +psphot-specific functions + bool psphotMarkPSF (); + bool psphotSubtractPSF (); + int psphotSortBySN (); + int psphotSaveImage (); + +** add the following entries to the psLibSDRS + +psLib extra utilities + bool psTimerStart (); - already in SDRS + void psTimerFree (); - already in SDRS + bool psTimerClear (); - add to psLibSDRS + psF64 psTimerMark (); - already in SDRS + psS32 psLogArguments (); - now: psArgumentVerbosity in SDRS + psS32 psTraceArguments (); - now: psArgumentVerbosity in SDRS + int psArgumentGet (); - already in SDRS + int psArgumentRemove (); - already in SDRS + psVector *psVectorCreate (); - add to psLibSDRS + int psImageCountPixelMask ();- add to psLibSDRS + psVector *psGetRowVectorFromImage(); - add to psLibSDRS? + bool p_psVectorPrintRow (); - add to psLibSDRS? + +// basic image functions + bool psImageInit (); - already in SDRS + void psImageSmooth_EAM (); - test psLib version and drop + +// psLine functions + psLine *psLineAlloc (); - add to psLibSDRS + bool psLineInit (); - add to psLibSDRS + bool psLineAdd (); - add to psLibSDRS + +** compare with current implementation / update psLib + we need to define a mechanism to carry the beta values and the + parameter limits + +// minimize + psBool p_psMinLM_GuessABP_EAM (); + psBool psMinimizeLMChi2_EAM(); + psF64 p_psMinLM_dLinear (); -- not included in pslib.h files + +** already in psLib : some need to be cleaned up by MHPCC + +// polynomial functions + psF64 Polynomial2DEval(); + psImage *psBuildSums2D(); + psPolynomial2D *VectorFitPolynomial2DOrd_EAM(); + psPolynomial2D *Polynomial2DAlloc(); + void psPolynomial2DDump (); + psPolynomial2D *RobustFit2D_nomask(); + psPolynomial2D *RobustFit2D(); + psVector *Polynomial2DEvalVector(); + + psVector *psBuildSums1D(); + void psPolynomial1DDump (); + psF64 Polynomial1DEval_EAM(); + psVector *Polynomial1DEvalVector_EAM(); + psPolynomial1D *Polynomial1DAlloc(); + psPolynomial1D *VectorFitPolynomial1DOrd_EAM(); + +** make sure these are all in the psModules SDRS: + +pmObjects.h: + pmMoments *pmMomentsAlloc(); + pmModel *pmModelAlloc(); + pmSource *pmSourceAlloc(); + psVector *pmFindVectorPeaks() + psArray *pmFindImagePeaks() + psList *pmCullPeaks() + pmSource *pmSourceLocalSky() + bool pmSourceMoments() + pmPSFClump pmSourcePSFClump() + bool pmSourceRoughClass() + bool pmSourceSetPixelsCircle() + pmModel *pmSourceModelGuess() + psArray *pmSourceContour() + bool pmSourceFitModel() + bool pmSourceAddModel() + bool pmSourceSubModel() + int pmModelParameterCount (); + char *pmModelGetType (); + pmModelType pmModelSetType (); + pmModelFunc pmModelFunc_GetFunction (); + pmModelFlux pmModelFlux_GetFunction (); + pmModelRadius pmModelRadius_GetFunction (); + pmModelLimits pmModelLimits_GetFunction (); + pmModelGuessFunc pmModelGuessFunc_GetFunction (); + pmModelFromPSFFunc pmModelFromPSFFunc_GetFunction (); + pmModelFitStatusFunc pmModelFitStatusFunc_GetFunction (); + +model function abstractions: + typedef psMinimizeLMChi2Func pmModelFunc; + typedef psF64 (*pmModelFlux)(const psVector *params); + typedef psF64 (*pmModelRadius)(const psVector *params, double flux); + typedef bool (*pmModelLimits)(psVector **beta_lim, psVector **params_min, psVector **params_max); + typedef bool (*pmModelGuessFunc)(pmModel *model, pmSource *source); + typedef bool (*pmModelFromPSFFunc)(pmModel *modelPSF, pmModel *modelFLT, pmPSF *psf); + typedef bool (*pmModelFitStatusFunc)(pmModel *model); + +conversions between elliptical shape representations + EllipseAxes EllipseMomentsToAxes (); + EllipseShape EllipseAxesToShape (); + EllipseAxes EllipseShapeToAxes (); + +output functions + bool pmSourcesWriteText (); -- be careful about psImageData... + bool pmSourcesWriteOBJ (); -- be careful about psImageData... + bool pmSourcesWriteCMP (); -- be careful about psImageData... + bool pmSourcesWriteCMF (); -- be careful about psImageData... + bool pmSourcesWriteSX (); -- be careful about psImageData... + int pmSourcesDophotType (); + bool pmPeaksWriteText (); + bool pmMomentsWriteText (); + bool pmModelWritePSFs (); + bool pmModelWriteFLTs (); + bool pmModelWriteNULLs (); + +// psf utilities + pmPSF *pmPSFAlloc (); + pmPSF_Test *pmPSF_TestAlloc (); + pmPSF_Test *pmPSF_TestModel (); + bool pmPSFFromModels (); + pmModel *pmModelFromPSF (); + bool pmSourcePhotometry (); + bool pmPSFMetricModel (); + +// psModule extra utilities + bool pmSourceFitModel_EAM(); -- requires updated version of psMinimize... + bool pmSourceLocalSky_EAM (); -- compare with pmSourceLocalSky & choose + bool pmSourceMoments_EAM(); -- compare with pmSourceMoments & choose + bool pmSourceDefinePixels(); -- be careful about psImageData... + bool pmModelFitStatus (); + int pmSourceDophotType (); + psF32 pmConfigLookupF32 (); -- merge with Paul's image container hierarchy + +** this needs to be confronted with the phase2 / image container hierarchy + +// psImageData functions + psImageData *psImageDataAlloc (); Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/regions.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/regions.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/regions.txt (revision 41083) @@ -0,0 +1,159 @@ + +I am having some trouble with functions that refer to both subimages +and regions. there are inconsistencies between the coordinate +conventions. I am going to go through the list of image functions are +decide if we can choose either 1) external coordinates always refer to +parent coordinates or 2) something else! + +bool psImageSet(const psImage *image, int x, int y, double complex value); +double complex psImageGet(const psImage *image, int x, int y); + - sdrs does not specify + - code uses subimage + +psRegion psRegionForImage(psImage *image, psRegion in); + - sdrs says subimage + - code uses subimage + +psImage *psFitsReadImage(psImage *out, const psFits *fits, psRegion region, int z); + - region here specifies a subimage of the disk image + +bool psFitsUpdateImage(psFits *fits, const psImage *input, int x0, int y0, int z); + - x0,y0 here refer to the disk image coordinates + +bool psFitsWriteImage(psFits *fits, psMetadata *header, const psImage +*input, int depth, const char *extname); + - note that the output disk image is truncated to the subimage + +bool psFitsInsertImage(psFits *fits, psMetadata *header, const psImage *input, int depth, + const char *extname, bool after); + - same as above + +psImage *psImageSubset(psImage *image, psRegion region); + - code uses region as coordinates in parent image + - sdrs does not specify + - XXX I think the range tests for region fail if the input is a + subimage + +psImage *psImageCopy(psImage *output, const psImage *input, psElemType type); + - maintains the col0,row0 values + +psVector *psImageRow(psVector *out, const psImage *input, int row); +psVector *psImageCol(psVector *out, const psImage *input, int column); + - code uses subimage coordinates for row and column + +psVector *psImageSlice() +psVector* psImageCut() + - code uses subimage coordinates (no correction) + - sdrs does not specify + +psImage *psImageRebin() + - mask and input image must currently agree + +psImage *psImageTransform() + - sdrs is unclear: I think the region defines the output image such + that the pixel in the input image *parent* coordinates which + corresponds to region.x0,y0 in turn corresponds to the output image + 0,0 pixel; the output image always has col0,row0 = 0,0 + +long psImageCountPixelMask (psImage *mask, psRegion region, psMaskType value); + - sdrs says the region corresponds to subimage coordinates + +psPolynomial2D *psImageFitPolynomial(psPolynomial2D *coeffs, const psImage *input); +psImage *psImageEvalPolynomial(psImage *input, const psPolynomial2D *coeffs); + - sdrs does not specify + - polynomial coordinates should refer to the *parent* image + +double complex psImagePixelInterpolate(input, x, y, ...) + - sdrs does not specify + +int psImageOverlaySection(psImage *image, const psImage *overlay, ) + - sdrs does not specify + +psImage *psPixelsToMask() +psPixels *psPixelsFromMask() + - sdrs does not specify + +void psImageMaskRegion() +void psImageKeepRegion() + - sdrs does not specify + +void psImageMaskCircle() +void psImageKeepCircle() + - sdrs does not specify + +pmReadout *pmSubtractSky(pmReadout *in, psPolynomial2D *poly, psImage *mask, psU8 maskVal, + int binFactor, psStats *stats, float clipSD); + - choice of polynomial coordinates + +stats *pmFringeStats (psArray *fringePoints, psImage *image, psMetadata *config); + - unspecified + +psF32 pmModelEval(pmModel *model, psImage *image, psS32 col, psS32 row); + - sdrs says col,row are in *subimage* coords + +psArray *pmFindImagePeaks(const psImage *image, float threshold); + - does not specify coordinate system + - code uses subimage? + +bool pmSourceDefinePixels() +bool pmSourceRedefinePixels() + - sdrs does not specify very clearly + + + +--------- + +functions which are not affected +psFitsReadImage() +psFitsUpdateImage() +psFitsWriteImage() +psFitsInsertImage() +psImageCopy() -- maintains col0,row0 +psImageRebin() -- mask and image must match + +functions which currently use parent coordinates (CORRECT) +psImageSubset() -- check on range tests +psImageMaskRegion() +psImageKeepRegion() +psImageMaskCircle() +psImageKeepCircle() +pmSourceMoments -- (OK if input peaks are in parent coords) + +functions which currently use subimage coordinates (FIX) +psImageSet() +psImageGet() +psRegionForImage() *fixed* +psImageRow() +psImageCol() +psImageSlice() +psImageCut() +psImageCountPixelMask (); +pmModelEval() +pmFindImagePeaks() *fixed* +pmSourceDefinePixels() *fixed by psRegionForImage fix* +pmSourceRedefinePixels() *fixed by psRegionForImage fix* + +functions which are unclear (CHECK) +psImageTransform() +psImageFitPolynomial() +psImageEvalPolynomial() +psImagePixelInterpolate() +psPixelsToMask() +pmSubtractSky() +pmFringeStats() + + +***** one ambiguity: if we have a subimage, but we do not have the + parent image data (subimage does not carry dimensions of parent + array), then a 0 or negative upper-edge of a region cannot be + defined relative to the parent. we can specify it relative to + the subimage. so, for example, given a subimage [32,100:32,100] + of a much larger image, then psRegionForImage with 0,0,0,0 as + input would return a region with values 32,100:32,100... + + + + + +*** things I've modified + psRegionForImage expects parent (not subimage) coordinates Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/sourcepixels.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/sourcepixels.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/sourcepixels.txt (revision 41083) @@ -0,0 +1,78 @@ + +I was having some bad results because of poor choices for the window +used for the fit or the subtraction. In the past, these were the same +size (or coupled), and as a result I would either use too many pixels +(slow) or too small of a subtraction window (bad residuals). But, the +fit does not need to operate on all of the pixels used for the +subtraction. + +I would like to subtract extended sources out to (say) 4 kron radii, +but fit out to (say) 1 or 2 kron radii. + +Here is where the window and/or the fit / subtraction apertures are +set: + +* psphotSourceStats : when a source if first detected (new peak), the + source is allocated and the window pixels are defined in the initial + loop of psphotSourceStatsReadout. The window is defined based on a + fixed radius of SKY_OUTER_RADIUS (a rather outdated name). There is + no circular mask set at this point. + + If the source seems saturated, I generate a new set of pixels using + a fixed radius (3 * PSF_MOMENTS_RADIUS = 12 * Sigma = 12 * 1.6 * + PSF_Sigma = 19*PSF sigma. this is a bit arbitrary, but sure seems + big enough..) + +* psphotGuessModels : this function sets up the initial PSF-based + guess, and in the process sets the fit radius and window based on + either the value 'PSF_FIT_RADIUS' or on the radius at which the + model flux is a specified fraction of the sky noise. + + NOTE : this also sets a separate radius for the SAT stars, but it is + probably too small (2 x fitRadius) + +* psphotFitSourcesLinear : this function sets a masked-based aperture + based on the value of model->fitRadius (currently disabled) + +* psphotBlendFits : EXT models have the fit radius and subtraction + window set by calls to psphotSetRadiusFootprint and + psphotSetRadiusModel. + + +-- other notes -- + +pmSourceRedefinePixels has a somewhat silly API: there is no need to +pass in the readout since the current pixels point to their parent +images. + +--- + +psphot/src/psphotSourceStats.c: pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); +psphot/src/psphotSourceStats.c: pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); +psphot/src/psphotSourceStats.c: // only slightly: pmSourceRedefinePixels uses the readout to pass the pointers to +psphot/src/psphotSourceStats.c: pmSourceRedefinePixels (source, &tmpReadout, source->peak->x, source->peak->y, BIG_RADIUS + 2); + +psphot/src/psphotMergeSources.c: pmSourceDefinePixels (newSource, readout, newSource->peak->x, newSource->peak->y, OUTER); +psphot/src/psphotMergeSources.c: pmSourceRedefinePixels (sourceOut, readoutOut, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); +psphot/src/psphotMergeSources.c: pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); +psphot/src/psphotEfficiency.c: if (!pmSourceDefinePixels(source, readout, x, y, sourceRadius)) { +psphot/src/psphotSourceMatch.c: pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); +psphot/src/psphotExtendedSourceAnalysisByObject.c: pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 1.5*radius); +psphot/src/psphotPetrosianStudy.c: pmSourceDefinePixels (source, readout, Xo, Yo, 128); +psphot/src/psphotReplaceUnfit.c:bool psphotRedefinePixels (pmConfig *config, const pmFPAview *view, const char *filerule) +psphot/src/psphotReplaceUnfit.c: if (!psphotRedefinePixelsReadout (config, view, filerule, i, recipe)) { +psphot/src/psphotReplaceUnfit.c:bool psphotRedefinePixelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { +psphot/src/psphotReplaceUnfit.c: pmSourceRedefinePixels (source, readout, Xo, Yo, radius); +psphot/src/psphotRadialAperturesByObject.c: // the original view; the following elements get destroyed by pmSourceRedefinePixels so save them: +psphot/src/psphotRadialAperturesByObject.c: pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, outerRadius + 2); +psphot/src/psphotRadialAperturesByObject.c: pmSourceRedefinePixelsByRegion (source, readout, oldRegion); + +psphot/src/psphotRadiusChecks.c: bool status = pmSourceRedefinePixels (source, readout, PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], model->fitRadius); +psphot/src/psphotRadiusChecks.c: bool status = pmSourceRedefinePixels (source, readout, PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], model->fitRadius); +psphot/src/psphotRadiusChecks.c: pmSourceRedefinePixels (source, readout, peak->xf, peak->yf, rawRadius); +psphot/src/psphotRadiusChecks.c: pmSourceRedefinePixels (source, readout, peak->xf, peak->yf, model->fitRadius); + +psphot/src/psphotExtendedSourceAnalysis.c: pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 1.5*radius); +psphot/src/psphotRadialApertures.c: // the original view; the following elements get destroyed by pmSourceRedefinePixels so save them: +psphot/src/psphotRadialApertures.c: pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, outerRadius + 2); +psphot/src/psphotRadialApertures.c: pmSourceRedefinePixelsByRegion (source, readout, oldRegion); Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/speed.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/speed.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/speed.txt (revision 41083) @@ -0,0 +1,129 @@ + +psphot -g +psModule -g +psLib -O2 +on alala: + +alala: psphot -vv starfield.fits starfield starfield.cnf -resid starfield.resid.3.fits + load data: 0.983568 sec + System random seed value used seed = 25904F7C7454AE25 hex + background: 1229.280029 +/- 27.700180 + image stats: 0.063085 sec + System random seed value used seed = 411E5A0600A49AF3 hex + fit background model: 0.580152 sec + smooth: 1.842415 sec + threshold: 83.100540 DN + 4356 peaks: 0.571587 sec + 4230 moments: 3.965099 sec + SN range: 3.355829 - 1043.401001 + identified 107 blended objects (0.121737 sec) + selected candidate 300 PSF objects + fit flt: 3.682850 sec for 300 sources + fit psf: 1.704141 sec for 300 sources + fit stats: 0.000884 +/- 0.004499 + try model PS_MODEL_QGAUSS, ap-fit: -0.007532 +/- 0.004499 : sky bias: -0.717999 + select psf model: 5.402742 sec + selected psf model PS_MODEL_QGAUSS, ApResid: -0.007532 +/- 0.004499 + fitting pixels with at least 27.700180 object counts + built models: 0.991345 (4230 objects) + built matrix: 1.472869 (27998 elements) + measure ensemble of PSFs: 4.334556 + fit PSF models: 12.205738 sec for 4230 objects + replace unfitted models: 0.021473 sec (4230 objects) + measure aperture residuals : 0.451886 sec + measure full-frame aperture residual: 0.454032 sec + aperture residual: -0.002842 +/- 0.002729 : 0.166072 bias + wrote output: 4.648821 sec + complete psphot run: 35.334080 sec + + +psphot -g +psModule -O2 +psLib -O2 +on alala: + +alala: psphot -vv starfield.fits starfield starfield.cnf -resid starfield.resid.3.fits + load data: 1.001502 sec + System random seed value used seed = 5F43A63F532D4B3F hex + background: 1229.280029 +/- 27.700180 + image stats: 0.063169 sec + System random seed value used seed = F5F5FF99A6578F61 hex + fit background model: 0.530384 sec + smooth: 1.856414 sec + threshold: 83.100540 DN + 4365 peaks: 0.237292 sec + 4239 moments: 4.087611 sec + SN range: 3.356469 - 1043.400879 + identified 107 blended objects (0.123372 sec) + selected candidate 300 PSF objects + fit flt: 3.741444 sec for 300 sources + fit psf: 1.676478 sec for 300 sources + fit stats: 0.000883 +/- 0.004499 + try model PS_MODEL_QGAUSS, ap-fit: -0.007532 +/- 0.004499 : sky bias: -0.717985 + select psf model: 5.458156 sec + selected psf model PS_MODEL_QGAUSS, ApResid: -0.007532 +/- 0.004499 + fitting pixels with at least 27.700180 object counts + built models: 0.791613 (4239 objects) + built matrix: 1.267909 (28101 elements) + measure ensemble of PSFs: 3.551364 + fit PSF models: 11.240197 sec for 4239 objects + replace unfitted models: 0.018051 sec (4239 objects) + measure aperture residuals : 0.411856 sec + measure full-frame aperture residual: 0.413843 sec + aperture residual: -0.002831 +/- 0.002717 : 0.164772 bias + wrote output: 4.355429 sec + complete psphot run: 33.126283 sec + +after reducing indirection in pmSourceMoments: + 4268 moments: 3.762452 sec + +psphot -O2 +psModule -O2 +psLib -O2 + +alala: psphot -vv starfield.fits starfield starfield.cnf -resid starfield.resid.3.fits + load data: 0.484324 sec + System random seed value used seed = 980050732A392EC7 hex + background: 1229.280029 +/- 27.700180 + image stats: 0.058759 sec + System random seed value used seed = 44E1A6EE77B0E9CE hex + fit background model: 0.395125 sec + smooth: 1.861776 sec + threshold: 83.100540 DN + 4351 peaks: 0.234874 sec + 4264 moments: 3.719435 sec + SN range: 2.023345 - 1043.401123 + identified 124 blended objects (0.115230 sec) + selected candidate 300 PSF objects + fit flt: 3.525501 sec for 300 sources + fit psf: 1.595031 sec for 300 sources + fit stats: 0.000884 +/- 0.004499 + try model PS_MODEL_QGAUSS, ap-fit: -0.007532 +/- 0.004499 : sky bias: -0.717976 + select psf model: 5.160442 sec + selected psf model PS_MODEL_QGAUSS, ApResid: -0.007532 +/- 0.004499 + fitting pixels with at least 27.700180 object counts + built models: 0.760805 (4264 objects) + built matrix: 0.908674 (23703 elements) + measure ensemble of PSFs: 3.115425 + fit PSF models: 10.162397 sec for 4264 objects + replace unfitted models: 0.017491 sec (4264 objects) + measure aperture residuals : 0.408242 sec + measure full-frame aperture residual: 0.410207 sec + aperture residual: -0.002908 +/- 0.002734 : 0.170692 bias + wrote output: 4.225560 sec + complete psphot run: 30.149281 sec + + + fit PSF models: 10.162397 sec for 4264 objects + select psf model: 5.160442 sec + wrote output: 4.225560 sec + 4264 moments: 3.719435 sec + measure ensemble of PSFs: 3.115425 sec + + other : 3.5 sec + + fixed per objects : 4 msec / object (17 of 30 sec) + + +** still can re-compile with -DPS_NO_TRACE +** ASSERTS cannot be compiled out at the moment! Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/stack.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/stack.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/stack.txt (revision 41083) @@ -0,0 +1,304 @@ + +20110303 + + psphotStack sample timing: + + * convolve images & stamps: 78 sec + * standard photometry : 250? + * petrosians : 6 sec + * ext fits : 350 sec + * radial aps : 140 sec + +20110302 + + * total number of images generated by this process: + + load: + raw : Nfilter x (image + variance + mask) (20M) + cnv : Nfilter x (image + variance + mask) (20M) + (8 * 4B) + (4 * 2B) + + match: (added) + match : Nfilter x (image + variance + mask) (20M) +(4 * 4B) + (2 * 2B) + + phot: (added) + chisq : 1 x (image + variance + mask) (10M) (freed at end of psphotStackReadout?) + backgnd : 1 x (image) (10M) or Nfilter (20M) [unsure] + + load: + 1089174 4000000 psFitsImage.c:467 + 1092246 4000000 psFitsImage.c:467 + 1095902 4000000 psFitsImage.c:467 + 1098974 4000000 psFitsImage.c:467 + 1102619 4000000 psFitsImage.c:467 + 1105669 4000000 psFitsImage.c:467 + 1109303 4000000 psFitsImage.c:467 + 1112353 4000000 psFitsImage.c:467 + 1090766 2000000 psFitsImage.c:467 + 1097494 2000000 psFitsImage.c:467 + 1104200 2000000 psFitsImage.c:467 + 1110884 2000000 psFitsImage.c:467 + + match: + 1128976 4000000 pmSubtractionMatch.c:178 + 1128979 4000000 pmSubtractionMatch.c:183 + 1652553 4000000 pmSubtractionMatch.c:178 + 1652556 4000000 pmSubtractionMatch.c:183 + 1128982 2000000 pmSubtractionMatch.c:189 + 1652559 2000000 pmSubtractionMatch.c:189 + + chisq: + 2133004 4000000 pmFPACopy.c:71 + 2133010 4000000 pmFPACopy.c:71 + 2133007 2000000 pmFPACopy.c:71 + + backmdl: + 2132413 4000000 pmFPAfileDefine.c:1275 + + ??? + 8614548 2000000 psBinaryOp.c:502 + 8617313 2000000 psBinaryOp.c:502 + + pmSource (modelFlx): + 2775 * 6k = 16.9M + + pmSource (maskObj): + 2775 * 3k = 8.4M + + (span + footprints account for ~5 - 10 M, rest in little things) + +20101221 + + psphotStackReadout is now correctly subtracting the PSF models from + the images as it measures the radial aperture fluxes. + + Some issues: + + * the source properties get buggered up by the radial aperture flux + analysis (we need to re-determine the psf, re-create the source + pixels, and re-fit the models (linearly) to subtract them + correctly). + + * the standard analysis sequence is not doing a second pass + + * the choice of the best model is ambiguous + + * the radial aperture (and source addition / subtraction) is only + using the PSF model + + * make sure psphotEfficiency actually subtracts the inserted fake + sources + + * XXX how does the PSF-convolved model work with changing psf models? + +20101207 + + header fields for PSPS: + + * stack needs to count # of inputs + * propagate the input header to the output image (DONE) + * save PSF parameters in header + * stack type is only known to the launcher + + * what are we doing for the stack calibrations?? + +skycellID : code +surveyID : code +filterID : code +stackMetaID : stack_id +photoCalID : photcode -> number +magSat : FSATUR saturation magnitude level ** not correctly set +completMag : FLIMIT 95% completion level in mag ** not correctly set +stackTypeID : STK_TYPE stack type identifier ** deep stack, nightly stack, best IQ stack? +refImageID : identifier of image used as reference for analysis +subtrImageID : N/A (stack) identifier of image subtracted to generate difference image +analVer : analysis version index ** index for tess_id + skycell_id + filter? +nP2Images : NINPUTS number of P2 images contributing to this cell ** missing from input stack +astroScat : astrometric scatter for chip ** measure scatter on stack creation? +photoScat : photometric scatter for chip ** internal scatter? +nAstroRef : number of astrometric reference sources ** (connected to above) +nPhoRef : number of photometric reference sources ** same +psfFwhm : PSF full width at half maximum ** 0.5(FWHM_MAJ + FWHM_MIN) +psfmodelID : * PSFMODEL PSF model identifier ** save the PSF model name in the header +psfWidMajor : * FWHM_MAJ PSF parameters +psfWidMinor : * FWHM_MIN PSF parameters +psfTheta : * ANGLE PSF parameters +psfExtra1 : * PSF_EXT1 PSF parameters ** (at field center?) +psfExtra2 : * PSF_EXT2 PSF parameters ** (not set for all models) +photoZero : local derived photometric zero point +photoColor : local derived photometric color term +ctype1 : * CTYPE1 name of astrometric projection in RA ** propagate from input stacks +ctype2 : * CTYPE2 name of astrometric projection in DEC +crval1 : * CRVAL1 RA corresponding to reference pixel +crval2 : * CRVAL2 DEC corresponding to reference pixel +crpix1 : * CRPIX1 reference pixel value for RA +crpix2 : * CRPIX2 reference pixel value for DEC +cdelt1 : * CDELT1 scale factor for RA +cdelt2 : * CDELT2 scale factor for DEC +pc001001 : * PC001001 elements of rotation/Dcale matrix +pc001002 : * PC001002 elements of rotation/Dcale matrix +pc002001 : * PC002001 elements of rotation/Dcale matrix +pc002002 : * PC002002 elements of rotation/Dcale matrix +polyOrder : * NPLYTERM polynomial order of astrometry fit ** default to 1 +pca1x3y0 : * PCA1X3Y0 polynomial coefficients for the astrometric fit +pca1x2y1 : * PCA1X2Y1 polynomial coefficients for the astrometric fit +pca1x1y2 : * PCA1X1Y2 polynomial coefficients for the astrometric fit +pca1x0y3 : * PCA1X0Y3 polynomial coefficients for the astrometric fit +pca1x2y0 : * PCA1X2Y0 polynomial coefficients for the astrometric fit +pca1x1y1 : * PCA1X1Y1 polynomial coefficients for the astrometric fit +pca1x0y2 : * PCA1X0Y2 polynomial coefficients for the astrometric fit +pca2x3y0 : * PCA2X3Y0 polynomial coefficients for the astrometric fit +pca2x2y1 : * PCA2X2Y1 polynomial coefficients for the astrometric fit +pca2x1y2 : * PCA2X1Y2 polynomial coefficients for the astrometric fit +pca2x0y3 : * PCA2X0Y3 polynomial coefficients for the astrometric fit +pca2x2y0 : * PCA2X2Y0 polynomial coefficients for the astrometric fit +pca2x1y1 : * PCA2X1Y1 polynomial coefficients for the astrometric fit +pca2x0y2 : * PCA2X0Y2 polynomial coefficients for the astrometric fit +calibModNum : calibration modification number +dataRelease : Data release + +20100506: + + we may load RAW and/or CNV images. + + * options->convolve: + * if only one of RAW or CNV exists, convolve it to match target PSF + * if both exist, convolve desired one + + * if matching PSF exists, use it (unless told to re-measure) + + * output goes to OUT (which is then used by psphot analysis routines) + + * detect + + * if (RAW) ? RAW : OUT + + * + + +20100503: + + psphotStack : outline + + * load N images + * make chisq detection image + * detect sources on all (N+1) images + * convolve N input images (to match, to target) + * measure source properties on convolved images + +20100126: + + * watch out for psphotSetMomentsWindow & MOMENTS_SX_MAX,etc + * watch out for psphotSignificanceImage.c:,psphotEfficiency using the FWHM_MAJ from psphotChoosePSF + +20100120 : more stack processing mods: + + there are a number of data collections generated during psphotReadout: + + * detections (currently a structure containing multiple arrays) + * sources (a psArray) + * psf (the psf model structure) + + * new sources vs old sources: + there is a sequence on the second pass in which we need to distinguish the sources + from the first pass from those on the second pass: + + - add noise (old sources only) + - find detections + - subtract noise (old sources only) + - generate sources (new detections only) + - source classifications (new sources only) + - guess models (new sources only) + - replace sources (old sources only) + - merge sources (new + old -> sources) + + currently we are distiguishing the old vs new based on different arrays. + can we use the processing flags to distinguish the these cases and carry around + only a single source list? + + * detections->peaks holds the most recently detected set of peaks + detections->oldPeaks holds the previous collection + + detections->footprints holds the full set of merged footprints, + including assigned peaks from the old and new set. + +20100107 : updates for stack processing + + I am building a new version of psphot that operates on a set of + images (eg, multi-filter or multi-epoch). The assumptions are: + + * each input image represents the same sky pixels : they are warped to a common frame. + * each image has been previously processed, with the background + subtracted (and the psf model determined?) + * all planes are / can be loaded into memory at once (otherwise I + need to add a lot of I/O layers) + + psphot currently expects the pmFPAfile of interest to be available in + config->files with the name PSPHOT.INPUT. To update the code for + stacks, I am extending this concept with the pmFPAfileSelectSingle + API: config->files may contain multiple PSPHOT.INPUT entries, and + functions which access this file / these files need to specify + *which* image they want. The number of PSPHOT.INPUT entries is + saved in config->arguments as PSPHOT.INPUT.NUM (this can be + generated from the filerule PSPHOT.INPUT for extension). + + As a result, we need a number of wrapper functions which loop over + all PSPHOT.INPUT.NUM entries and perform a particular operation on + one of the entries. Here are the functions which I have modified + in this way (function -> child) + + * psphotAddPhotcode -> psphotAddPhotcodeReadout + * note that the photcode is now saved on readout->analysis + * psphotSetMaskAndVariance -> psphotSetMaskAndVarianceReadout + * psphotModelBackground -> psphotModelBackgroundReadoutFileIndex + * psphotSubtractBackground -> psphotSubtractBackgroundReadout + + side notes on + + * psphotModelBackground vs psphotBackgroundModel : I've renamed + psphotBackgroundModel (used only by ppStack for now) to + psphotModelBackgroundReadoutNoFile. I've also named the + single-readout version used by psphotModelBackground to + psphotModelBackgroundReadoutFileIndex. These two functions do the + same thing, but psphotModelBackgroundReadoutNoFile does not save + a pmFPAfile on config->files + + +20090606 : design notes on the multi-image photometry analysis + + assumptions: + + * each input image represents the same sky pixels : they are warped to a common frame. + + * each image has been previously processed, with the background subtracted and the psf model determined + + basic outline: + + * load images, masks, variance : for now, we should load all images. in the future, we might be able to split this into segments. + + * option: model and subtract background for each image (should not be needed) + + * option: smooth each image with a psf or fraction (not needed if stack smooths too much?) + + * generate the significance (chisq) image: X_i = sum(f_i^2 / var_i) + + * perform the peak detection on X_i + + * for each of the input images: + + * generate the significance image S_i = f_i^2 / var_i + + * perform the peak detection on S_i + + * merge the list of peaks + + * generate the footprints + + * linear fit to the peaks with the set of images (psf for each image) + + + design issues: + + * pmSource represents the analysis for an object on a single image + * extended pmSourceSet to group connected pmSources on multiple images? Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.txt (revision 41083) @@ -0,0 +1,84 @@ + + -- threadable? + | + 0.000673 sec NPEAK 0.00 msec / item (505) - ? Find SatStars + 0.231828 sec NPIX 0.00 msec / item (PIX) - ? Replace Background + 0.349591 sec NPIX 0.00 msec / item (2048*4096) - ? Background Model + 0.609246 sec NPIX 0.00 msec / item (2048*4096) - ? Find Peaks + 0.733840 sec NPIX 0.00 msec / item (2048*4096) - ? Find Peaks + 6.076478 sec NPIX 0.00 msec / item (2048*4096) - ? Smooth Images + 7.683749 sec NPIX 0.00 msec / item (2048*4096) - ? Smooth Image + 0.003003 sec NPEAK 0.01 msec / item (505) - ? Source Size + 0.003934 sec NPEAK 0.01 msec / item (??) - ? Source Sizes + 0.019364 sec NPEAK 0.04 msec / item (505) - ? Find Blends + 0.045397 sec NPEAK 0.09 msec / item (??) - ? Replaced Flux + 0.046370 sec NPEAK 0.09 msec / item (505) - ? Add Noise + 0.046680 sec NPEAK 0.09 msec / item (505) - ? Sub Noise + 0.050980 sec NPEAK 0.10 msec / item (505) - ? Replaced Flux + 2.785612 sec NFOOT 0.27 msec / item (10455) - ? Find Footprints + 0.322163 sec NPEAK 0.64 msec / item (505) - ? Rough Class + 0.163793 sec NPEAK 0.66 msec / item (??) - ? Rough Class + 0.820221 sec NFOOT 1.40 msec / item (587) - ? Find Footprints + 2.311492 sec NPEAK 4.58 msec / item (??) - ? Built Models + 2.413531 sec NPEAK 4.78 msec / item (505) - ? Build Models + 1.635674 sec NPEAK 6.54 msec / item (??) - ? Measure Moments + 3.547435 sec NPEAK 7.02 msec / item (505) - ? Measure Moments + 8.118224 sec NSRCS 7.81 msec / item (1040) - ? ApTrend + 4.165990 sec NPEAK 8.25 msec / item (505) - ? Fit PSF (linear) + 4.294968 sec NPEAK 8.50 msec / item (505) - ? Fit PSF (linear) + 4.638916 sec NPEAK 9.19 msec / item (??) - ? Fit PSF (linear) +154.599482 sec NFOOT 14.79 msec / item (10455) - ? Cull Footprints + 1.259951 sec NPSF 17.75 msec / item (71) - ? Fit PSF (non-linear) + 22.009468 sec NFOOT 37.49 msec / item (587) - ? Cull Footprints: + 15.069461 sec NPEAK 38.34 msec / item (393) - ? Fit PSF (non-linear) + EXT (27) + 2.938324 sec NPSF 41.38 msec / item (71) - ? Fit EXT (non-linear) + 2.723019 sec NPSF 53.39 msec / item (51) - ? PSF Residuals + +244.837692 sec complete psphot run: + +Note: + +* PSF Residuals is the most expensive per item, but will always be + performed on a limited number of items (typically < 300 PSF stars). + The maximum impact is thus limited. + +* The non-linear fitting is (not surprisingly) very expensive per + item, but again will only be performed on a limited subset (S/N > + 20?). + +* The Cull Footprints is quite expensive per item, and has the serious + problem that it must be performed on all peaks. This will dominate + all processing time unless we can address it. + +* The linear fitting in this example seems to surprisingly expensive + per item, but I think it is not quite accurate. + +* ApTrend is only measured on a subset of the sources (S/N > XX, max + total number) + +* Measure Moments and Build Models are potentially a substantial drain + since these (probably) should be applied to all objects. Moments is + currently being limited to brighter objects. + +Thread strategies: + + +* tests with wbb4.031117_0154.052.fits test -chip 7 : + +* baseline: + grow : 18.313876 sec + merged : 18.772070 sec + total : 26.958080 sec (dt = 8.18) + +* upgrade to remove psArrayRemove: + grow : 18.327212 sec + merged : 18.778043 sec + total : 25.553486 sec (dt = 6.78; ddt = 1.4) + +* recycle idImage: + grow : 18.383393 sec + merged : 18.848875 sec + total : 24.721451 sec (dt = 5.88; ddt = 2.3) + +(no real difference??) + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v2.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v2.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v2.txt (revision 41083) @@ -0,0 +1,94 @@ +# msec - multithread? +# item | +15.558560 sec 37.31 NBRT 417 : y non-linear fits + 0.000736 sec 0.00 NPEAK 550 : ? satstar blends + 0.005947 sec 0.00 NPEAK 1315 : ? source sizes + 0.076172 sec 0.01 NPEAK 7636 : ? set footprint ids (NPIX?) + 0.003383 sec 0.01 NPEAK 550 : ? source sizes + 0.019354 sec 0.04 NPEAK 550 : ? other blends + 0.050572 sec 0.08 NPEAK 627 : ? set footprint ids + 0.049778 sec 0.09 NPEAK 550 : ? replaced models + 0.051736 sec 0.09 NPEAK 550 : ? add noise + 0.051929 sec 0.09 NPEAK 550 : ? sub noise + 0.055094 sec 0.10 NPEAK 550 : ? replaced models + 0.152204 sec 0.20 NPEAK 765 : ? rough class + 0.8 sec 0.26 NPEAK 3100 : ? merge footprints + 0.268983 sec 0.49 NPEAK 550 : ? rough class + 0.7 sec 1.25 NPEAK 560 : ? cull peaks + 2.092846 sec 2.74 NPEAK 765 : ? moments + 8.9 sec 3.05 NPEAK 2914 : ? cull peaks + 3.339909 sec 4.37 NPEAK 765 : ? built models + 2.572153 sec 4.68 NPEAK 550 : ? build models + 3.816792 sec 6.94 NPEAK 550 : ? moments + 9.337548 sec 7.10 NPEAK 1315 : ? measure magnitudes : for 1315 objects (643 with apertures) + 4.301656 sec 7.82 NPEAK 550 : ? linear fit + 4.429642 sec 8.05 NPEAK 550 : ? linear fit + 4.947403 sec 9.00 NPEAK 550 : ? linear fit + 0.2 sec 0.02 NPIX 8300 k : ? found grown footprints: + 0.205680 sec 0.02 NPIX 8300 k : ? find footprints + 0.211664 sec 0.03 NPIX 8300 k : ? replace background + 0.348357 sec 0.04 NPIX 8300 k : ? subtracted background + 0.598805 sec 0.07 NPIX 8300 k : ? find peaks + 0.599952 sec 0.07 NPIX 8300 k : ? find peaks + 0.723508 sec 0.09 NPIX 8300 k : ? found footprints: + 1.4 sec 0.17 NPIX 8300 k : ? found grown footprints: + 0.809106 sec 12.64 NPSF 64 : ? psf fit + 2.155031 sec 34.21 NPSF 63 : ? full fit + 2.592620 sec 74.07 NPSF 35 : ? psf residuals + 5.931090 sec 0.71 sNPIX 8300 k : ? smoothed signficance + 7.780419 sec 0.94 sNPIX 8300 k : ? smoothed signficance +12.947970 sec 1.56 sNPIX 8300 k : ? convolved with grow disc: +14.525486 sec 1.75 sNPIX 8300 k : ? convolved with grow disc: + +Notes: + +Note: + +* PSF Residuals is the most expensive per item, but will always be + performed on a limited number of items (typically < 300 PSF stars). + The maximum impact is thus limited. + +* The non-linear fitting is (not surprisingly) very expensive per + item, but again will only be performed on a limited subset (S/N > + 20?). + +* the linear fits are the next most costly, and will dominate the + timing since all sources must be so fit, and multiple times. + +* measure magnitudes is expensive, but only runs once + +* measure moments is expensive, but only runs once (and/or on a bright subset?) + +Adding it up: + +GPC1 : 10^11 det for 60*5000* 60 chips --> 5500 det / chip + : 23 megapix + +* cost per object: + + 56.5 msec -> 310 sec for detection analysis (on kawelu, not optimized) + +* cost for bright detections + + ~ 35 sec (for 1000 bright sources) + +* cost per megapixel: + + 5.5 sec + + -> 126 sec for GPC1 + +* other overhead (psf construction, background, smoothing) + + -> 20 sec + +** total : 490 sec (kawelu, not optimized, 1.0GHz!) + +** ~ 220 sec optimized +** ~ 100 sec on 2.3GHz machines +(probably there are other gains on the real hardware: memory bandwidth, ?) + +assuming 100sec is all we get, we need ~1.5 cores per chip to keep up at 60sec / exposure (average) + +120 cores for ppImage (chip) + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v3.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v3.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v3.txt (revision 41083) @@ -0,0 +1,39 @@ +0.000408 sec found 0 satstar blend peaks, leaving 551 sources: +0.002058 sec measure source sizes for 551 sources: +0.003553 sec measure source sizes for 1302 sources: +0.009676 sec identified 36 blended objects: +0.026266 sec replaced models for 551 objects: +0.026892 sec sub noise for 551 objects: +0.026924 sec add noise for 551 objects: +0.029050 sec replaced models for 551 objects: +0.032975 sec set footprint array IDs: +0.047061 sec set footprint array IDs: +0.080533 sec rough classification: +0.107024 sec replace background flux : +0.111200 sec found 631 footprints: +0.132417 sec rough classification: +0.182467 sec subtracted background model: +0.2 sec cull peaks +0.297289 sec 2676 peaks: +0.3 sec rest of foot +0.300492 sec 849 peaks: +0.364908 sec found 7613 footprints: +0.393265 sec measure ensemble of PSFs: +0.405241 sec build median image: +0.454711 sec measure ensemble of PSFs: +0.680027 sec measure ensemble of PSFs: +0.900696 sec 751 sources, 290 moments, 9 failed: +1.1 sec rest of footpr +1.147541 sec built models for 551 objects: +1.289976 sec generate residuals for 35 objects: +1.456877 sec built models for 751 objects: +1.5 sec select psf model (ex resid) +1.661166 sec 551 sources, 551 moments, 8 failed: +2.988601 sec built smoothed signficance image: +3.677652 sec built smoothed signficance image: +4.354904 sec measure magnitudes : for 1302 objects (634 with apertures) +5.8 sec cull peaks +6.594287 sec convolved with grow disc: +7.413455 sec convolved with grow disc: +7.616990 sec fit models: for 417 objects (354 psf, 31 ext, 32 failed, 134 skipped) +55.393567 sec complete psphot run: Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v4.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v4.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/timing.v4.txt (revision 41083) @@ -0,0 +1,105 @@ + +timing on ipp008 *unoptimized* : + +06 : 0.003753 sec : psphotDeblendSatstars +22 : 0.005017 sec : psphotRoughClass +08 : 0.026426 sec : psphotRoughClass +07 : 0.038862 sec : psphotBasicDeblend +24 : 0.215222 sec : psphotReplaceAllSources +16 : 0.217380 sec : psphotAddNoise +20 : 0.217596 sec : psphotSubNoise +14 : 0.233694 sec : psphotReplaceAllSources +21 : 0.366264 sec : psphotSourceStats +29 : 0.435839 sec : psphotSkyReplace +23 : 0.572896 sec : psphotGuessModels +01 : 0.650438 sec : psphotSubtractBackground +03 : 0.711178 sec : psphotFindDetections (peaks) +18 : 0.735544 sec : psphotFindDetections (peaks) +12 : 0.775521 sec : psphotSourceSize +26 : 0.777268 sec : psphotSourceSize +10 : 1.154189 sec : psphotGuessModels +00 : 1.589681 sec : psphotModelBackground +11 : 1.724959 sec : psphotFitSourcesLinear +15 : 1.853397 sec : psphotFitSourcesLinear +25 : 2.392153 sec : psphotFitSourcesLinear +04 : 2.949543 sec : psphotFindDetections (footprints) +27 : 3.238548 sec : psphotApResid +05 : 3.339873 sec : psphotSourceStats +19 : 6.059952 sec : psphotFindDetections (footprints) +28 : 6.948404 sec : psphotMagnitudes +13 : 8.305355 sec : psphotBlendFit +09 : 11.317836 sec : psphotChoosePSF +17 : 16.353613 sec : psphotFindDetections (smooth) +02 : 16.857492 sec : psphotFindDetections (smooth) + +complete psphot run: 95.722628 sec + +timing on ipp008 *optimized* : + +06 : 0.001866 sec : psphotDeblendSatstars +22 : 0.003328 sec : psphotRoughClass +08 : 0.016906 sec : psphotRoughClass +07 : 0.020168 sec : psphotBasicDeblend +24 : 0.068499 sec : psphotReplaceAllSources +16 : 0.069682 sec : psphotAddNoise +20 : 0.072236 sec : psphotSubNoise +14 : 0.087402 sec : psphotReplaceAllSources +29 : 0.093756 sec : psphotSkyReplace +21 : 0.151939 sec : psphotSourceStats +03 : 0.203476 sec : psphotFindDetections (peaks) +18 : 0.210160 sec : psphotFindDetections (peaks) +01 : 0.242636 sec : psphotSubtractBackground +23 : 0.366264 sec : psphotGuessModels +11 : 0.506939 sec : psphotFitSourcesLinear +15 : 0.539799 sec : psphotFitSourcesLinear +12 : 0.542919 sec : psphotSourceSize +26 : 0.550523 sec : psphotSourceSize +25 : 0.724011 sec : psphotFitSourcesLinear +00 : 0.777303 sec : psphotModelBackground +10 : 0.780365 sec : psphotGuessModels +05 : 1.332723 sec : psphotSourceStats +27 : 1.728141 sec : psphotApResid +04 : 2.009312 sec : psphotFindDetections (footprints) +28 : 4.153111 sec : psphotMagnitudes +13 : 4.527469 sec : psphotBlendFit +19 : 4.538490 sec : psphotFindDetections (footprints) +09 : 5.504840 sec : psphotChoosePSF +02 : 7.316287 sec : psphotFindDetections (smooth) +17 : 7.367393 sec : psphotFindDetections (smooth) + +complete psphot run: 49.315078 sec + +timing on ipp008 *threaded* : currently (2009.01.22, 11:48 HST) only affects psphotFindDetections (smooth) + +06 : 0.001843 sec : psphotDeblendSatstars +22 : 0.003371 sec : psphotRoughClass +08 : 0.016955 sec : psphotRoughClass +07 : 0.020379 sec : psphotBasicDeblend +16 : 0.069632 sec : psphotAddNoise +20 : 0.072203 sec : psphotSubNoise +24 : 0.085762 sec : psphotReplaceAllSources +14 : 0.087238 sec : psphotReplaceAllSources +29 : 0.093757 sec : psphotSkyReplace +21 : 0.153800 sec : psphotSourceStats +03 : 0.214963 sec : psphotFindDetections (peaks) +18 : 0.232736 sec : psphotFindDetections (peaks) +01 : 0.248905 sec : psphotSubtractBackground +23 : 0.458357 sec : psphotGuessModels +26 : 0.510125 sec : psphotSourceSize +11 : 0.513049 sec : psphotFitSourcesLinear +15 : 0.538855 sec : psphotFitSourcesLinear +12 : 0.654045 sec : psphotSourceSize +00 : 0.774769 sec : psphotModelBackground +25 : 1.024339 sec : psphotFitSourcesLinear +05 : 1.329678 sec : psphotSourceStats +10 : 1.347421 sec : psphotGuessModels +04 : 2.043960 sec : psphotFindDetections (footprints) +27 : 2.075074 sec : psphotApResid +02 : 2.824503 sec : psphotFindDetections (smooth) ** threaded ** +17 : 2.986548 sec : psphotFindDetections (smooth) ** threaded ** +28 : 4.184633 sec : psphotMagnitudes +19 : 4.323182 sec : psphotFindDetections (footprints) +09 : 5.611547 sec : psphotChoosePSF +13 : 5.620031 sec : psphotBlendFit + +complete psphot run: 41.861923 sec Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/versions.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/versions.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/versions.txt (revision 41083) @@ -0,0 +1,49 @@ + +2006.02.02 + I'm tagging psphot after I updated it to work with the readouts + using the structures being used for ppImage. In this release, I + have also moved to autoconf for psphot, and the psphotReadout + portion is being defined as a library against which the other + elements are linked. It should be possible to use this library with + ppImage, but I have not yet tested that. + + link psphot_dev_08 against psLib:eam_rel9_b2 and + psModule:eam_rel9_b2. + +2006.01.18 + the tag below (psphot_dev_07) was moved to a later release. It + should be used with psLib:eam_rel9_b1 from 2006/1/18 and + psModule:eam_rel9_p0. + +2005.12.23 + psphot_dev_07 should be compiled against psLib:eam_rel9_b1 and + psModule:eam_rel9_b1. This version now handles the ensemble PSF + fitting, detection of blended sources, fitting of 'blends' (groups + of PSFs with distinct be 'blended' peaks) and 'blobs' (unresolved + extended objects, with a test for double PSF vs extended source + model. + +2005.11.25 + + psphot_dev_05 should be compiled against psLib tag eam_rel8_b2 and + psModules tag eam_rel8_b1. this version removes all _EAM versions + of psLib and psModules code. It now uses the psModules version of + the object code. + + psphot_dev_04 should be compiled against psLib tag eam_rel8_b2 this + version removes all _EAM versions of psLib code, so that it is now + consistent with psLib.v8 (fixes are pushed into the eam_rel8_b2 + branch). + +2005.11.16 + + psphot_dev_03 should be compiled against psLib tag eam_rel8_b1, a + branch starting from psLib rel8_0. I needed to make some minor + fixes in psLib to work successfully with psLib rel8_0. + + psphot_dev_02 was getting segfaults from a bug added into psLib in + rel8_0 in p_psVectorClippedStats. I also fixed some errors arising + from changing types (argument to psMetadataParseConfig). + + + Index: /branches/ccl_branches/ipponly-20191108/psphot/doc/visual.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/doc/visual.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/doc/visual.txt (revision 41083) @@ -0,0 +1,47 @@ + +Visualization Options + +image : display image +backgnd : display image +signif : display image + +show image : image 1 or object 1 +show background : image 2 or backgnd 1 +show significance : image 3 or signif 1 +show residual : image 2 or resid 1 + +XXX: these all need to have the image: +show peaks : object 1 or peaks 1 +show moments : object 2 or image.moments 1 +show footprints : object 3 or footprints 1 +show rough class : object 3 or class 3 +show flags : (disabled) +show size class : object 2 or class 2 +show petrosian : object 2 or petro 1 + +plot final moments (2D) : moments 1 +plot test moments (2D) : moments 2 +plot source sizes : class 1 +plot ap resid : apresid 1 +plot chisq : chisq 1 + +show PSF model : psf 1 +show PSF stars : psf 2 +show Sat stars : ? + +show radial profiles : profile + +petrosian visualization: + + psphotPetrosianVisualEllipticalContour + psphotPetrosianVisualStats + psphotPetrosianVisualProfileRadii + psphotPetrosianVisualProfileByAngle + +psphot visualization in psModules: + + pmSourceVisualPSFModelResid + pmSourceVisualShowModelFit + pmSourceVisualShowModelFits + pmSourceVisualPlotPSFMetricSubpix + pmSourceVisualPlotPSFMetric Index: /branches/ccl_branches/ipponly-20191108/psphot/psphot-config.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/psphot-config.in (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/psphot-config.in (revision 41083) @@ -0,0 +1,87 @@ +#! /bin/sh + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ +top_srcdir=@ABS_SRCDIR@ + +usage() +{ + cat <data.F32[0] = So; + params->data.F32[1] = Zo; + params->data.F32[2] = Xo; + params->data.F32[3] = Yo; + params->data.F32[4] = 1 / SigmaX; + params->data.F32[5] = 1 / SigmaY; + params->data.F32[6] = Sxy; + params->data.F32[7] = length; + params->data.F32[8] = theta; +*****************************************************************************/ + +# define PM_MODEL_FUNC pmModelFunc_STRAIL +# define PM_MODEL_FLUX pmModelFlux_STRAIL +# define PM_MODEL_GUESS pmModelGuess_STRAIL +# define PM_MODEL_LIMITS pmModelLimits_STRAIL +# define PM_MODEL_RADIUS pmModelRadius_STRAIL +# define PM_MODEL_FROM_PSF pmModelFromPSF_STRAIL +# define PM_MODEL_PARAMS_FROM_PSF pmModelParamsFromPSF_STRAIL +# define PM_MODEL_FIT_STATUS pmModelFitStatus_STRAIL + +psF32 PM_MODEL_FUNC(psVector *deriv, + const psVector *params, + const psVector *x) +{ + psF32 *PAR = params->data.F32; + + psF32 trailLength = PAR[7]; + psF32 theta = PAR[8]; + + psF32 x0 = PAR[2]; //streak center + psF32 y0 = PAR[3]; //streak center + + //S values (1/sigma for x and y case, sigma for xy case) + psF32 sx = PAR[4]; + psF32 sy = PAR[5]; + psF32 sxy = PAR[6]; + + psF32 sinT=sin(theta); + psF32 cosT=cos(theta); + psF32 sin2T=sin(2.0*theta); + psF32 cos2T=cos(2.0*theta); + + // printf("Trying object at %4.1f,%4.1f with length %3.1f and angle %1.3f\r", x0, y0, length, theta); + + //current location relative to trail center + psF32 X = x->data.F32[0] - x0; + psF32 Y = x->data.F32[1] - y0; + + //x' and y' location (trail-orienter coords) + psF32 xs = X*cosT + Y*sinT; + psF32 ys = -1.0*X*sinT + Y*cosT; + + //initialize variables to be changed below + psF32 x1 = 0; + psF32 y1 = 0; + psF32 px = 0; + psF32 py = 0; + psF32 z = 0; + psF32 zx = 0; + psF32 t = 0; + psF32 tx = 0; + psF32 r = 0; + psF32 rx = 0; + psF32 f = 0; + + psF32 sxrot = 0; + psF32 syrot = 0; + psF32 sxyrot = 0; + psF32 dsxrot = 0; + psF32 dsyrot = 0; + psF32 dsxyrot = 0; + + // psF32 Rx = 0; + // psF32 Ry = 0; + // psF32 Rxy = 0; + + + //calculate new S values (1/sigma) for rotated frame + psF32 sxrotsq = PS_SQR(cosT*sx) + PS_SQR(sinT*sy) + cosT*sinT*sxy; + psF32 syrotsq = PS_SQR(cosT*sy) + PS_SQR(sinT*sx) - cosT*sinT*sxy; + + // psF32 testtwo=10.1; + // psF32 testone=fabsf(testtwo); + // fprintf (stderr, "Test: %f is the absolute value of %f?\n",testone,testtwo); + if (sxrotsq<0) { + sxrot = sqrt(-(sxrotsq)); + syrot = sqrt(syrotsq); + fprintf (stderr, "error in sxrotsq: Neg, sxrotsq=%f sx=%f sy=%f sxy=%f theta=%f\n",sxrotsq,sx,sy,sxy,theta); + } else if (syrotsq<0) { + sxrot = sqrt(sxrotsq); + syrot = sqrt(-(syrotsq)); + fprintf (stderr, "error in syrotsq: Neg, syrotsq=%f sx=%f sy=%f sxy=%f theta=%f\n",syrotsq,sx,sy,sxy,theta); + } else if (sxrotsq==0){ + sxrot = 0.01; + syrot = sqrt(syrotsq); + fprintf (stderr, "error in sxrotsq: Zero, sxrotsq=%f \n",sxrotsq); + } else if (syrotsq==0) { + syrot = 0.01; + sxrot = sqrt(sxrotsq); + fprintf (stderr, "error in syrotsq: Zero, syrotsq=%f \n",syrotsq); + // return(0); + }else { + sxrot = sqrt(sxrotsq); + syrot = sqrt(syrotsq); + } + + sxyrot = sxy*cos2T + (PS_SQR(sy) - PS_SQR(sx))*sin2T; + + if (isnan(sxrot)) { + fprintf (stderr, "error in sxrot syrot=%f sx=%f sy=%f sxy=%f cosT=%f sinT=%f\n",syrot,sx,sy,sxy,cosT,sinT); + } else if (isnan(syrot)) { + fprintf (stderr, "error in syrot sxrot=%f sx=%f sy=%f sxy=%f cosT=%f sinT=%f\n",sxrot,sx,sy,sxy,cosT,sinT); + } + + //calculate length of pipe (not of trail motion) + psF32 length = trailLength - 2.0*2.0/sxrot; + + + if (xs < length/(-2.0)) { + x1 = (xs+length/2.0)*cosT - ys*sinT; //Endpoint1 + y1 = (xs+length/2.0)*sinT + ys*cosT; //Endpoint1 + //1.6 factor comes from by-eye testing of fits, sqrt from the later squaring of it + //1.6 ~= phi (golden mean)...coincidence? + px = sxrot*x1/sqrt(1.6); + py = syrot*y1; + + //first find out what the falloff in the x direction is... + zx = 0.5*PS_SQR(px); + tx = 1 + zx + zx*zx/2.0 + zx*zx*zx/6.0; + rx = 1.0 / (tx + zx*zx*zx*zx/24.0); + + //...and now in the y direction + z = 0.5*PS_SQR(py)+sxyrot*x1*y1; + t = 1 + z + z*z/2.0; + r = 1.0 / (t + z*z*z/6.0); /* exp (-Z) */ + f = PAR[1]*rx*r + PAR[0]; + + } else if (xs > length/2.0){ + x1 = (xs-length/2.0)*cosT - ys*sinT; //Endpoint2 + y1 = (xs-length/2.0)*sinT + ys*cosT; //Endpoint2 + px = sxrot*x1/sqrt(1.6); + py = syrot*y1; + zx = 0.5*PS_SQR(px); + tx = 1 + zx + zx*zx/2.0 + zx*zx*zx/6.0; + rx = 1.0 / (tx + zx*zx*zx*zx/24.0); + + z = 0.5*PS_SQR(py)+sxyrot*x1*y1; + t = 1 + z + z*z/2.0; + r = 1.0 / (t + z*z*z/6.0); /* exp (-Z) */ + f = PAR[1]*rx*r + PAR[0]; + + if (isnan(r)) { + fprintf (stderr, "error in +r t=%f z=%f\n",t,z); + } + + } else { + x1 = -ys*sinT; + y1 = ys*cosT; + px = sx*x1; + py = sy*y1; + z = 0.5*PS_SQR(px) + 0.5*PS_SQR(py) + sxy*x1*y1; + t = 1 + z + z*z/2.0; + r = 1.0 / (t + z*z*z/6.0); /* exp (-Z) */ + rx = 1.0; //so that dF/dF0 can be generalized + f = PAR[1]*r + PAR[0]; + + if (isnan(r)) { + fprintf (stderr, "error in midr t=%f z=%f\n",t,z); + } + } + + + //ok...so this is df/dPAR[X] + if (deriv != NULL) { + psF32 q = 1; + //stable + deriv->data.F32[0] = +1.0; + deriv->data.F32[1] = +r * rx; + + if (isnan(deriv->data.F32[0])) { + fprintf (stderr, "error in deriv0\n"); + } else if (isnan(deriv->data.F32[1])) { + fprintf (stderr, "error in deriv1 r=%f rx=%f\n",r,rx); + } + + dsxrot = 2.0*PS_SQR(sy)*sinT*cosT - 2.0*PS_SQR(sx)*sinT*cosT + sxy*(PS_SQR(cosT)-PS_SQR(sinT)); + dsyrot = 2.0*PS_SQR(sx)*sinT*cosT - 2.0*PS_SQR(sy)*sinT*cosT - sxy*(PS_SQR(cosT)-PS_SQR(sinT)); + dsxyrot = 2.0*cos2T*(PS_SQR(sy)-PS_SQR(sx)) - 2.0*sxy*sin2T; + + if (isnan(dsxrot)) { + fprintf (stderr, "error in dsxrot\n"); + } else if (isnan(dsyrot)) { + fprintf (stderr, "error in dsyrot\n"); + } else if (isnan(dsxyrot)) { + fprintf (stderr, "error in dsxyrot\n"); + } + + //initialize variables definied in the if statements + // psF32 XXX=0; + // psF32 YYY=0; + + + // variable over piecewise func + // change the endcaps to be 4th order gaussians with sxrot_fit=1.6*sxrot + // y' direction can ge adequately modelled by a 3rd order gaussian with syrot + if (xs < length/(-2.0)) { + + q=PAR[1]*r*rx; + deriv->data.F32[2] = -q*(rx*tx/1.6*x1*PS_SQR(sxrot) + r*t*y1*sxyrot); + deriv->data.F32[3] = -q*r*t*(y1*PS_SQR(syrot) + x1*sxyrot); + deriv->data.F32[4] = -q*(rx*tx*x1*sx*PS_SQR(cosT)/1.6*(x1+2.0*cosT/sxrot) + r*t*y1*sx*sinT*(y1*sinT+2.0*PS_SQR(syrot)*PS_SQR(cosT)/(sxrot*sxrot*sxrot)) + 2.0*r*t*sx*(-1.0*x1*y1*sin2T+y1*sxyrot*(cosT*cosT*cosT)/(sxrot*sxrot*sxrot)+x1*sxyrot*(sinT*cosT*cosT)/(sxrot*sxrot*sxrot))); + deriv->data.F32[5] = -q*(rx*tx*x1*sy*PS_SQR(sinT)/1.6*(x1+2.0*cosT/sxrot) + r*t*y1*sy*(y1*PS_SQR(cosT)+2.0*PS_SQR(syrot)*(sinT*sinT*sinT)/(sxrot*sxrot*sxrot)) + 2.0*r*t*sy*(x1*y1*sin2T+y1*sxyrot*(sinT*sinT*cosT)/(sxrot*sxrot*sxrot)+x1*sxyrot*(sinT*sinT*sinT)/(sxrot*sxrot*sxrot))); + deriv->data.F32[6] = -q*(rx*tx*x1*cosT*sinT/3.2*(x1+2.0*cosT/sxrot) + r*t*y1*cosT*sinT/2.0*(-y1+2.0*PS_SQR(syrot)*sinT/(sxrot*sxrot*sxrot)) + r*t*(x1*y1*cos2T+y1*sxyrot*sinT*cosT*cosT/(sxrot*sxrot*sxrot)+x1*sxyrot*sinT*sinT*cosT/(sxrot*sxrot*sxrot))); + deriv->data.F32[7] = -q*(rx*tx*PS_SQR(sxrot)*x1*cosT/3.2 + r*t*PS_SQR(syrot)*y1*sinT/2.0 + r*t*sxyrot/2.0*(y1*cosT+x1*sinT)); + deriv->data.F32[8] = -q*(rx*tx*x1/3.2*(x1*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))+PS_SQR(sxrot)*(2.0*cosT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot) - sinT*length)) + r*t*y1/2.0*(y1*(2.0*sinT*cosT*(PS_SQR(sx)-PS_SQR(sy))-sxy*(PS_SQR(cosT)-PS_SQR(sinT)))+PS_SQR(syrot)*(2.0*sinT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot)+cosT*length)) + r*t*(2.0*x1*y1*(cos2T*(PS_SQR(sy)-PS_SQR(sx))-sxy*sin2T)+y1*sxyrot/2.0*(2.0*cosT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot)-length*sinT)+x1*sxyrot/2.0*(2.0*sinT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot)+length*cosT))); + + /* if (isnan(deriv->data.F32[2])) { + fprintf (stderr, "error in deriv2\n"); + } else if (isnan(deriv->data.F32[3])) { + fprintf (stderr, "error in deriv3\n"); + } else if (isnan(deriv->data.F32[4])) { + fprintf (stderr, "error in deriv4\n"); + } else if (isnan(deriv->data.F32[5])) { + fprintf (stderr, "error in deriv5\n"); + } else if (isnan(deriv->data.F32[6])) { + fprintf (stderr, "error in deriv6\n"); + } else if (isnan(deriv->data.F32[7])) { + fprintf (stderr, "error in deriv7\n"); + } else if (isnan(deriv->data.F32[8])) { + fprintf (stderr, "error in deriv8\n"); + } + */ + + + + } else if (xs > length/2.0){ + + q=PAR[1]*r*rx; + deriv->data.F32[2] = -q*(rx*tx/1.6*x1*PS_SQR(sxrot) + r*t*y1*sxyrot); + deriv->data.F32[3] = -q*r*t*(y1*PS_SQR(syrot) + x1*sxyrot); + deriv->data.F32[4] = -q*(rx*tx*x1*sx*PS_SQR(cosT)/1.6*(x1-2.0*cosT/sxrot) + r*t*y1*sx*sinT*(y1*sinT-2.0*PS_SQR(syrot)*PS_SQR(cosT)/(sxrot*sxrot*sxrot)) + 2.0*r*t*sx*(-1.0*x1*y1*sin2T-y1*sxyrot*(cosT*cosT*cosT)/(sxrot*sxrot*sxrot)-x1*sxyrot*(sinT*cosT*cosT)/(sxrot*sxrot*sxrot))); + deriv->data.F32[5] = -q*(rx*tx*x1*sy*PS_SQR(sinT)/1.6*(x1-2.0*cosT/sxrot) + r*t*y1*sy*(y1*PS_SQR(cosT)-2.0*PS_SQR(syrot)*(sinT*sinT*sinT)/(sxrot*sxrot*sxrot)) + 2.0*r*t*sy*(x1*y1*sin2T-y1*sxyrot*(sinT*sinT*cosT)/(sxrot*sxrot*sxrot)-x1*sxyrot*(sinT*sinT*sinT)/(sxrot*sxrot*sxrot))); + deriv->data.F32[6] = -q*(rx*tx*x1*cosT*sinT/3.2*(x1-2.0*cosT/sxrot) + r*t*y1*cosT*sinT/2.0*(-y1-2.0*PS_SQR(syrot)*sinT/(sxrot*sxrot*sxrot)) + r*t*(x1*y1*cos2T-y1*sxyrot*sinT*cosT*cosT/(sxrot*sxrot*sxrot)-x1*sxyrot*sinT*sinT*cosT/(sxrot*sxrot*sxrot))); + deriv->data.F32[7] = q*(rx*tx*PS_SQR(sxrot)*x1*cosT/3.2 + r*t*PS_SQR(syrot)*y1*sinT/2.0 + r*t*sxyrot/2.0*(y1*cosT+x1*sinT)); + deriv->data.F32[8] = -q*(rx*tx*x1/3.2*(x1*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))-PS_SQR(sxrot)*(2.0*cosT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot) - sinT*length)) + r*t*y1/2.0*(y1*(2.0*sinT*cosT*(PS_SQR(sx)-PS_SQR(sy))-sxy*(PS_SQR(cosT)-PS_SQR(sinT)))-PS_SQR(syrot)*(2.0*sinT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot)+cosT*length)) + r*t*(2.0*x1*y1*(cos2T*(PS_SQR(sy)-PS_SQR(sx))-sxy*sin2T)-y1*sxyrot/2.0*(2.0*cosT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot)-length*sinT)-x1*sxyrot/2.0*(2.0*sinT*(2.0*sinT*cosT*(PS_SQR(sy)-PS_SQR(sx))+sxy*(PS_SQR(cosT)-PS_SQR(sinT)))/(sxrot*sxrot*sxrot)+length*cosT))); + + + /* if (isnan(deriv->data.F32[2])) { + fprintf (stderr, "error in deriv2\n"); + } else if (isnan(deriv->data.F32[3])) { + fprintf (stderr, "error in deriv3\n"); + } else if (isnan(deriv->data.F32[4])) { + fprintf (stderr, "error in deriv4\n"); + } else if (isnan(deriv->data.F32[5])) { + fprintf (stderr, "error in deriv5\n"); + } else if (isnan(deriv->data.F32[6])) { + fprintf (stderr, "error in deriv6\n"); + } else if (isnan(deriv->data.F32[7])) { + fprintf (stderr, "error in deriv7\n"); + } else if (isnan(deriv->data.F32[8])) { + fprintf (stderr, "error in deriv8\n"); + } + */ + + } else { + // this does not change from before, as the y' falloff can be modelled by the standard 3rd order gaussian + // note difference from a pure gaussian: q = PAR[1]*r + q = PAR[1]*r*r*t; + deriv->data.F32[2] = q*(PS_SQR(sx*sinT)*x1 - PS_SQR(sy)*y1*cosT*sinT - sxy*x1*sinT*cosT + sxy*y1*PS_SQR(sinT)); + deriv->data.F32[3] = q*(-1*PS_SQR(sx)*x1*sinT*cosT + PS_SQR(sy*cosT)*y1 + sxy*x1*PS_SQR(cosT) - sxy*y1*sinT*cosT); + deriv->data.F32[4] = -q*sx*PS_SQR(x1); + deriv->data.F32[5] = -q*sy*PS_SQR(y1); + deriv->data.F32[6] = -q*x1*y1; + deriv->data.F32[7] = 0; + deriv->data.F32[8] = -q*( PS_SQR(sx)*x1*(xs*sinT - ys*cosT) + PS_SQR(sy)*y1*(xs*cosT - ys*sinT) + sxy*x1*(xs*cosT - ys*sinT) + sxy*y1*(xs*sinT - ys*cosT) ); + + if (isnan(deriv->data.F32[2])) { + fprintf (stderr, "error in deriv2\n"); + } else if (isnan(deriv->data.F32[3])) { + fprintf (stderr, "error in deriv3\n"); + } else if (isnan(deriv->data.F32[4])) { + fprintf (stderr, "error in deriv4\n"); + } else if (isnan(deriv->data.F32[5])) { + fprintf (stderr, "error in deriv5\n"); + } else if (isnan(deriv->data.F32[6])) { + fprintf (stderr, "error in deriv6\n"); + } else if (isnan(deriv->data.F32[7])) { + fprintf (stderr, "error in deriv7\n"); + } else if (isnan(deriv->data.F32[8])) { + fprintf (stderr, "error in deriv8\n"); + } + + + } + } + return(f); +} + +//fixed +// XXX this needs to apply the axis ratio limits to prevent avoid solutions +# define AR_MAX 20.0 +# define AR_RATIO 0.99 +bool PM_MODEL_LIMITS (psMinConstraintMode mode, int nParam, float *params, float *beta) { + + float beta_lim = 0; + float params_min = 0; + float params_max = 0; + float f1, f2, q1; + float q2 = 0; + + // we need to calculate the limits for SXY specially + if (nParam == PM_PAR_SXY) { + f1 = 1.0 / PS_SQR(params[PM_PAR_SYY]) + 1.0 / PS_SQR(params[PM_PAR_SXX]); + f2 = 1.0 / PS_SQR(params[PM_PAR_SYY]) - 1.0 / PS_SQR(params[PM_PAR_SXX]); + q1 = PS_SQR(f1)*AR_RATIO - PS_SQR(f2); + assert (q1 > 0); + q2 = 0.5*sqrt (q1); + } + + switch (mode) { + case PS_MINIMIZE_BETA_LIMIT: + switch (nParam) { + case PM_PAR_SKY: beta_lim = 1000; break; + case PM_PAR_I0: beta_lim = 10000; break; // too small? + case PM_PAR_XPOS: beta_lim = 50; break; + case PM_PAR_YPOS: beta_lim = 50; break; + case PM_PAR_SXX: beta_lim = 0.5; break; + case PM_PAR_SYY: beta_lim = 0.5; break; + case PM_PAR_SXY: beta_lim = 1.0; break; // set this to q2? + case 7: beta_lim = 10.0; break; + case 8: beta_lim = M_PI/6.0; break; + + default: + psAbort("invalid parameter %d for beta test", nParam); + } + if (fabs(beta[nParam]) > fabs(beta_lim)) { + beta[nParam] = (beta[nParam] > 0) ? fabs(beta_lim) : -fabs(beta_lim); + return false; + } + return true; + case PS_MINIMIZE_PARAM_MIN: + switch (nParam) { + case PM_PAR_SKY: params_min = -1000; break; + case PM_PAR_I0: params_min = 0; break; + case PM_PAR_XPOS: params_min = -100; break; + case PM_PAR_YPOS: params_min = -100; break; + case PM_PAR_SXX: params_min = 0.5; break; + case PM_PAR_SYY: params_min = 0.5; break; + case PM_PAR_SXY: params_min = -5.0; break; // set this to -q2? + case 7: params_min = 0; break; + case 8: params_min = -1*M_PI; break; + + default: + psAbort("invalid parameter %d for param min test", nParam); + } + if (params[nParam] < params_min) { + params[nParam] = params_min; + return false; + } + return true; + case PS_MINIMIZE_PARAM_MAX: + switch (nParam) { + case PM_PAR_SKY: params_max = 1e5; break; + case PM_PAR_I0: params_max = 1e8; break; + case PM_PAR_XPOS: params_max = 1e4; break; + case PM_PAR_YPOS: params_max = 1e4; break; + case PM_PAR_SXX: params_max = 100; break; + case PM_PAR_SYY: params_max = 100; break; + case PM_PAR_SXY: params_max = +q2; break; + case 7: params_max = 150; break; + case 8: params_max = M_PI; break; + default: + psAbort("invalid parameter %d for param max test", nParam); + } + if (params[nParam] > params_max) { + params[nParam] = params_max; + return false; + } + return true; + default: + psAbort("invalid choice for limits"); + } + psAbort("should not reach here"); + return false; +} + +//fixed +psF64 PM_MODEL_FLUX(const psVector *params) +{ + float f, norm, z; + + psF32 *PAR = params->data.F32; + + psF64 A1 = PS_SQR(PAR[4]); + psF64 A2 = PS_SQR(PAR[5]); + psF64 A3 = PS_SQR(PAR[6]); + psF32 Rx=2./PS_SQR(PAR[4]); + psF32 Ry=2./PS_SQR(PAR[5]); + psF32 Rxy=PAR[6]; + + + psF32 theta = PAR[8]; + psF32 sinT=sin(theta); + psF32 cosT=cos(theta); + + psF32 Syrot = ( PS_SQR(sinT)/Rx + PS_SQR(cosT)/Ry - Rxy*sinT*cosT ); //rotated sigma y + + psF64 A4 = Syrot*PAR[7]; + + psF64 Area = 2.0 * M_PI / sqrt(A1*A2 - A3) + A4; + // Area is equivalent to 2 pi sigma^2 + rectangle + + // the area needs to be multiplied by the integral of f(z) + norm = 0.0; + for (z = 0.005; z < 50; z += 0.01) { + f = 1.0 / (1 + z + z*z/2 + z*z*z/6); + norm += f; + } + norm *= 0.01; + + psF64 Flux = params->data.F32[1] * Area * norm; + + return(Flux); +} + +// define this function so it never returns Inf or NaN +// also prevent 0 returns, and just send a v small number +// return the radius which yields the requested flux + +//fixed, but need to change how it is called to accomodate 2 radii +psF64 PM_MODEL_RADIUS (const psVector *params, psF64 flux) +{ + if (flux <= 0) return (1.0); + if (params->data.F32[1] <= 0) return (1.0); + if (flux >= params->data.F32[1]) return (1.0); + + psF32 *PAR = params->data.F32; + psF32 sigma = sqrt(2.0) * hypot (1.0 / params->data.F32[4], 1.0 / params->data.F32[5]); + + psF32 theta = PAR[8]; + psF32 sinT=sin(theta); + psF32 cosT=cos(theta); + psF32 Rx=2./PS_SQR(PAR[4]); + psF32 Ry=2./PS_SQR(PAR[5]); + psF32 Rxy=PAR[6]; + psF32 length=PAR[7]; + + psF32 Syrot = ( PS_SQR(sinT)/Rx + PS_SQR(cosT)/Ry - Rxy*sinT*cosT ); //rotated sigma y + + psF64 radius = 0; + if (flux > 0){ + psF64 radius0 = sigma * sqrt (2.0 * log(params->data.F32[1] / flux)); + psF64 radius1 = Syrot * sqrt (2.0 * log(params->data.F32[1] / flux)); + + if (radius0 > radius1) { + radius=radius0+length/2.0; + } else { + radius=radius1+length/2.0; + } + } else { + radius = 1000; + } + + if (radius < 0.01){ + radius = 0.01; + } + + if (isnan(radius)) { + fprintf (stderr, "error in code\n"); + } + return (radius); +} + +//fixed I think...no good way of guessing as far as I can tell +bool PM_MODEL_GUESS (pmModel *model, pmSource *source, psImageMaskType maskVal, psImageMaskType markVal) +{ + pmMoments *Smoments = source->moments; + pmPeak *peak = source->peak; + psF32 *params = model->params->data.F32; + + psEllipseAxes axes; + psEllipseShape shape; + psEllipseMoments moments; + + moments.x2 = Smoments->Mxx; + moments.y2 = Smoments->Myy; + moments.xy = Smoments->Mxy; + //sometimes these moment inputs are zero...why? + + // solve the math to go from Moments To Shape + // limit the axis ratio to 20 (a guess) + axes = psEllipseMomentsToAxes(moments, 20.0); + shape = psEllipseAxesToShape(axes); + + params[PM_PAR_SKY] = Smoments->Sky; + params[PM_PAR_I0] = peak->rawFlux; + params[PM_PAR_XPOS] = peak->xf; + params[PM_PAR_YPOS] = peak->yf; + + params[7] = 2 * axes.major; + params[8] = axes.theta; + + if (moments.x2 == 0 || moments.y2 == 0){ + params[4] = 2.0; + params[5] = 2.0; + params[6] = 0.0; + } else { + params[4] = 1.0 / shape.sx; + params[5] = 1.0 / shape.sy; + params[6] = shape.sxy; + } + + // printf("Who is NaN? momx: %4.3f momy: %4.3f momxy: %4.3f\n", moments.x2,moments.y2, moments.xy); + + return(true); +} + +//fixed +bool PM_MODEL_FROM_PSF (pmModel *modelPSF, pmModel *modelFLT, const pmPSF *psf) +{ + psF32 *out = modelPSF->params->data.F32; + psF32 *in = modelFLT->params->data.F32; + + out[0] = in[0]; + out[1] = in[1]; + out[2] = in[2]; + out[3] = in[3]; + out[7] = in[7]; + out[8] = in[8]; + + for (int i = 4; i < 7; i++) { + pmTrend2D *trend = psf->params->data[i-4]; + out[i] = pmTrend2DEval (trend, out[2], out[3]); + } + return(true); +} + +// construct the PSF model from the FLT model and the psf +// XXX is this sufficiently general do be a global function, not a pmModelClass function? +bool PM_MODEL_PARAMS_FROM_PSF (pmModel *model, const pmPSF *psf, float Xo, float Yo, float Io) +{ + psF32 *PAR = model->params->data.F32; + + // we require these two parameters to exist + assert (psf->params->n > PM_PAR_YPOS); + assert (psf->params->n > PM_PAR_XPOS); + + PAR[PM_PAR_SKY] = 0.0; + PAR[PM_PAR_I0] = Io; + PAR[PM_PAR_XPOS] = Xo; + PAR[PM_PAR_YPOS] = Yo; + + // supply the model-fitted parameters, or copy from the input + for (int i = 0; i < psf->params->n; i++) { + if (i == PM_PAR_SKY) continue; + pmTrend2D *trend = psf->params->data[i]; + PAR[i] = pmTrend2DEval(trend, Xo, Yo); + } + + // the 2D PSF model fits polarization terms (E0,E1,E2) + // convert to shape terms (SXX,SYY,SXY) + // XXX user-defined value for limit? + bool useReff = pmModelUseReff (model->type); + if (!pmPSF_FitToModel (PAR, 0.1, useReff)) { + psError(PM_ERR_PSF, false, "Failed to fit object at (r,c) = (%.1f,%.1f)", Xo, Yo); + return false; + } + + // apply the model limits here: this truncates excessive extrapolation + // XXX do we need to do this still? should we put in asserts to test? + for (int i = 0; i < psf->params->n; i++) { + // apply the limits to all components or just the psf-model parameters? + if (psf->params->data[i] == NULL) + continue; + + bool status = true; + status &= PM_MODEL_LIMITS (PS_MINIMIZE_PARAM_MIN, i, PAR, NULL); + status &= PM_MODEL_LIMITS (PS_MINIMIZE_PARAM_MAX, i, PAR, NULL); + if (!status) { + psTrace ("psModules.objects", 5, "Hitting parameter limits at (r,c) = (%.1f, %.1f)", Xo, Yo); + model->flags |= PM_MODEL_STATUS_LIMITS; + } + } + return(true); +} + +//done I think +bool PM_MODEL_FIT_STATUS (pmModel *model) +{ + psF32 dP; + bool status; + + psF32 *PAR = model->params->data.F32; + psF32 *dPAR = model->dparams->data.F32; + + dP = 0; + dP += PS_SQR(dPAR[4] / PAR[4]); + dP += PS_SQR(dPAR[5] / PAR[5]); + dP = sqrt (dP); + + status = true; + status &= (dP < 0.5); + status &= (PAR[1] > 0); + status &= ((dPAR[1]/PAR[1]) < 0.5); + // status &= ((dPAR[7]/PAR[7]) < 0.5); + // status &= ((dPAR[8]/PAR[8]) < 0.5); + + if (status) return true; + return false; +} + +# undef PM_MODEL_FUNC +# undef PM_MODEL_FLUX +# undef PM_MODEL_GUESS +# undef PM_MODEL_LIMITS +# undef PM_MODEL_RADIUS +# undef PM_MODEL_FROM_PSF +# undef PM_MODEL_PARAMS_FROM_PSF +# undef PM_MODEL_FIT_STATUS Index: /branches/ccl_branches/ipponly-20191108/psphot/src/models/pmModel_TEST1.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/models/pmModel_TEST1.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/models/pmModel_TEST1.c (revision 41083) @@ -0,0 +1,323 @@ +/****************************************************************************** + * this file defines the TEST1 source shape model. Note that these model functions are loaded + * by pmModelGroup.c using 'include', and thus need no 'include' statements of their own. The + * models use a psVector to represent the set of parameters, with the sequence used to specify + * the meaning of the parameter. The meaning of the parameters may thus vary depending on the + * specifics of the model. All models which are used a PSF representations share a few + * parameters, for which # define names are listed in pmModel.h: + + * PM_PAR_SKY 0 - local sky : note that this is unused and may be dropped in the future + * PM_PAR_I0 1 - central intensity + * PM_PAR_XPOS 2 - X center of object + * PM_PAR_YPOS 3 - Y center of object + * PM_PAR_SXX 4 - X^2 term of elliptical contour (sqrt(2) * SigmaX) + * PM_PAR_SYY 5 - Y^2 term of elliptical contour (sqrt(2) * SigmaY) + * PM_PAR_SXY 6 - X*Y term of elliptical contour + *****************************************************************************/ + +# define PM_MODEL_FUNC pmModelFunc_TEST1 +# define PM_MODEL_FLUX pmModelFlux_TEST1 +# define PM_MODEL_GUESS pmModelGuess_TEST1 +# define PM_MODEL_LIMITS pmModelLimits_TEST1 +# define PM_MODEL_RADIUS pmModelRadius_TEST1 +# define PM_MODEL_FROM_PSF pmModelFromPSF_TEST1 +# define PM_MODEL_PARAMS_FROM_PSF pmModelParamsFromPSF_TEST1 +# define PM_MODEL_FIT_STATUS pmModelFitStatus_TEST1 + +// the model is a function of the pixel coordinate (pixcoord[0,1] = x,y) +psF32 PM_MODEL_FUNC(psVector *deriv, + const psVector *params, + const psVector *pixcoord) +{ + psF32 *PAR = params->data.F32; + + // XXX this is fitting sigma_x/sqrt(2), sigma_y/sqrt(2) + psF32 X = pixcoord->data.F32[0] - PAR[PM_PAR_XPOS]; + psF32 Y = pixcoord->data.F32[1] - PAR[PM_PAR_YPOS]; + psF32 px = X / PAR[PM_PAR_SXX]; + psF32 py = Y / PAR[PM_PAR_SYY]; + psF32 z = PS_SQR(px) + PS_SQR(py) + PAR[PM_PAR_SXY]*X*Y; + psF32 t = 1 + z + z*z/2.0; + psF32 r = 1.0 / (t + z*z*z/6.0); /* exp (-Z) */ + psF32 f = PAR[PM_PAR_I0]*r + PAR[PM_PAR_SKY]; + + if (deriv != NULL) { + psF32 *dPAR = deriv->data.F32; + psF32 q = PAR[PM_PAR_I0]*r*r*t; + dPAR[PM_PAR_SKY] = +1.0; + dPAR[PM_PAR_I0] = +r; + dPAR[PM_PAR_XPOS] = q*(2.0*px/PAR[PM_PAR_SXX] + Y*PAR[PM_PAR_SXY]); + dPAR[PM_PAR_YPOS] = q*(2.0*py/PAR[PM_PAR_SYY] + X*PAR[PM_PAR_SXY]); + dPAR[PM_PAR_SXX] = +2.0*q*px*px/PAR[PM_PAR_SXX]; + dPAR[PM_PAR_SYY] = +2.0*q*py*py/PAR[PM_PAR_SYY]; + dPAR[PM_PAR_SXY] = -q*X*Y; + } + return(f); +} + +// define the parameter limits +bool PM_MODEL_LIMITS (psMinConstraintMode mode, int nParam, float *params, float *beta) +{ + float beta_lim = 0; + float params_min = 0; + float params_max = 0; + + switch (mode) { + case PS_MINIMIZE_BETA_LIMIT: + switch (nParam) { + case PM_PAR_SKY: beta_lim = 1000; break; + case PM_PAR_I0: beta_lim = 3e6; break; + case PM_PAR_XPOS: beta_lim = 5; break; + case PM_PAR_YPOS: beta_lim = 5; break; + case PM_PAR_SXX: beta_lim = 0.5; break; + case PM_PAR_SYY: beta_lim = 0.5; break; + case PM_PAR_SXY: beta_lim = 0.5; break; + default: + psAbort("invalid parameter %d for beta test", nParam); + } + if (fabs(beta[nParam]) > fabs(beta_lim)) { + beta[nParam] = (beta[nParam] > 0) ? fabs(beta_lim) : -fabs(beta_lim); + return false; + } + return true; + case PS_MINIMIZE_PARAM_MIN: + switch (nParam) { + case PM_PAR_SKY: params_min = -1000; break; + case PM_PAR_I0: params_min = 0; break; + case PM_PAR_XPOS: params_min = -100; break; + case PM_PAR_YPOS: params_min = -100; break; + case PM_PAR_SXX: params_min = 0.5; break; + case PM_PAR_SYY: params_min = 0.5; break; + case PM_PAR_SXY: params_min = -5.0; break; + default: + psAbort("invalid parameter %d for param min test", nParam); + } + if (params[nParam] < params_min) { + params[nParam] = params_min; + return false; + } + return true; + case PS_MINIMIZE_PARAM_MAX: + switch (nParam) { + case PM_PAR_SKY: params_max = 1e5; break; + case PM_PAR_I0: params_max = 1e8; break; + case PM_PAR_XPOS: params_max = 1e4; break; + case PM_PAR_YPOS: params_max = 1e4; break; + case PM_PAR_SXX: params_max = 100; break; + case PM_PAR_SYY: params_max = 100; break; + case PM_PAR_SXY: params_max = +5.0; break; + default: + psAbort("invalid parameter %d for param max test", nParam); + } + if (params[nParam] > params_max) { + params[nParam] = params_max; + return false; + } + return true; + default: + psAbort("invalid choice for limits"); + } + psAbort("should not reach here"); + return false; +} + +// make an initial guess for parameters +bool PM_MODEL_GUESS (pmModel *model, pmSource *source, psImageMaskType maskVal, psImageMaskType markVal) +{ + pmMoments *moments = source->moments; + pmPeak *peak = source->peak; + psF32 *PAR = model->params->data.F32; + + psEllipseMoments emoments; + emoments.x2 = moments->Mxx; + emoments.y2 = moments->Myy; + emoments.xy = moments->Mxy; + + // force the axis ratio to be < 20.0 + psEllipseAxes axes = psEllipseMomentsToAxes (emoments, 20.0); + psEllipseShape shape = psEllipseAxesToShape (axes); + + PAR[PM_PAR_SKY] = moments->Sky; + PAR[PM_PAR_I0] = peak->rawFlux; + PAR[PM_PAR_XPOS] = peak->xf; + PAR[PM_PAR_YPOS] = peak->yf; + PAR[PM_PAR_SXX] = PS_MAX(0.5, M_SQRT2*shape.sx); + PAR[PM_PAR_SYY] = PS_MAX(0.5, M_SQRT2*shape.sy); + PAR[PM_PAR_SXY] = shape.sxy; + PAR[PM_PAR_SXY] = 0.0; // XXX we can get this right if we do the integral + + return(true); +} + +psF64 PM_MODEL_FLUX(const psVector *params) +{ + float norm, z; + psEllipseShape shape; + + psF32 *PAR = params->data.F32; + + shape.sx = PAR[PM_PAR_SXX] / sqrt(2.0); + shape.sy = PAR[PM_PAR_SYY] / sqrt(2.0); + shape.sxy = PAR[PM_PAR_SXY]; + + // Area is equivalent to 2 pi sigma^2 + psEllipseAxes axes = psEllipseShapeToAxes (shape, 20.0); + psF64 Area = 2.0 * M_PI * axes.major * axes.minor; + + // the area needs to be multiplied by the integral of f(z) + norm = 0.0; + +# define DZ 0.25 + + float f0 = 1.0; + float f1, f2; + for (z = DZ; z < 50; z += DZ) { + f1 = 1.0 / (1 + z + z*z/2.0 + z*z*z/6.0); + z += DZ; + f2 = 1.0 / (1 + z + z*z/2.0 + z*z*z/6.0); + norm += f0 + 4*f1 + f2; + f0 = f2; + } + norm *= DZ / 3.0; + + psF64 Flux = PAR[PM_PAR_I0] * Area * norm; + + return(Flux); +} + +// define this function so it never returns Inf or NaN +// return the radius which yields the requested flux +psF64 PM_MODEL_RADIUS (const psVector *params, psF64 flux) +{ + psEllipseShape shape; + + if (flux <= 0) + return (1.0); + if (params->data.F32[PM_PAR_I0] <= 0) + return (1.0); + if (flux >= params->data.F32[PM_PAR_I0]) + return (1.0); + + psF32 *PAR = params->data.F32; + + shape.sx = PAR[PM_PAR_SXX] / sqrt(2.0); + shape.sy = PAR[PM_PAR_SYY] / sqrt(2.0); + shape.sxy = PAR[PM_PAR_SXY]; + + // this estimates the radius assuming f(z) is roughly exp(-z) + psEllipseAxes axes = psEllipseShapeToAxes (shape, 20.0); + psF64 radius = axes.major * sqrt (2.0 * log(params->data.F32[PM_PAR_I0] / flux)); + + if (isnan(radius)) psAbort("error in code: never return invalid radius"); + if (radius < 0) psAbort("error in code: never return invalid radius"); + + return (radius); +} + +bool PM_MODEL_FROM_PSF (pmModel *modelPSF, pmModel *modelFLT, const pmPSF *psf) +{ + psF32 *out = modelPSF->params->data.F32; + psF32 *in = modelFLT->params->data.F32; + + // we require these two parameters to exist + assert (psf->params->n > PM_PAR_YPOS); + assert (psf->params->n > PM_PAR_XPOS); + + for (int i = 0; i < psf->params->n; i++) { + if (psf->params->data[i] == NULL) { + out[i] = in[i]; + } else { + pmTrend2D *trend = psf->params->data[i]; + out[i] = pmTrend2DEval(trend, in[PM_PAR_XPOS], in[PM_PAR_YPOS]); + } + } + + // the 2D model for SXY actually fits SXY / (SXX^-2 + SYY^-2); correct here + out[PM_PAR_SXY] = pmPSF_SXYtoModel (out); + + return(true); +} + +// construct the PSF model from the FLT model and the psf +// XXX is this sufficiently general do be a global function, not a pmModelClass function? +bool PM_MODEL_PARAMS_FROM_PSF (pmModel *model, const pmPSF *psf, float Xo, float Yo, float Io) +{ + psF32 *PAR = model->params->data.F32; + + // we require these two parameters to exist + assert (psf->params->n > PM_PAR_YPOS); + assert (psf->params->n > PM_PAR_XPOS); + + PAR[PM_PAR_SKY] = 0.0; + PAR[PM_PAR_I0] = Io; + PAR[PM_PAR_XPOS] = Xo; + PAR[PM_PAR_YPOS] = Yo; + + // supply the model-fitted parameters, or copy from the input + for (int i = 0; i < psf->params->n; i++) { + if (i == PM_PAR_SKY) continue; + pmTrend2D *trend = psf->params->data[i]; + PAR[i] = pmTrend2DEval(trend, Xo, Yo); + } + + // the 2D PSF model fits polarization terms (E0,E1,E2) + // convert to shape terms (SXX,SYY,SXY) + // XXX user-defined value for limit? + bool useReff = pmModelUseReff (model->type); + if (!pmPSF_FitToModel (PAR, 0.1, useReff)) { + psError(PM_ERR_PSF, false, "Failed to fit object at (r,c) = (%.1f,%.1f)", Xo, Yo); + return false; + } + + // apply the model limits here: this truncates excessive extrapolation + // XXX do we need to do this still? should we put in asserts to test? + for (int i = 0; i < psf->params->n; i++) { + // apply the limits to all components or just the psf-model parameters? + if (psf->params->data[i] == NULL) + continue; + + bool status = true; + status &= PM_MODEL_LIMITS (PS_MINIMIZE_PARAM_MIN, i, PAR, NULL); + status &= PM_MODEL_LIMITS (PS_MINIMIZE_PARAM_MAX, i, PAR, NULL); + if (!status) { + psTrace ("psModules.objects", 5, "Hitting parameter limits at (r,c) = (%.1f, %.1f)", Xo, Yo); + model->flags |= PM_MODEL_STATUS_LIMITS; + } + } + return(true); +} + +// XXX double-check these definitions below +// this test is invalid if the parameters are derived +// from the PSF model +bool PM_MODEL_FIT_STATUS (pmModel *model) +{ + psF32 dP; + bool status; + + psF32 *PAR = model->params->data.F32; + psF32 *dPAR = model->dparams->data.F32; + + dP = 0; + dP += PS_SQR(dPAR[PM_PAR_SXX] / PAR[PM_PAR_SXX]); + dP += PS_SQR(dPAR[PM_PAR_SYY] / PAR[PM_PAR_SYY]); + dP = sqrt (dP); + + status = true; + status &= (dP < 0.5); + status &= (PAR[PM_PAR_I0] > 0); + status &= ((dPAR[PM_PAR_I0]/PAR[PM_PAR_I0]) < 0.5); + + if (status) + return true; + return false; +} + +# undef PM_MODEL_FUNC +# undef PM_MODEL_FLUX +# undef PM_MODEL_GUESS +# undef PM_MODEL_LIMITS +# undef PM_MODEL_RADIUS +# undef PM_MODEL_FROM_PSF +# undef PM_MODEL_PARAMS_FROM_PSF +# undef PM_MODEL_FIT_STATUS Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psmakecff.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psmakecff.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psmakecff.c (revision 41083) @@ -0,0 +1,163 @@ +# ifdef HAVE_CONFIG_H +# include +# endif + +// psmakecff : A program to make read a cmf file and write a cff file +// The real work is done in psModules. + +#include +#include +#include +#include "psphot.h" + +// For simplicilty, this program's (simple) functions are all contained in this file. +static pmConfig* psmakecffArguments(int, char**); +static bool psmakecffParseCamera(pmConfig *); +static bool psmakecffImageLoop(pmConfig*); + +int main (int argc, char **argv) { + + psMemInit(); + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psmakecffArguments (argc, argv); + assert(config); + +// psphotVersionPrint(); + + if (!psmakecffParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (1); + } + + if (!psmakecffImageLoop (config)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit(1); + } + + exit (0); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status + +void usage() { + fprintf(stderr, "usage: psmakecff -sources \n"); + exit (1); +} + +static pmConfig* psmakecffArguments(int argc, char **argv) { + + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + int N; + + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + if ((N = psArgumentGet (argc, argv, "-sources"))) { + pmConfigFileSetsMD(config->arguments, &argc, argv, "SOURCES", "-sources", "-sourceslist"); + } else { + usage(); + } + if (argc < 2) { + usage(); + } + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", PS_DATA_STRING, "", argv[1]); + + return config; +} + +static bool psmakecffParseCamera(pmConfig *config) { + bool status = false; + + pmFPAfile *sources = pmFPAfileDefineFromArgs(&status, config, "PSPHOT.INPUT.CMF", "SOURCES"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to build FPA from sources file"); + return false; + } + + pmFPAfile *output = pmFPAfileDefineOutputFromFile (config, sources, "PSPHOT.OUTPUT.CFF"); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "failed to build output FPA"); + return false; + } + output->save = true; + + return true; +} + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +bool psmakecffImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT.CMF"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAfile *output = psMetadataLookupPtr (&status, config->files, "PSPHOT.OUTPUT.CFF"); + if (!output) { + psError(PSPHOT_ERR_PROG, false, "Can't find output data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + pmHDU *lastHDU = NULL; // Last HDU updated + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { + psLogMsg ("psmakecff", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psmakecff."); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psmakecff", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psmakecff", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + // XXX: probably should do this + // psphotVersionHeaderFull(hdu->header); + lastHDU = hdu; + } + } + + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psmakecff."); + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for FPA in psphot."); + + // fail if we encountered an unhandled error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphot.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphot.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphot.c (revision 41083) @@ -0,0 +1,37 @@ +# include "psphotStandAlone.h" +# define FORCED_PHOTOMETRY 0 + +int main (int argc, char **argv) { + + psMemInit(); + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotArguments (argc, argv); + assert(config); + + psphotVersionPrint(); + + // load input data (config and images (signal, noise, mask) + if (!psphotParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotImageLoop (config, PSPHOT_SINGLE)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", PS_LOG_WARN, "complete psphot run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphot.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphot.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphot.h (revision 41083) @@ -0,0 +1,594 @@ +/* This file defines the library functions available to external programs. It must be included + * by programs which are compiled against psphot functions. + */ + +#ifndef PSPHOT_H +#define PSPHOT_H + +#include +#include "psphotErrorCodes.h" + +#define PSPHOT_RECIPE "PSPHOT" // Name of the recipe to use + +#define PSPHOT_RECIPE_PSF_FAKE_ALLOW "PSF.FAKE.ALLOW" // Name for recipe component permitting fake PSFs + +# define READOUT_OR_INTERNAL(VIEW,FILE)((FILE)->mode == PM_FPA_MODE_INTERNAL) ? (FILE)->readout : pmFPAviewThisReadout((VIEW), (FILE)->fpa) + +typedef enum { + PSPHOT_SINGLE, + PSPHOT_FORCED, + PSPHOT_FULL_FORCE, + PSPHOT_MAKE_PSF, + PSPHOT_MODEL_TEST, + PSPHOT_MINIMAL, +} psphotImageLoopMode; + +// top-level psphot functions +const char *psphotCVSName(void); +psString psphotVersion(void); +psString psphotSource(void); +psString psphotVersionLong(void); +bool psphotVersionHeader(psMetadata *header); +bool psphotVersionHeaderFull(psMetadata *header); +void psphotVersionPrint(void); + +bool psphotImageLoop (pmConfig *config, psphotImageLoopMode mode); + +bool psphotModelTest (pmConfig *config, const pmFPAview *view, psMetadata *recipe); +bool psphotInit (void); +bool psphotReadout (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotReadoutFindPSF(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources); +bool psphotReadoutKnownSources(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources); +bool psphotReadoutForcedKnownSources(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources); +bool psphotReadoutMinimal(pmConfig *config, const pmFPAview *view, const char *filerule); + +bool psphotReadoutCleanup (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotReadoutCleanupReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotReadoutCleanupMinimal (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotReadoutCleanupReadoutMinimal (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotDefineFiles (pmConfig *config, pmFPAfile *input); +void psphotFilesActivate(pmConfig *config, bool state); + +bool psphotSetMaskBits (pmConfig *config); +bool psphotSetMaskRecipe (pmConfig *config, psImageMaskType maskValue, psImageMaskType markValue); + +// XXX test functions +psArray *psphotFakeSources (void); +bool psphotMaskCosmicRayFootprintCheck (psArray *sources); + +// psphotReadout functions +bool psphotAddPhotcode (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotAddPhotcodeReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotSetMaskAndVariance (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSetMaskAndVarianceReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotUpdateVariance (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotUpdateVarianceReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotModelBackground (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotModelBackgroundReadoutFileIndex (pmConfig *config, const pmFPAview *view, const char *filerule, int index); +bool psphotModelBackground_Threaded (psThreadJob *job); +bool psphotLoadBackgroundModel (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotLoadBackgroundModelReadoutFileIndex (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotMaskBackground (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotMaskBackgroundReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotSubtractBackground (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSubtractBackgroundReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotFindDetections (pmConfig *config, const pmFPAview *view, const char *filerule, bool firstPass); +bool psphotFindDetectionsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool firstPass); + +bool psphotSourceStats (pmConfig *config, const pmFPAview *view, const char *filerule, bool setWindow); +bool psphotSourceStatsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool setWindow); + +bool psphotFullForceSourceStats (pmConfig *config, const pmFPAview *view, const char *filerule, bool setWindow); +bool psphotFullForceSourceStatsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool setWindow); + +bool psphotDeblendSatstars (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotDeblendSatstarsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotBasicDeblend (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotBasicDeblendReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotRoughClass (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotRoughClassReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotRoughClassRegion (int nRegion, psRegion *region, psArray *sources, psMetadata *analysis, psMetadata *recipe, const bool havePSF); + +bool psphotImageQuality (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotImageQualityReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotChoosePSF (pmConfig *config, const pmFPAview *view, const char *filerule, bool newSources); +bool psphotChoosePSFReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool newSources); + +bool psphotGuessModels (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotGuessModelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotMergeSources (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotMergeSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotFitSourcesLinear (pmConfig *config, const pmFPAview *view, const char *filerule, bool final, bool skipNegativeFluxSources); +bool psphotFitSourcesLinearReadout (psMetadata *recipe, pmReadout *readout, psArray *sources, pmPSF *psf, bool final, pmSourceFitVarMode fitVarMode, bool skipNegativeFluxSources); + +bool psphotSourceSize (pmConfig *config, const pmFPAview *view, const char *filerule, bool getPSFsize); +bool psphotSourceSizeReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool getPSFsize); + +bool psphotBlendFit (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotBlendFitReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotBlendFit_Threaded (psThreadJob *job); + +bool psphotReplaceAllSources (pmConfig *config, const pmFPAview *view, const char *filerule, bool ignoreState); +bool psphotReplaceAllSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool ignoreState); + +bool psphotAddNoise (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSubNoise (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotAddOrSubNoise (pmConfig *config, const pmFPAview *view, const char *filerule, bool add); +bool psphotAddOrSubNoiseReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool add); +bool psphotAddOrSubNoise_Threaded (psThreadJob *job); + +bool psphotChooseAnalysisOptions (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotChooseAnalysisOptionsReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotChooseAnalysisOptionsByObject(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects); + +bool psphotExtendedSourceAnalysis (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotExtendedSourceAnalysisReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotExtendedSourceAnalysis_Threaded (psThreadJob *job); + +bool psphotExtendedSourceFits (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotExtendedSourceFitsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotExtendedSourceFits_Threaded (psThreadJob *job); + +bool psphotGalaxyParams (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotGalaxyParamsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotGalaxyParams_Threaded (psThreadJob *job); + +bool psphotApResid (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotApResidReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotMagnitudes (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf, int index); +bool psphotMagnitudes_Threaded (psThreadJob *job); + +bool psphotLensing (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotLensingReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); +bool psphotLensingPSFtrendsReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotEfficiency (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotEfficiencyReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotPSFWeights(pmConfig *config, pmReadout *readout, const pmFPAview *view, psArray *sources); +bool psphotPSFWeights_Threaded (psThreadJob *job); + +bool psphotSkyReplace (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSkyReplaceReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotSourceFreePixels (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSourceFreePixelsReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +// in psphotSourceStats.c: +bool psphotSourceStats_Threaded (psThreadJob *job); +bool psphotSourceStatsUpdate (psArray *sources, pmConfig *config, pmReadout *readout); +bool psphotSetMomentsWindow (psMetadata *recipe, psMetadata *analysis, psArray *sources, psImageMaskType maskVal); + +// in psphotChoosePSF.c: +bool psphotPSFstats (pmReadout *readout, pmPSF *psf); +bool psphotPSFstatsSources (pmReadout *readout, psArray *sources, pmPSF *psf); +bool psphotMomentsStats (pmReadout *readout, psArray *sources); + +bool psphotKronMasked (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotKronMaskedReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf); + +// in psphotGuessModel.c +bool psphotGuessModel_Threaded (psThreadJob *job); + +// in psphotMergeSources.c: +bool psphotLoadExtSources (pmConfig *config, const pmFPAview *view, const char *filerule); +psArray *psphotLoadPSFSources (pmConfig *config, const pmFPAview *view); +bool psphotRepairLoadedSources (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotCheckExtSources (pmConfig *config, const pmFPAview *view, const char *filerule); + +// generate the detection structure for the supplied array of sources +bool psphotDetectionsFromSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *sources); + +// generate the detection structure for the supplied array of sources +bool psphotSetSourceParams (pmConfig *config, psArray *sources, pmPSF *psf); + +// in psphotModelBackground.c: +// Create a background model for a readout, without saving the result as a pmFPAfile on config->files. Otherwise identical to psphotModelBackgroundFileIndex. +psImage *psphotModelBackgroundReadoutNoFile(pmReadout *readout, const pmConfig *config); +psImageBinning *psphotBackgroundBinning(const psImage *image, const pmConfig *config); + +// in psphotReplaceUnfit.c: +bool psphotRemoveAllSourcesByArray (const psArray *sources, const psMetadata *recipe); +bool psphotRemoveAllSources (pmConfig *config, const pmFPAview *view, const char *filerule, bool ignoreState); +bool psphotRemoveAllSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filename, int index, psMetadata *recipe, bool ignoreState); +bool psphotReplaceUnfitSources (psArray *sources, psImageMaskType maskVal); + +// thread-related: +bool psphotSetThreads (void); +bool psphotChooseCellSizes (int *Cx, int *Cy, pmReadout *readout, int nThreads); +bool psphotCoordToCell (int *group, int *cell, float x, float y, int Cx, int Cy); +psArray *psphotAssignSources (int Cx, int Cy, psArray *sources); + +// used by psphotFindDetections +pmReadout *psphotSignificanceImage (pmReadout *readout, psMetadata *recipe, psImageMaskType maskVal); +psArray *psphotFindPeaks (pmReadout *significance, pmReadout *readout, psMetadata *recipe, const float threshold, const int nMax, int *totalPeaks); +bool psphotFindFootprints (pmDetections *detections, pmReadout *significance, pmReadout *readout, psMetadata *recipe, const float threshold, const int pass, psImageMaskType maskVal); +psErrorCode psphotCullPeaks(const pmReadout *readout, const pmReadout *signifRO, const psMetadata *recipe, psArray *footprints); + +// in psphotApResid.c: +pmTrend2D *psphotApResidTrend (float *apResidSysErr, pmReadout *readout, int Nx, int Ny, psVector *xPos, psVector *yPos, psVector *apResid, psVector *dMag); +bool psphotApResidMags_Threaded (psThreadJob *job); + +// basic support functions +void psphotModelClassInit (void); +bool psphotGrowthCurve (pmReadout *readout, pmPSF *psf, bool ignore, psImageMaskType maskVal); + +// functions to set the correct source pixels +bool psphotInitRadiusPSF (psMetadata *recipe, pmReadout *readout); +bool psphotInitRadiusEXT (psMetadata *recipe, pmReadout *readout); + +bool psphotCheckRadiusPSF (pmReadout *readout, pmSource *source, pmModel *model, psImageMaskType markVal); +bool psphotCheckRadiusPSFBlend (pmReadout *readout, pmSource *source, pmModel *model, psImageMaskType markVal, float dR); +bool psphotSetRadiusFootprint (float *radius, pmReadout *readout, pmSource *source, psImageMaskType markVal, float factor); +bool psphotSetRadiusModel (pmModel *model, pmReadout *readout, pmSource *source, psImageMaskType markVal, bool deep); + +bool psphotDumpMoments (psMetadata *recipe, psArray *sources); +psMetadata *psphotDefineHeader (psMetadata *recipe); +int psphotSaveImage (psMetadata *header, psImage *image, char *filename); +bool psphotDumpConfig (pmConfig *config); +pmReadout *psphotSelectBackground (pmConfig *config, const pmFPAview *view, int index); +pmReadout *psphotSelectBackgroundStdev (pmConfig *config, const pmFPAview *view, int index); + +// PSF / DBL / EXT evaluation functions +bool psphotEvalPSF (pmSource *source, pmModel *model); +bool psphotEvalDBL (pmSource *source, pmModel *model); +bool psphotEvalEXT (pmSource *source, pmModel *model); + +// functions to support the source fitting process +bool psphotInitLimitsPSF (psMetadata *recipe, pmReadout *readout); +bool psphotInitLimitsEXT (psMetadata *recipe, pmReadout *readout); +bool psphotFitBlend (pmReadout *readout, pmSource *source, pmPSF *psf, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal); +bool psphotFitBlob (pmReadout *readout, pmSource *source, psArray *sources, pmPSF *psf, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal); +bool psphotFitPSF (pmReadout *readout, pmSource *source, pmPSF *psf, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal); +pmModel *psphotFitEXT (pmModel *guessModel, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, pmModelType modelType, psImageMaskType maskVal, psImageMaskType markVal); +psArray *psphotFitDBL (pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal); + +// functions to support simultaneous multi-source fitting +bool psphotFitSet (pmSource *oneSrc, pmModel *oneModel, char *fitset, pmSourceFitMode mode, psImageMaskType maskVal); + +// plotting functions (available if libkapa is installed) +bool psphotPlotMoments (pmConfig *config, pmFPAview *view, psArray *sources); +bool psphotPlotPSFModel (pmConfig *config, pmFPAview *view, psArray *sources); +bool psphotFitInit (int nThreads); +bool psphotFitSummary (void); +bool psphotFitSummaryExtended (void); +bool psphotFitInitExtended (void); + +bool psphotLoadPSF (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotLoadPSFReadout (pmConfig *config, const pmFPAview *view, const char *outFilename, const char *inFilename, int index); + +bool psphotSetHeaderNstars (psMetadata *recipe, psArray *sources); +bool psphotRadialPlot (int *kapa, const char *filename, pmSource *source); +bool psphotSourcePlots (pmReadout *readout, psArray *sources, psMetadata *recipe); +bool psphotMosaicSubimage (psImage *outImage, pmSource *source, int Xo, int Yo, int DX, int DY, bool normalize); + +bool psphotMakeResiduals (psArray *sources, psMetadata *recipe, pmPSF *psf, psImageMaskType maskVal); + +pmModel *psphotPSFConvModel (pmReadout *readout, pmSource *source, pmModelType modelType, psImageMaskType maskVal, psImageMaskType markVal, int psfSize); + +psKernel *psphotKernelFromPSF (pmSource *source, int nPix); + +// functions related to extended source analysis +bool psphotRadialProfile (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal); +bool psphotRadialProfilesByAngles (pmSource *source, int Nsec, float Rmax); +float psphotRadiusFromProfile (pmSource *source, psVector *radius, psVector *flux, float fluxMin, float fluxMax); +bool psphotRadiiFromProfiles (pmSource *source, float fluxMin, float fluxMax); +bool psphotEllipticalProfile (pmSource *source, bool RAW_RADIUS); +bool psphotEllipticalContour (pmSource *source); +bool psphotLimitRadialApertures(psMetadata *recipe, long nSources); + +// psphotVisual functions +bool psphotVisualShowImage (pmReadout *readout); +bool psphotVisualShowBackground (pmConfig *config, const pmFPAview *view, pmReadout *readout); +bool psphotVisualShowSignificance (psImage *image, float min, float max); +bool psphotVisualShowLogSignificance (psImage *image, float min, float max); +bool psphotVisualShowPeaks (pmDetections *detections); +bool psphotVisualShowSources (psArray *sources); +bool psphotVisualShowFootprints (pmDetections *detections); +bool psphotVisualShowMoments (psArray *sources); +bool psphotVisualPlotMoments (psMetadata *recipe, psMetadata *analysis, psArray *sources); +bool psphotVisualShowRoughClass (psArray *sources); +bool psphotVisualShowPSFStars (psMetadata *recipe, pmPSF *psf, psArray *sources); +bool psphotVisualShowSatStars (psMetadata *recipe, pmPSF *psf, psArray *sources); +bool psphotVisualShowPSFModel (pmReadout *readout, pmPSF *psf); +bool psphotVisualPlotRadialProfile (int myKapa, pmSource *source, psImageMaskType maskVal, pmSourceMode showmode); +bool psphotVisualPlotRadialProfiles (psMetadata *recipe, psArray *sources, pmSourceMode showmode); +bool psphotVisualShowFlags (psArray *sources); +bool psphotVisualShowSourceSize (pmReadout *readout, psArray *sources); +bool psphotVisualPlotSourceSizeAlt (psMetadata *recipe, psMetadata *analysis, psArray *sources); +bool psphotVisualShowResidualImage (pmReadout *readout, bool reshow); +bool psphotVisualShowObjectRegions (pmReadout *readout, psMetadata *recipe, psArray *sources); +bool psphotVisualPlotApResid (psArray *sources, float mean, float error, bool useApMag); +bool psphotVisualPlotChisq (psArray *sources); +bool psphotVisualPlotSourceSize (psMetadata *recipe, psMetadata *analysis, psArray *sources); +bool psphotVisualShowPetrosians (psArray *sources); +bool psphotVisualEraseOverlays (int channel, char *overlay); +bool psphotVisualClose(void); + +int psphotKapaChannel (int channel); +void plotline (int myKapa, Graphdata *graphdata, float x0, float y0, float x1, float y1); + + +bool psphotPetrosian (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal); +bool psphotPetrosianRadialBins (pmSource *source, float radiusMax, float skynoise); +bool psphotPetrosianStats (pmSource *source); + +// currently disabled: +// bool psphotIsophotal (pmSource *source, psMetadata *recipe, psImageMaskType maskVal); +// bool psphotAnnuli (pmSource *source, psMetadata *recipe, psImageMaskType maskVal); +// bool psphotKron (pmSource *source, psMetadata *recipe, psImageMaskType maskVal); + +// XXX visualization functions related to radial profiles (disabled) +bool psphotPetrosianVisualProfileByAngle (psVector *radius, psVector *flux); +bool psphotPetrosianVisualProfileRadii (psVector *radius, psVector *flux, psVector *radiusBin, psVector *fluxBin, float peakFlux, float RadiusRef); +bool psphotPetrosianVisualEllipticalContour (pmSourceRadialFlux *radFlux, pmSourceExtendedPars *extpars); +bool psphotPetrosianVisualStats (psVector *radBin, psVector *fluxBin, + psVector *refRadius, psVector *meanSB, + psVector *petRatio, psVector *petRatioErr, psVector *fluxSum, + float petRadius, float ratioForRadius, + float petFlux, float radiusForFlux); + +bool psphotRadialBins (psMetadata *recipe, pmSource *source, float radiusMax, float skynoise); + +int psphotKapaOpen (void); +bool psphotKapaClose (void); +bool psphotImageBackgroundCellHistogram (psVector *values, float mean, float sigma, int ix, int iy); +bool psphotDiagnosticPlots (const pmConfig *config, const char *name, ...); + +bool psphotMakeFluxScale (psImage *image, psMetadata *recipe, pmPSF *psf); +bool psphotMakeGrowthCurve (pmReadout *readout, psMetadata *recipe, pmPSF *psf, psArray *sources); +bool psphotDumpPSFStars (pmReadout *readout, pmPSFtry *try, float radius, psImageMaskType maskVal, psImageMaskType markVal); + +bool psphotLoadSRCTEXT (pmFPA *fpa, pmConfig *config); + +bool psphotCheckStarDistribution (psArray *sources, psArray *stars, pmPSFOptions *options); +int psphotSupplementStars (psArray *stars, psArray *sources, psImageBinning *binning, int ix, int iy); + +pmConfig *psphotForcedArguments(int argc, char **argv); +bool psphotForcedReadout(pmConfig *config, const pmFPAview *view, const char *filerule); + +pmConfig *psphotFullForceArguments(int argc, char **argv); +bool psphotFullForceReadout(pmConfig *config, const pmFPAview *view, const char *filerule); + +pmConfig *psphotMinimalArguments(int argc, char **argv); +bool psphotReadoutMinimal(pmConfig *config, const pmFPAview *view, const char *filerule); + +pmConfig *psphotMakePSFArguments(int argc, char **argv); +bool psphotMakePSFReadout(pmConfig *config, const pmFPAview *view, const char *filerule); + +pmConfig *psphotModelTestArguments(int argc, char **argv); +bool psphotModelTestReadout(pmConfig *config, const pmFPAview *view, const char *filerule); + +bool psphotFullForceSummaryReadout (pmConfig * config, const pmFPAview *view); + +int psphotFileruleCount(const pmConfig *config, const char *filerule); +bool psphotFileruleCountSet(const pmConfig *config, const char *filerule, int num); + +bool psphotAddKnownSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources); + + +/**** psphotStack prototypes ****/ + +pmConfig *psphotStackArguments(int argc, char **argv); +bool psphotStackParseCamera (pmConfig *config); +bool psphotStackImageLoop (pmConfig *config); +bool psphotStackReadout (pmConfig *config, const pmFPAview *view); +bool psphotStackUpdateReadout (pmConfig *config, const pmFPAview *view); +bool psphotStackChisqImage (pmConfig *config, const pmFPAview *view, const char *ruleDet, const char *ruleCnv); +bool psphotStackChisqImageAddReadout(const pmConfig *config, // Configuration + const pmFPAview *view, + const char *filename, + pmReadout **chiReadout, + int index); +bool psphotStackAllocateOutput( const pmConfig *config, pmFPAview *view, psMetadata *recipe); + +bool psphotStackRemoveChisqFromInputs (pmConfig *config, const char *filerule); +bool psphotStackMatchPSFsetup (pmConfig *config, const pmFPAview *view, const char *filerule, const char *fPSF); +bool psphotStackMatchPSFsetupReadout (pmConfig *config, const pmFPAview *view, const char *filerule, const char *fPSF, int index); +bool psphotStackLoadWCS(pmConfig *config, const pmFPAview *view, const char *filerule); +bool pmFPAfileRemoveSingle(psMetadata *files, const char *name, int num); + +psArray *psphotMatchSources (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotMatchSourcesReadout (psArray *objects, pmConfig *config, const pmFPAview *view, const char *filerule, int index); +bool psphotMatchSourcesToObjects (psArray *objects, psArray *sources, float RADIUS); +bool psphotFilterMatchedSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects); +psArray *psphotLinkSources (pmConfig *config, const pmFPAview *view, const char *filerule); + +bool psphotFitSourcesLinearStack (pmConfig *config, psArray *objects, bool final); + +typedef enum { + PSPHOT_CNV_SRC_NONE, + PSPHOT_CNV_SRC_AUTO, + PSPHOT_CNV_SRC_CNV, + PSPHOT_CNV_SRC_RAW, +} psphotStackConvolveSource; + +/// Options for stacking process +typedef struct { + // Setup + + int numCols; // size of image (X) + int numRows; // size of image (Y) + + int num; // Number of inputs + bool convolve; // Convolve images? + psphotStackConvolveSource convolveSource; + + // Prepare + pmPSF *psf; // Target PSF + psVector *inputSeeing; // Input seeing FWHMs + psVector *inputMask; // Mask for inputs + + psVector *targetSeeing; // Target seeing FWHMs + psArray *sourceLists; // Individual lists of sources for matching + psVector *norm; // Normalisation for each image + psArray *psfs; + + // Convolve + psArray *kernels; // PSF-matching kernels --- required in the stacking + psArray *regions; // PSF-matching regions --- required in the stacking + psVector *matchChi2; // chi^2 for stamps from matching +} psphotStackOptions; + +typedef struct { + float Q; + float NSigma; + float clampSN; + int extModelType; +} psphotGalaxyShapeOptions; + +/*** psphotStackMatchPSF prototypes ***/ +bool psphotStackMatchPSFs (pmConfig *config, const pmFPAview *view); +bool psphotStackMatchPSFsReadout (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index); +bool psphotStackMatchPSFsPrepare (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index); + +bool psphotStackMatchPSFsNext (pmConfig *config, const pmFPAview *view, const char *filerule, int lastSize); +bool psphotStackMatchPSFsNextReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, int lastSize); +int psphotStackMatchPSFsEntries (pmConfig *config, const pmFPAview *view, const char *filerule); + +// psphotStackMatchPSFsUtils +// psVector *SetOptWidths (bool *optimum, psMetadata *recipe); +// pmReadout *makeFakeReadout(pmConfig *config, pmReadout *raw, psArray *sources, pmPSF *psf, psImageMaskType maskVal, int fullSize); +// bool rescaleData(pmReadout *readout, pmConfig *config, psphotStackOptions *options, int index); +// bool renormKernel(pmReadout *readout, psphotStackOptions *options, int index); +bool saveMatchData (pmReadout *readout, psphotStackOptions *options, int index); +bool matchKernel(pmConfig *config, pmReadout *cnv, pmReadout *raw, psphotStackOptions *options, int index); +// bool dumpImageDiff(pmReadout *readoutConv, pmReadout *readoutFake, pmReadout *readoutRef, int index, char *rootname); +// bool dumpImage(pmReadout *readoutOut, pmReadout *readoutRef, int index, char *rootname); +// bool loadKernel (pmConfig *config, pmReadout *readoutCnv, psphotStackOptions *options, int index); + +bool psphotStackSetInputsToSkip(pmConfig *config, const pmFPAview *view, const char *filerule, bool set) ; + +bool psphotStackRenormaliseVariance(const pmConfig *config, pmReadout *readout); + +bool psphotStackPSF(const pmConfig *config, psphotStackOptions *options); + +psphotStackOptions *psphotStackOptionsAlloc (int num); +psphotStackConvolveSource psphotStackConvolveSourceFromString (const char *string); +pmFPAfile *psphotStackGetConvolveSource (pmConfig *config, psphotStackOptions *options, int index); + +bool psphotCopySources (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); +bool psphotCopySourcesReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); +bool psphotCopyEfficiency (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); +bool psphotCopyEfficiencyReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); + +bool psphotRadialApertures (pmConfig *config, const pmFPAview *view, const char *filerule, int entry); +bool psphotRadialAperturesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, int entry); +bool psphotRadialApertures_Threaded (psThreadJob *job); +bool psphotRadialApertureSource (pmSource *source, pmReadout *readout, int entry, psVector *pixRadius2, psVector *pixFlux, psVector *pixVer); +// bool psphotRadialApertureSource (pmSource *source, int entry); + +bool psphotExtendedSourceAnalysisByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule); +bool psphotRadialAperturesByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule, int nMatchedPSF); + +bool psphotStackObjectsUnifyPosition (psArray *objects); + +bool psphotFitSersicIndex (pmModel *model, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal); + +bool psphotFitSersicIndexPCM (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize); +pmModel *psphotFitPCM (pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, pmModelType modelType, psImageMaskType maskVal, psImageMaskType markVal, int psfSize); + +bool psphotCleanInputs (pmConfig *config, const pmFPAview *view, const char *filerule); + +bool psphotResetModels (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotResetModelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotRedefinePixels (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotRedefinePixelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe); + +bool psphotSourceChildren (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); +bool psphotSourceChildrenReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); +psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *fileruleOut, const char *fileruleSrc, psArray *objectsSrc, bool sourcesSubtracted); + +bool psphotSourceParents (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); +bool psphotSourceParentsReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); + +bool psphotCopyPeaks (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc); +bool psphotCopyPeaksReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index); + +bool psphotSersicModelClassGuessPCM (pmPCMdata *pcm, pmSource *source); +void psphotSersicModelClassInit (); +void psphotSersicModelClassCleanup (); + +bool psphotSetRadiusMoments (float *fitRadius, float *windowRadius, pmReadout *readout, pmSource *source, psImageMaskType markVal); +bool psphotSetRadiusMomentsExact (float *fitRadius, float *windowRadius, pmReadout *readout, pmSource *source, psImageMaskType markVal); + +bool psphotFootprintSaddles(pmReadout *readout, psArray *footprints); +bool psphotMaskFootprint (pmReadout *readout, pmSource *source, psImageMaskType markVal); + +bool psphotKronIterate (pmConfig *config, const pmFPAview *view, const char *filerule, int pass); +bool psphotKronIterateReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char * filerule, pmReadout *readout, psArray *sources, pmPSF *psf, int index, int pass); +bool psphotKronIterate_Threaded (psThreadJob *job); + +bool psphotKronFlux (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotKronFluxReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char *filerule, pmReadout *readout, psArray *sources); +bool psphotKronFlux_Threaded (psThreadJob *job); +bool psphotKronFluxSource (pmSource *source, psImageMaskType maskVal); + +bool psphotPetroFlux (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotPetroFluxReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char * filerule, pmReadout *readout, psArray *sources); +bool psphotPetroFlux_Threaded (psThreadJob *job); +bool psphotPetroFluxSource (pmSource *source, psImageMaskType maskVal); + +bool psphotGalaxyShape (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotGalaxyShapeReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char * filerule, pmReadout *readout, psArray *sources, pmPSF *psf); +bool psphotGalaxyShape_Threaded (psThreadJob *job); +bool psphotGalaxyShapeGrid (pmSource *source, pmSourceFitOptions *fitOptions, psphotGalaxyShapeOptions *opt, psImageMaskType maskVal, int psfSize); +bool psphotGalaxyShapeSource (pmPCMdata *pcm, pmSource *source, pmSourceGalaxyFits *galaxyFits, psImageMaskType maskVal, int psfSize, bool saveResults); +psphotGalaxyShapeOptions *psphotGalaxyShapeOptionsAlloc(); +bool psphotGalaxyShapeOptionsSet(pmSource *source, pmSourceGalaxyFits *galaxyFits, psphotGalaxyShapeOptions *defaultOptions); + +bool psphotRadialProfileWings (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotRadialProfileWingsReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources); +bool psphotRadialProfileWings_Threaded (psThreadJob *job); + +bool psphotStackObjectsSelectForAnalysis (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects); + +bool psphotSetNFrames (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSetNFramesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotGenerateModelVariance (pmConfig *config, const pmFPAview *view, pmFPAfile *file, int index, psMetadata *recipe, pmReadout *readout, psArray *sources); +pmSourceFitVarMode psphotGetFitVarMode (psMetadata *recipe); +bool psphotFreeModelVariance (pmReadout *readout, psArray *sources); + +bool psphotModelBackgroundReadout(psImage *model, // Model image + psImage *modelStdev, // Model stdev image + psMetadata *analysis, // Analysis metadata for outputs + pmReadout *readout, // Readout for which to generate a background model + psImageBinning *binning, // Binning parameters + const pmConfig *config,// Configuration + bool useVarianceImage + ); + +bool psphotSatstarProfileModel (pmSource *source, psImageMaskType maskVal); +bool psphotSatstarProfileCreate (pmSource *source, psVector **logRmodelOut, psVector **logFmodelOut, psVector *logR, psVector *flux, float Rmax); +bool psphotSatstarProfileOp (pmSource *source, psImageMaskType maskVal, float FACTOR, pmModelOpMode mode, bool add); +bool psphotVisualRadialProfileSatstar (pmSource *source, psImageMaskType maskVal); +bool psphotAddOrSubSatstarsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int fileIndex, psMetadata *recipe, bool add); +bool psphotSatstarPhotometry (pmSource *source); + +bool psphotSourceMemory(pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotSourceMemoryReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index); + +bool psphotChipParams(pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotChipParams_Threaded(psThreadJob *job); + +const char * psphotGetFilerule(const char *baseRule); +extern bool psphotINpsphotStack; + + +#endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotAddNoise.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotAddNoise.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotAddNoise.c (revision 41083) @@ -0,0 +1,217 @@ +# include "psphotInternal.h" + +bool psphotAddOrSubNoise_Threaded (psThreadJob *job); +bool psphotMaskSource(pmSource *source, bool add, psImageMaskType maskVal); + +bool psphotAddNoise (pmConfig *config, const pmFPAview *view, const char *filerule) { + return psphotAddOrSubNoise (config, view, filerule, true); +} + +bool psphotSubNoise (pmConfig *config, const pmFPAview *view, const char *filerule) { + return psphotAddOrSubNoise (config, view, filerule, false); +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotAddOrSubNoise (pmConfig *config, const pmFPAview *view, const char *filerule, bool add) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotAddOrSubNoiseReadout (config, view, filerule, i, recipe, add)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on to modify noise for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// the return state indicates if any sources were actually replaced +bool psphotAddOrSubNoiseReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool add) { + + bool status = false; + + psTimerStart ("psphot.noise"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + psAssert (readout->parent, "missing cell?"); + psAssert (readout->parent->concepts, "missing concepts?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + // if no work to do, should just return true + if (!sources) return false; + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + psAssert (markVal, "missing mask value?"); + + // increase variance by factor*(object noise): + // weight = flux/gain + rn^2/g^2 + // we are adding factor*flux/gain + + float FACTOR = psMetadataLookupF32 (&status, recipe, "NOISE.FACTOR"); + PS_ASSERT (status, false); + float SIZE = psMetadataLookupF32 (&status, recipe, "NOISE.SIZE"); + PS_ASSERT (status, false); + + if (SIZE <= 0) { + return false; + } + + float GAIN = psMetadataLookupF32(&status, readout->parent->concepts, "CELL.GAIN"); // Cell gain + PS_ASSERT (status, false); + if (isfinite(GAIN)) { + FACTOR /= GAIN; + } + + psphotVisualShowImage (readout); + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_ADD_NOISE"); + psArrayAdd(job->args, 1, cells->data[j]); // sources + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, FACTOR, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, SIZE, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, add, PS_TYPE_U8); + +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add/sub noise."); + return false; + } +# else + if (!psphotAddOrSubNoise_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add/sub noise."); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psFree(cellGroups); + psError(PS_ERR_UNKNOWN, false, "Unable to add/sub noise."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + // we have no returned data from this operation + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + + if (add) { + psLogMsg ("psphot.noise", PS_LOG_WARN, "add noise for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.noise")); + } else { + psLogMsg ("psphot.noise", PS_LOG_WARN, "sub noise for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.noise")); + } + + psFree (cellGroups); + + psphotVisualShowImage (readout); + + return true; +} + +bool psphotAddOrSubNoise_Threaded (psThreadJob *job) { + + psArray *sources = job->args->data[0]; + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[1], PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[2], PS_TYPE_IMAGE_MASK_DATA); + psF32 FACTOR = PS_SCALAR_VALUE(job->args->data[3], F32); + psF32 SIZE = PS_SCALAR_VALUE(job->args->data[4], F32); + psBool add = PS_SCALAR_VALUE(job->args->data[5], U8); + + // loop over all source + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // add or subtract noise for a saturated star. satstars modeled as a radial profile + // need special handling + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) { + psphotSatstarProfileOp (source, maskVal, FACTOR, PM_MODEL_OP_NOISE, add); + continue; + } + + // skip sources which were not subtracted + // NOTE: this bit is not modified when pmSourceOp applies to noise + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) continue; + + pmSourceNoiseOp (source, PM_MODEL_OP_FULL | PM_MODEL_OP_NOISE, FACTOR, SIZE, add, maskVal, 0, 0); + + psphotMaskSource (source, add, markVal); + } + return true; +} + +bool psphotMaskSource(pmSource *source, bool add, psImageMaskType maskVal) { + + if (!source) return false; + if (!source->peak) return false; // XXX how can we have a peak-less source? + if (source->type == PM_SOURCE_TYPE_DEFECT) return false; + if (source->type == PM_SOURCE_TYPE_SATURATED) return false; + + float Xc = source->peak->xf - source->pixels->col0 - 0.5; + float Yc = source->peak->yf - source->pixels->row0 - 0.5; + + psImageMaskType notMaskVal = ~maskVal; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + + float radius = hypot (ix - Xc, iy - Yc) ; + + if (radius > 4) continue; + + if (add) { + source->maskView->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= maskVal; + } else { + source->maskView->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] &= notMaskVal; + } + } + } + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotAnnuli.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotAnnuli.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotAnnuli.c (revision 41083) @@ -0,0 +1,61 @@ +# include "psphotInternal.h" + +bool psphotAnnuli (pmSource *source, psMetadata *recipe, psImageMaskType maskVal) { + + assert (source->extpars); + assert (source->extpars->profile); + assert (source->extpars->profile->radius); + assert (source->extpars->profile->flux); + + bool status; + + psVector *radius = source->extpars->profile->radius; + psVector *variance = source->extpars->profile->variance; + psVector *flux = source->extpars->profile->flux; + + // XXX how do I define the radii? we can put a vector in the recipe... + // radialBins defines the bounds or start and stop (we can skip some that way... + psVector *radialBinsLower = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.LOWER"); + psVector *radialBinsUpper = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + assert (radialBinsLower->n == radialBinsUpper->n); + + psVector *fluxValues = psVectorAlloc (radialBinsLower->n, PS_TYPE_F32); + psVector *fluxSquare = psVectorAlloc (radialBinsLower->n, PS_TYPE_F32); + psVector *fluxVariance = psVectorAlloc (radialBinsLower->n, PS_TYPE_F32); + psVector *pixelCount = psVectorAlloc (radialBinsLower->n, PS_TYPE_F32); + psVectorInit (fluxValues, 0.0); + psVectorInit (fluxSquare, 0.0); + psVectorInit (fluxVariance, 0.0); + psVectorInit (pixelCount, 0.0); + + // XXX this code assumes the radii are in pixels. convert from arcsec with plate scale + // XXX assume the annulii above are not overlapping? much faster... + // XXX this might be must faster in the reverse order: loop over annulii and use disection to + // skip to the start of the annulus. + for (int i = 0; i < flux->n; i++) { + for (int j = 0; j < radialBinsLower->n; j++) { + if (radius->data.F32[i] < radialBinsLower->data.F32[j]) continue; + if (radius->data.F32[i] > radialBinsUpper->data.F32[j]) continue; + fluxValues->data.F32[j] += flux->data.F32[i]; + fluxSquare->data.F32[j] += PS_SQR(flux->data.F32[i]); + fluxVariance->data.F32[j] += variance->data.F32[i]; + pixelCount->data.F32[j] += 1.0; + } + } + + for (int j = 0; j < radialBinsLower->n; j++) { + fluxValues->data.F32[j] /= pixelCount->data.F32[j]; + fluxSquare->data.F32[j] /= pixelCount->data.F32[j]; + fluxSquare->data.F32[j] -= PS_SQR(fluxValues->data.F32[j]); + } + + source->extpars->annuli = pmSourceAnnuliAlloc (); + source->extpars->annuli->flux = fluxValues; + source->extpars->annuli->fluxErr = fluxVariance; + source->extpars->annuli->fluxVar = fluxSquare; + + psFree (pixelCount); + + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotApResid.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotApResid.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotApResid.c (revision 41083) @@ -0,0 +1,523 @@ +# include "psphotInternal.h" +// # define DEBUG + +# define SKIPSTAR(MSG) { psTrace ("psphot", 3, "invalid : %s", MSG); continue; } +// measure the aperture residual statistics and 2D variations + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotApResid (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Aperture Residuals ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotApResidReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure aperture residual for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotApResidReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + int Nfail = 0; + int Nskip = 0; + int Npsf; + bool status; + pmModel *model; + pmSource *source; + + psTimerStart ("psphot.apresid"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping ap resid"); + return true; + } + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + bool measureAptrend = psMetadataLookupBool (&status, recipe, "MEASURE.APTREND"); + if (!measureAptrend) { + // save nan values since these were not calculated + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APMIFIT", PS_META_REPLACE, "aperture residual", NAN); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "DAPMIFIT", PS_META_REPLACE, "ap residual scatter", NAN); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NAPMIFIT", PS_META_REPLACE, "number of apresid stars", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APLOSS", PS_META_REPLACE, "aperture loss (mag)", NAN); + return true; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // clipping for extreme outliers + // XXX this is not currently defined in the recipe + float MAX_AP_OFFSET = psMetadataLookupF32 (&status, recipe, "MAX_AP_OFFSET"); + + // options for how the photometry is calculated + // XXX are these sensible? + bool IGNORE_GROWTH = psMetadataLookupBool (&status, recipe, "IGNORE_GROWTH"); + bool INTERPOLATE_AP = psMetadataLookupBool (&status, recipe, "INTERPOLATE_AP"); + + // XXX is this still needed? the pmTrend2D stuff should be auto-adjusting... + int APTREND_NSTAR_MIN = psMetadataLookupS32(&status, recipe, "APTREND.NSTAR.MIN"); + assert (status); + + // maximum order for aperture correction + int APTREND_ORDER_MAX = psMetadataLookupS32(&status, recipe, "APTREND.ORDER.MAX"); + assert (status); + + if (APTREND_ORDER_MAX < 1) { + psError(PSPHOT_ERR_CONFIG, true, "APTREND.ORDER.MAX must be 1 or more"); + return false; + } + + pmSourcePhotometryMode photMode = 0; + if (!IGNORE_GROWTH) photMode |= PM_SOURCE_PHOT_GROWTH; + if (INTERPOLATE_AP) photMode |= PM_SOURCE_PHOT_INTERP; + + // set limits on the aperture magnitudes + pmSourceMagnitudesInit (config, recipe); + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_APRESID_MAGS"); + + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, psf); + PS_ARRAY_ADD_SCALAR(job->args, photMode, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nskip + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(cellGroups); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[5]; + Nskip += scalar->data.S32; + scalar = job->args->data[6]; + Nfail += scalar->data.S32; + } + psFree(job); + } + } + + psFree (cellGroups); + + // gather the stats to assess the aperture residuals + psVector *mag = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *xPos = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *yPos = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *apResid = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *dMag = psVectorAllocEmpty (300, PS_TYPE_F32); + Npsf = 0; + +# ifdef DEBUG + FILE *f = fopen ("apresid.dat", "w"); + psAssert (f, "failed open"); +# endif + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + model = source->modelPSF; + + if (source->type != PM_SOURCE_TYPE_STAR) SKIPSTAR ("NOT STAR"); + if (source->mode & PM_SOURCE_MODE_SATSTAR) SKIPSTAR ("SATSTAR"); + if (source->mode & PM_SOURCE_MODE_BLEND) SKIPSTAR ("BLEND"); + if (source->mode & PM_SOURCE_MODE_FAIL) SKIPSTAR ("FAIL STAR"); + if (source->mode & PM_SOURCE_MODE_POOR) SKIPSTAR ("POOR STAR"); + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) SKIPSTAR ("SATSTAR"); + + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) SKIPSTAR ("EXTENDED"); + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) SKIPSTAR ("COSMIC RAY"); + if (source->mode & PM_SOURCE_MODE_DEFECT) SKIPSTAR ("DEFECT"); + if (source->mode2 & PM_SOURCE_MODE2_MATCHED) SKIPSTAR ("MATCHED"); + + if (!isfinite(source->apMag) || !isfinite(source->psfMag)) { + continue; + } + + // XXX make this user-configurable? + if (source->psfMagErr > 0.03) continue; + + // aperture residual for this source + float dap = source->apMag - source->psfMag; + + // sanity clipping : if the model is very discrepant, put your expectation in the recipe + if ((MAX_AP_OFFSET > 0) && (fabs(dap) > MAX_AP_OFFSET)) { + Nfail ++; + psTrace ("psphot", 3, "fail : bad dap %f %f", dap, MAX_AP_OFFSET); + continue; + } + +# ifdef DEBUG + fprintf (f, "%6.1f %6.1f : %6.1f %6.1f : %8.3f %8.3f %8.3f : %f : %f %f %f : %f\n", + source->peak->xf, source->peak->yf, + source->modelPSF->params->data.F32[PM_PAR_XPOS], source->modelPSF->params->data.F32[PM_PAR_YPOS], + source->psfMag, source->apMag, source->psfMagErr, + source->modelPSF->params->data.F32[PM_PAR_I0], + source->modelPSF->params->data.F32[PM_PAR_SXX], source->modelPSF->params->data.F32[PM_PAR_SXY], source->modelPSF->params->data.F32[PM_PAR_SYY], + source->modelPSF->params->data.F32[PM_PAR_7]); +# endif + if (!isfinite(source->psfMag)) psAbort ("nan in psfMag"); + if (!isfinite(source->psfMagErr)) psAbort ("nan in psfMagErr"); + if (!isfinite(source->apMag)) psAbort ("nan in apMag"); + if (!isfinite(model->params->data.F32[PM_PAR_XPOS])) psAbort ("nan in xPos"); + if (!isfinite(model->params->data.F32[PM_PAR_YPOS])) psAbort ("nan in yPos"); + + psVectorAppend (mag, source->psfMag); + psVectorAppend (dMag,source->psfMagErr); + psVectorAppend (apResid, dap); + psVectorAppend (xPos, model->params->data.F32[PM_PAR_XPOS]); + psVectorAppend (yPos, model->params->data.F32[PM_PAR_YPOS]); + Npsf ++; + } + + psLogMsg ("psphot.apresid", PS_LOG_DETAIL, "measure aperture residuals for %d objects (%d skipped, %d failed, %ld invalid)\n", + Npsf, Nskip, Nfail, sources->n - Npsf - Nskip - Nfail); + +# ifdef DEBUG + fclose (f); +# endif + + // XXX choose a better value here? + if (Npsf < APTREND_NSTAR_MIN) { + psWarning("Only %d valid aperture residual sources (need %d), giving up", Npsf, APTREND_NSTAR_MIN); + goto escape; + } + + // this is a bit tricky, because we have two cases (MAP vs POLY), and they have a different + // definition for 'order' (order_MAP = order_POLY + 1). in addition, we have a + // user-specified MAX order, which we should respect, regardless of the mode + + // set the max order (0 = constant) which the number of psf stars can support: + // we require only 3 stars for n = 0, increase stars / cell for higher order + int MaxOrderForStars = 0; + if (Npsf >= 16) MaxOrderForStars = 1; // 4 cells + if (Npsf >= 54) MaxOrderForStars = 2; // 9 cells + if (Npsf >= 128) MaxOrderForStars = 3; // 16 cells + if (Npsf >= 300) MaxOrderForStars = 4; // 25 cells + if (Npsf > 576) MaxOrderForStars = 5; // 36 cells + + pmTrend2DMode mode = PM_TREND_MAP; + if (mode == PM_TREND_MAP) { + MaxOrderForStars ++; + } + APTREND_ORDER_MAX = PS_MIN (APTREND_ORDER_MAX, MaxOrderForStars); + + psFree (psf->ApTrend); + psf->ApTrend = NULL; + float errorFloorMin = FLT_MAX; + + // as we loop over orders, we need to refer to the initial selection, but we modify the + // option values to match the current guess: save the max values here: + int NX = readout->image->numCols; + int NY = readout->image->numRows; + for (int i = 1; i <= APTREND_ORDER_MAX; i++) { + + int Nx, Ny; + if (NX > NY) { + Nx = i; + Ny = PS_MAX (1, (int)(i * (NY / (float)(NX)) + 0.5)); + } else { + Ny = i; + Nx = PS_MAX (1, (int)(i * (NX / (float)(NY)) + 0.5)); + } + + float errorFloor; + pmTrend2D *apTrend = psphotApResidTrend (&errorFloor, readout, Nx, Ny, xPos, yPos, apResid, dMag); + if (!apTrend) { + continue; + } + + // apply ApTrend results + // float xc = 0.5*readout->image->numCols + readout->image->col0 + 0.5; + // float yc = 0.5*readout->image->numRows + readout->image->row0 + 0.5; + // float ApResid = pmTrend2DEval (psf->ApTrend, xc, yc); // ap-fit at chip center + // if (!isfinite(ApResid)) psAbort("nan apresid @ center"); + + // store the minimum errorFloor and best ApTrend to keep + if (errorFloor < errorFloorMin) { + errorFloorMin = errorFloor; + psFree (psf->ApTrend); + psf->ApTrend = psMemIncrRefCounter(apTrend); + } + psFree (apTrend); + } + if (psf->ApTrend == NULL) { + psWarning("Failed to find a valid aperture residual value"); + goto escape; + } + + // apply ApTrend results + float xc = 0.5*readout->image->numCols + readout->image->col0 + 0.5; + float yc = 0.5*readout->image->numRows + readout->image->row0 + 0.5; + + psf->ApResid = pmTrend2DEval (psf->ApTrend, xc, yc); // ap-fit at chip center + psf->dApResid = errorFloorMin; + psf->nApResid = Npsf; + + // save results for later output + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APMIFIT", PS_META_REPLACE, "aperture residual", psf->ApResid); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "DAPMIFIT", PS_META_REPLACE, "ap residual scatter", psf->dApResid); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NAPMIFIT", PS_META_REPLACE, "number of apresid stars", psf->nApResid); + + // curve-of-growth offset + if (psf->growth) { + float gaussSigma = psMetadataLookupF32(&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + gaussSigma = psMetadataLookupF32(&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float apScale = psMetadataLookupF32(&status, recipe, "PSF_APERTURE_SCALE"); + float PSF_APERTURE = (int)(apScale*gaussSigma); + + float offset = pmGrowthCurveCorrect (psf->growth, PSF_APERTURE); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APLOSS", PS_META_REPLACE, "aperture loss (mag)", psf->growth->apLoss); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APREFOFF", PS_META_REPLACE, "offset to ref aperture", offset); + } else { + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APLOSS", PS_META_REPLACE, "aperture loss (mag)", NAN); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APREFOFF", PS_META_REPLACE, "offset to ref aperture", NAN); + } + + psLogMsg ("psphot.apresid", PS_LOG_DETAIL, "aperture residual: %f +/- %f\n", psf->ApResid, psf->dApResid); + psLogMsg ("psphot.apresid", PS_LOG_WARN, "measure full-frame aperture residuals for %d sources: %f sec\n", Npsf, psTimerMark ("psphot.apresid")); + + psFree (xPos); + psFree (yPos); + psFree (apResid); + psFree (mag); + psFree (dMag); + + psphotVisualPlotApResid (sources, psf->ApResid, psf->dApResid, true); + + return true; + +escape: + // save nan values since these were not calculated + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APMIFIT", PS_META_REPLACE, "aperture residual", NAN); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "DAPMIFIT", PS_META_REPLACE, "ap residual scatter", NAN); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NAPMIFIT", PS_META_REPLACE, "number of apresid stars", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "APLOSS", PS_META_REPLACE, "aperture loss (mag)", NAN); + + psFree (xPos); + psFree (yPos); + psFree (apResid); + psFree (mag); + psFree (dMag); + return true; + // this is a quality error, not a programming error +} + +pmTrend2D *psphotApResidTrend (float *apResidSysErr, pmReadout *readout, int Nx, int Ny, psVector *xPos, psVector *yPos, psVector *apResid, psVector *dMag) { + + // the mask marks the values not used to calculate the ApTrend + psVector *mask = psVectorAlloc(xPos->n, PS_TYPE_VECTOR_MASK); + psVectorInit (mask, 0); + + // XXX allow user to set this optionally? + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + + // measure Trend2D for the current spatial scale + pmTrend2D *apTrend = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + + // XXX somewhat arbitrary: soften the errors so the few bright stars do not totally dominate: + // XXX use this or not? probably not, since this is the point of the systematic error analysis + psVector *dMagSoft = psVectorAlloc (dMag->n, PS_TYPE_F32); + for (int i = 0; i < dMag->n; i++) { + dMagSoft->data.F32[i] = hypot(dMag->data.F32[i], 0.005); + } + + // XXX test for errors here + bool goodFit = false; + if (!pmTrend2DFit (&goodFit, apTrend, mask, 0xff, xPos, yPos, apResid, dMagSoft)) { + // XXX this is probably a real error, and I should exit + psWarning("Failed to fit trend for %d x %d map", Nx, Ny); + psFree (apTrend); + return NULL; + } + if (!goodFit) { + psWarning("Failed to fit trend for %d x %d map", Nx, Ny); + psFree (apTrend); + return NULL; + } + if (apTrend->mode == PM_TREND_MAP) { + // p_psImagePrint (2, apTrend->map->map, "ApTrend Before"); // XXX TEST: + psImageMapRepair (apTrend->map->map); + // p_psImagePrint (2, apTrend->map->map, "ApTrend After"); // XXX TEST: + } + + // construct the fitted values and the residuals + psVector *apResidFit = pmTrend2DEvalVector (apTrend, mask, 0xff, xPos, yPos); + psVector *apResidRes = (psVector *) psBinaryOp (NULL, (void *) apResid, "-", (void *) apResidFit); + + // measure systematic error + *apResidSysErr = psVectorSystematicError (apResidRes, dMag, 0.10); + if (!isfinite(*apResidSysErr)) { + psWarning("Failed to find systematic error for %d x %d map", Nx, Ny); + psFree (apTrend); + return NULL; + } + + psLogMsg ("psphot.apresid", PS_LOG_INFO, "result of %d x %d grid\n", Nx, Ny); + psLogMsg ("psphot.apresid", PS_LOG_INFO, "systematic scatter floor: %f\n", *apResidSysErr); + + if (psTraceGetLevel("psphot") >= 4) { + char filename[64]; + snprintf (filename, 64, "apresid.%dx%d.dat", Nx, Ny); + FILE *dumpFile = fopen (filename, "w"); + for (int i = 0; i < xPos->n; i++) { + fprintf (dumpFile, "%f %f %f %f %f %f %x\n", + xPos->data.F32[i], yPos->data.F32[i], + dMag->data.F32[i], hypot(dMag->data.F32[i], *apResidSysErr), + apResid->data.F32[i], apResidRes->data.F32[i], + mask->data.PS_TYPE_VECTOR_MASK_DATA[i]); + } + fclose (dumpFile); + } + + psFree (mask); + psFree (stats); + psFree (dMagSoft); + psFree (apResidFit); + psFree (apResidRes); + + return apTrend; +} + +bool psphotApResidMags_Threaded (psThreadJob *job) { + + int Nskip = 0; + int Nfail = 0; + + psScalar *scalar = NULL; + + psArray *sources = job->args->data[0]; + pmPSF *psf = job->args->data[1]; + pmSourcePhotometryMode photMode = PS_SCALAR_VALUE(job->args->data[2],S32); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[3],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[4],PS_TYPE_IMAGE_MASK_DATA); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = (pmSource *) sources->data[i]; + + if (source->type != PM_SOURCE_TYPE_STAR) SKIPSTAR ("NOT STAR"); + if (source->mode & PM_SOURCE_MODE_SATSTAR) SKIPSTAR ("SATSTAR"); + if (source->mode & PM_SOURCE_MODE_BLEND) SKIPSTAR ("BLEND"); + if (source->mode & PM_SOURCE_MODE_FAIL) SKIPSTAR ("FAIL STAR"); + if (source->mode & PM_SOURCE_MODE_POOR) SKIPSTAR ("POOR STAR"); + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // clear the mask bit and set the circular mask pixels + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", markVal); + + bool status = pmSourceMagnitudes (source, psf, photMode, maskVal, markVal, source->apRadius); + + // clear the mask bit + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + if (!status) { + Nskip ++; + psTrace ("psphot", 3, "skip : bad source mag"); + continue; + } + + if (!isfinite(source->apMag) || !isfinite(source->psfMag)) { + Nfail ++; + psTrace ("psphot", 3, "fail : %f, %f : nan mags : %f %f", source->peak->xf, source->peak->yf, source->apMag, source->psfMag); + continue; + } + source->mode |= PM_SOURCE_MODE_AP_MAGS; + } + + // change the value of a scalar on the array (wrap this and put it in psArray.h) + scalar = job->args->data[5]; + scalar->data.S32 = Nskip; + + scalar = job->args->data[6]; + scalar->data.S32 = Nfail; + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotArguments.c (revision 41083) @@ -0,0 +1,248 @@ +# include "psphotStandAlone.h" + +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile) +{ + fprintf(ofile, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "%s -help\n" + "\n" + "%s -version\n" + "\n" + "where:\n" + " FileNameList is a text file containing filenames, one per line\n" + " MaskFileNameList is a text file of mask filenames, one per line\n" + " VarFileNameList is a text file of variance filenames, one per line\n" + " OutFileBaseName is the 'root name' for output files\n" + "\n" + "additional options:\n" + " -modeltest xObj yObj [-model DEFAULT|ModelName] \n" + " [-fitmode DEFAULT|PSF|CONV] [-fitset FitFileName]\n" + " Test fit for object at the given coordinates. ModelName\n" + " is one of PS_MODEL_GAUSS, PS_MODEL_PGAUSS, PS_MODEL_QGAUSS,\n" + " PS_MODEL_RGAUSS, PS_MODEL_PS1_V1, or PS_MODEL_SERSIC.\n" + " FitFileName is a file of x,y,Io triples.\n" + " -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n" + " specify PSF rather than letting %s estimate it\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify additional sources for PSF generation\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -photcode PhotoCodeName\n" + " specify photocode\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -nthreads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n", + program,program,program,program,program); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "Try '%s -help' for more options and explanation\n", + program,program,program); + if (exitCode != PS_EXIT_SUCCESS) + psErrorStackPrint(stderr, "Error reading arguments\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + +pmConfig *psphotArguments(int argc, char **argv) { + + int N; + bool status; + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + // generic arguments (version, dumpconfig) + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // thread arguments + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + // run the test model (requires X,Y coordinate) + if ((N = psArgumentGet (argc, argv, "-modeltest"))) { + if (argc<=N+2) { + psErrorStackPrint(stderr, "Expected to see 2 more arguments; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + + psMetadataAddBool (options, PS_LIST_TAIL, "TEST_FIT", 0, "", true); + psMetadataAddF32 (options, PS_LIST_TAIL, "TEST_FIT_X", 0, "", atof(argv[N+1])); + psMetadataAddF32 (options, PS_LIST_TAIL, "TEST_FIT_Y", 0, "", atof(argv[N+2])); + + psArgumentRemove (N, &argc, argv); + psArgumentRemove (N, &argc, argv); + psArgumentRemove (N, &argc, argv); + + // specify the modeltest model + if ((N = psArgumentGet (argc, argv, "-model"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "TEST_FIT_MODEL", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // specify the test fit mode + if ((N = psArgumentGet (argc, argv, "-fitmode"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "TEST_FIT_MODE", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-fitset"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "TEST_FIT_SET", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + } + + // photcode : used in output to supplement header data (argument or recipe?) + if ((N = psArgumentGet (argc, argv, "-photcode"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "PHOTCODE", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // break : used from recipe throughout psphotReadout + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF + if ((N = psArgumentGet (argc, argv, "-region"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + // + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "PSPHOT.PSF", "-psf", "-psflist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC", "-src", "-srclist"); + // XXX allow this format? pmConfigFileSetsMD (config->arguments, &argc, argv, "FORCE", "-force", "-forcelist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "EXPNUM", "-expnum", "-expnumlist"); + + if (argc == 1) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (psArgumentGet(argc, argv, "-help") || + psArgumentGet(argc, argv, "-h")) + writeHelpInfo(argv[0], config, stdout); + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotArguments...\n"); + return (config); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotBasicDeblend.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotBasicDeblend.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotBasicDeblend.c (revision 41083) @@ -0,0 +1,166 @@ +# include "psphotInternal.h" + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotBasicDeblend (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotBasicDeblendReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on basic deblend analysis for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotBasicDeblendReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int fileIndex) { + + int N; + bool status; + float threshold; + pmSource *source, *testSource; + + psTimerStart ("psphot.deblend.basic"); + + int Nblend = 0; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, fileIndex); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping basic deblend"); + return true; + } + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + float FRACTION = psMetadataLookupF32 (&status, recipe, "DEBLEND_PEAK_FRACTION"); + if (!status) FRACTION = 0.25; + + float NSIGMA = psMetadataLookupF32 (&status, recipe, "DEBLEND_SKY_NSIGMA"); + if (!status) NSIGMA = 5.0; + + // we need sources spatially-sorted to find overlaps + sources = psArraySort (sources, pmSourceSortByY); + + // source analysis is done in peak order (brightest first) + // we use an index for this so the spatial sorting is kept + psVector *SN = psVectorAlloc (sources->n, PS_DATA_F32); + for (int i = 0; i < SN->n; i++) { + source = sources->data[i]; + SN->data.F32[i] = source->peak->rawFlux; + } + psVector *index = psVectorSortIndex (NULL, SN); + // this results in an index of increasing SN + + // examine sources in decreasing SN order + for (int i = sources->n - 1; i >= 0; i--) { + N = index->data.U32[i]; + source = sources->data[N]; + + if (source->mode & PM_SOURCE_MODE_BLEND) continue; + + // temporary array for overlapping objects we find + psArray *overlap = psArrayAllocEmpty (100); + + // search backwards for overlapping sources + for (int j = N - 1; j >= 0; j--) { + testSource = sources->data[j]; + if (testSource->peak->x < source->pixels->col0) continue; + if (testSource->peak->x >= source->pixels->col0 + source->pixels->numCols) continue; + if (testSource->peak->y < source->pixels->row0) break; + if (testSource->peak->y >= source->pixels->row0 + source->pixels->numRows) { + fprintf (stderr, "warning: invalid condition\n"); + continue; + } + psArrayAdd (overlap, 100, testSource); + } + + // search forwards for overlapping sources + for (int j = N + 1; j < sources->n; j++) { + testSource = sources->data[j]; + if (testSource->peak->x < source->pixels->col0) continue; + if (testSource->peak->x >= source->pixels->col0 + source->pixels->numCols) continue; + if (testSource->peak->y < source->pixels->row0) { + fprintf (stderr, "warning: invalid condition\n"); + continue; + } + if (testSource->peak->y >= source->pixels->row0 + source->pixels->numRows) break; + psArrayAdd (overlap, 100, testSource); + } + + if (overlap->n == 0) { + psFree (overlap); + continue; + } + + // this source has overlapping neighbors, check for actual blends + // generate source contour (1/4 peak counts) + // set the threshold based on user inputs + + // threshold is fraction of the source peak flux + // image is background subtracted; source->moments->Sky should always be 0.0 + threshold = FRACTION * sqrt(source->peak->detValue); + // threshold is no less than NSIGMA + threshold = PS_MAX (threshold, NSIGMA); + + // generate a basic contour (set of x,y coordinates at-or-below flux level) + psArray *contour = pmSourceContour (source->pixels, source->peak->x, source->peak->y, threshold); + if (contour == NULL) { + psFree (overlap); + continue; + } + + // the source contour consists of two vectors, xv and yv. the contour is + // a series of coordinate pairs, (xv[i],yv[i]) & (xv[i+1],yv[i+1]). both + // coordinate pairs have the same yv[] value, with xv[i] corresponding to + // the left boundary and xv[i+1] corresponding to the right boundary + + // a blend can only be associated with one primary source + + psVector *xv = contour->data[0]; + psVector *yv = contour->data[1]; + for (int k = 0; k < overlap->n; k++) { + testSource = overlap->data[k]; + if (testSource->mode & PM_SOURCE_MODE_BLEND) continue; + if (testSource->peak->rawFlux > source->peak->rawFlux) continue; + for (int j = 0; j < xv->n; j+=2) { + if (fabs(yv->data.F32[j] - testSource->peak->y) > 0.5) continue; + if (xv->data.F32[j+0] > testSource->peak->x) break; + if (xv->data.F32[j+1] < testSource->peak->x) break; + + testSource->mode |= PM_SOURCE_MODE_BLEND; + + // add this to the list of source->blends + if (source->blends == NULL) { + source->blends = psArrayAllocEmpty (16); + } + psArrayAdd (source->blends, 16, testSource); + + Nblend ++; + j = xv->n; + } + } + psFree (overlap); + psFree (contour); + } + psLogMsg ("psphot.deblend", PS_LOG_INFO, "identified %d blended objects: %f sec\n", Nblend, psTimerMark ("psphot.deblend.basic")); + + psFree (SN); + psFree (index); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotBlendFit.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotBlendFit.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotBlendFit.c (revision 41083) @@ -0,0 +1,579 @@ +# include "psphotInternal.h" +bool psphotBlendFitSetSource(pmSource *source, psImage *IDimage, float sigSigma); +bool psphotBlendSetSourceSatstar (psImage *image, pmSource *source); +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X); + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotBlendFit (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Fit Sources (Non-Linear) ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotBlendFitReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (non-linear) for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// XXX I don't like this name +bool psphotBlendFitReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + int Nfit = 0; + int Npsf = 0; + int Next = 0; + int Nfail = 0; + bool status; + + psTimerStart ("psphot.fit.nonlinear"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend fit"); + return true; + } + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // do not thread if we are trying to study the fitting process + if (psTraceGetLevel ("psphot.psphotFitEXT") >= 6) { + nThreads = 0; + } + + // source fitting parameters for extended source fits + int fitIter = psMetadataLookupS32(&status, recipe, "EXT_FIT_ITER"); // Max number of fit iterations + assert (status && fitIter > 0); + + float fitMinTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MIN_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + fitMinTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + psAbort("PSF_FIT_MIN_TOL (and PSF_FIT_TOL) not defined or positive"); + } + } + + float fitMaxTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_TOL"); // Fit tolerance + if (!status || !isfinite(fitMaxTol) || fitMaxTol <= 0) { + fitMaxTol = 1.0; + } + + bool chisqConvergence = psMetadataLookupBool (&status, recipe, "LMM_FIT_CHISQ_CONVERGENCE"); // Fit tolerance + if (!status) { + // default to the old method (chisqConvergence) + chisqConvergence = true; + } + + int gainFactorMode = psMetadataLookupS32 (&status, recipe, "LMM_FIT_GAIN_FACTOR_MODE"); // Fit tolerance + if (!status) { + // default to the old method (chisqConvergence) + gainFactorMode = 0; + } + + bool poisson = psMetadataLookupBool(&status, recipe, "POISSON.ERRORS.PHOT.LMM"); // Poisson errors? + assert (status); + + float maxChisqDOF = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_CHISQ"); // Fit tolerance + + float skySig = psMetadataLookupF32(&status, recipe, "SKY_SIG"); + assert (status && isfinite(skySig) && skySig > 0); + +# if (0) + // **** test block : generate an ID image where pixels are set based the source models (flux > 0.1 peak && flux > 2.0 skySig) + psImage *IDimage = psImageAlloc (readout->image->numCols, readout->image->numRows, PS_TYPE_S32); + psImageInit (IDimage, 0); + + // start with the currently known moments (Mxx, Myy, Mxy) and generate a window image + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + psphotBlendFitSetSource (source, IDimage, skySig); + } + psphotSaveImage (NULL, IDimage, "idimage.fits"); +# endif + + // Define source fitting parameters for extended source fits + pmSourceFitOptions *fitOptions = pmSourceFitOptionsAlloc(); + fitOptions->nIter = fitIter; + fitOptions->minTol = fitMinTol; + fitOptions->maxTol = fitMaxTol; + fitOptions->maxChisqDOF = maxChisqDOF; + fitOptions->poissonErrors = poisson; + fitOptions->weight = PS_SQR(skySig); + fitOptions->mode = PM_SOURCE_FIT_PSF; + fitOptions->covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + + fitOptions->gainFactorMode = gainFactorMode; + fitOptions->chisqConvergence = chisqConvergence; + + psphotInitLimitsPSF (recipe, readout); + psphotInitLimitsEXT (recipe, readout); + psphotInitRadiusPSF (recipe, readout); + + // starts the timer, sets up the array of fitSets + psphotFitInit (nThreads); + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend"); + psFree (fitOptions); + return true; + } + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_BLEND_FIT"); + psArray *newSources = psArrayAllocEmpty(16); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, recipe); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, psf); + psArrayAdd(job->args, 1, newSources); // return for new sources + psArrayAdd(job->args, 1, fitOptions); // default fit options + psFree (newSources); + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfit + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Npsf + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Next + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree (fitOptions); + return NULL; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree (fitOptions); + return NULL; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[6]; + Nfit += scalar->data.S32; + scalar = job->args->data[7]; + Npsf += scalar->data.S32; + scalar = job->args->data[8]; + Next += scalar->data.S32; + scalar = job->args->data[9]; + Nfail += scalar->data.S32; + + // add these back onto sources + psArray *newSources = job->args->data[4]; + for (int j = 0; j < newSources->n; j++) { + psArrayAdd (sources, 16, newSources->data[j]); + } + } + psFree(job); + } + } + psFree (cellGroups); + + if (psTraceGetLevel("psphot") >= 6) { + psphotSaveImage (NULL, readout->image, "image.v2.fits"); + } + psFree (fitOptions); + + psLogMsg ("psphot.psphotBlendFit", PS_LOG_WARN, "fit models: %f sec for %d objects (%d psf, %d ext, %d failed, %ld skipped)\n", psTimerMark ("psphot.fit.nonlinear"), Nfit, Npsf, Next, Nfail, sources->n - Nfit); + psphotFitSummary (); + + psphotVisualShowResidualImage (readout, false); + psphotVisualShowObjectRegions (readout, recipe, sources); + psphotVisualShowFlags (sources); + + return true; +} + +bool psphotBlendFit_Threaded (psThreadJob *job) { + + bool status = false; + int Nfit = 0; + int Npsf = 0; + int Next = 0; + int Nfail = 0; + psScalar *scalar = NULL; + + pmReadout *readout = job->args->data[0]; + psMetadata *recipe = job->args->data[1]; + psArray *sources = job->args->data[2]; + pmPSF *psf = job->args->data[3]; + psArray *newSources = job->args->data[4]; + pmSourceFitOptions *fitOptions = job->args->data[5]; + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // S/N limit to perform full non-linear fits + float FIT_SN_LIM = psMetadataLookupF32 (&status, recipe, "FULL_FIT_SN_LIM"); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion AnalysisRegion = psRegionForImage (readout->image, psRegionFromString (region)); + if (psRegionIsNaN (AnalysisRegion)) psAbort("analysis region mis-defined"); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + +# if (PS_TRACE_ON) +# define TEST_X 653 +# define TEST_Y 466 +# define TESTING 1 + int TEST_ON = false; + if (TESTING && (fabs(source->peak->xf - TEST_X) < 5) && (fabs(source->peak->yf - TEST_Y) < 5)) { + fprintf (stderr, "test object\n"); + psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5); + TEST_ON = true; + } +# undef TEST_X +# undef TEST_Y +# endif + + // skip non-astronomical objects (very likely defects) + if (source->mode & PM_SOURCE_MODE_BLEND) goto skip_blend; + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) goto skip_cr; + if (source->type == PM_SOURCE_TYPE_DEFECT) goto skip_defect; + if (source->type == PM_SOURCE_TYPE_SATURATED) goto skip_sat; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) goto skip_sat; + + // skip DBL second sources (ie, added by psphotFitBlob + if (source->mode & PM_SOURCE_MODE_PAIR) goto skip_blend; + + // do not include MOMENTS_FAILURES in the fit + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) goto skip_generic; + + // limit selection to some SN limit + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + if (source->moments->KronFlux < FIT_SN_LIM * source->moments->KronFluxErr) goto skip_generic; + } else { + if (sqrt(source->peak->detValue) < FIT_SN_LIM) goto skip_generic; + } + // exclude sources outside optional analysis region + if (source->peak->xf < AnalysisRegion.x0) goto skip_generic; + if (source->peak->yf < AnalysisRegion.y0) goto skip_generic; + if (source->peak->xf > AnalysisRegion.x1) goto skip_generic; + if (source->peak->yf > AnalysisRegion.y1) goto skip_generic; + + // if model is NULL, we don't have a starting guess + if (source->modelPSF == NULL) goto skip_generic; + + // skip sources which are insignificant flux? + // XXX this is somewhat ad-hoc + if (source->modelPSF->params->data.F32[1] < 0.1) { + psTrace ("psphot", 5, "skipping near-zero source: %f, %f : %f\n", + source->modelPSF->params->data.F32[1], + source->modelPSF->params->data.F32[2], + source->modelPSF->params->data.F32[3]); + goto skip_generic; + } + + // replace object in image & remove excess noise + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + Nfit ++; + + if (0) { + psF32 Mxx = source->moments->Mxx; + psF32 Myy = source->moments->Myy; + fprintf (stderr, "1: Mxx: %f, Myy: %f\n", Mxx, Myy); + } + + // try fitting PSFs or extended sources depending on source->mode + // these functions subtract the resulting fitted source + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + if (psphotFitBlob (readout, source, newSources, psf, fitOptions, maskVal, markVal)) { +# if (PS_TRACE_ON) + if (TEST_ON) { + psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0); + } +# endif + psTrace ("psphot", 5, "source at %7.1f, %7.1f is ext", source->peak->xf, source->peak->yf); + Next ++; + continue; + } + } else { + if (psphotFitBlend (readout, source, psf, fitOptions, maskVal, markVal)) { +# if (PS_TRACE_ON) + if (TEST_ON) { + psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0); + } +# endif + source->type = PM_SOURCE_TYPE_STAR; + psTrace ("psphot", 5, "source at %7.1f, %7.1f is psf", source->peak->xf, source->peak->yf); + Npsf ++; + continue; + } + } + + if (0) { + psF32 Mxx = source->moments->Mxx; + psF32 Myy = source->moments->Myy; + fprintf (stderr, "2: Mxx: %f, Myy: %f\n", Mxx, Myy); + } + + psTrace ("psphot", 5, "source at %7.1f, %7.1f failed", source->peak->xf, source->peak->yf); + Nfail ++; + + // re-subtract the object, leave local sky, re-bump noise + pmSourceCacheModel (source, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + continue; + + skip_blend: + psTrace ("psphot", 5, "source at %7.1f, %7.1f skipped", source->peak->xf, source->peak->yf); + continue; + skip_cr: + psTrace ("psphot", 5, "source at %7.1f, %7.1f skipped", source->peak->xf, source->peak->yf); + continue; + skip_defect: + psTrace ("psphot", 5, "source at %7.1f, %7.1f skipped", source->peak->xf, source->peak->yf); + continue; + skip_sat: + psTrace ("psphot", 5, "source at %7.1f, %7.1f skipped", source->peak->xf, source->peak->yf); + continue; + skip_generic: + psTrace ("psphot", 5, "source at %7.1f, %7.1f skipped", source->peak->xf, source->peak->yf); + continue; + } + + // change the value of a scalar on the array (wrap this and put it in psArray.h) + scalar = job->args->data[6]; + scalar->data.S32 = Nfit; + + scalar = job->args->data[7]; + scalar->data.S32 = Npsf; + + scalar = job->args->data[8]; + scalar->data.S32 = Next; + + scalar = job->args->data[9]; + scalar->data.S32 = Nfail; + + return true; +} + +bool psphotBlendFitSetSource(pmSource *source, psImage *IDimage, float skySigma) { + + if (!source) return false; + if (!source->peak) return false; // XXX how can we have a peak-less source? + +# if (0) +# define TEST_X 653 +# define TEST_Y 466 + if ((fabs(source->peak->xf - TEST_X) < 5) && (fabs(source->peak->yf - TEST_Y) < 5)) { + fprintf (stderr, "test object\n"); + } +# undef TEST_X +# undef TEST_Y +# endif + + if (!source->moments) return false; + if (source->type == PM_SOURCE_TYPE_DEFECT) return false; + if (source->type == PM_SOURCE_TYPE_SATURATED) return false; + if (source->mode2 == PM_SOURCE_MODE2_MATCHED) return false; + psAssert(IDimage, "need a window"); + + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) { + // find the radius at which we hit something like 0.1*SATURATION + psphotBlendSetSourceSatstar (IDimage, source); + return true; + } + + if (!isfinite(source->moments->Mrf) || source->moments->Mrf < 0 ) return false; + + // we have a source with moments Mx, My, Mxx, Mxy, Myy. we just need to define a Gaussian that has + // these values (and peak of 1.0) + + int Nx = IDimage->numCols; + int Ny = IDimage->numRows; + + float Xo = source->moments->Mx; + float Yo = source->moments->My; + + psEllipseMoments moments; + moments.x2 = source->moments->Mxx; + moments.y2 = source->moments->Myy; + moments.xy = source->moments->Mxy; + + psEllipseAxes axes = psEllipseMomentsToAxes(moments, 20.0); + if (! (isfinite(axes.major) && isfinite(axes.minor) && isfinite(axes.theta)) ) { + fprintf (stderr, "invalid axes found id: %4d major: %f minor: %f theta: %f\n", source->id, axes.major, axes.minor, axes.theta); + return false; + } + + // Use 1st radial moment as size, not sigma. Why this factor of 0.5 ? + float scale = 0.5 * source->moments->Mrf / axes.major; + axes.major *= scale; + axes.minor *= scale; + + psEllipseShape shape = psEllipseAxesToShape(axes); + if (! (isfinite(shape.sx) && isfinite(shape.sy) && isfinite(shape.sxy)) ) { + fprintf (stderr, "invalid shape found id: %d sx: %f sy %f sxy: %f\n", source->id, shape.sx, shape.sy, shape.sxy); + return false; + } + + float Smajor = axes.major; + + // we want to go to 2.15 sigma (0.1 Io) + int minX = PS_MIN(PS_MAX(Xo - 2.15*Smajor, 0), Nx - 1); + int maxX = PS_MIN(PS_MAX(Xo + 2.15*Smajor, 0), Nx - 1); + int minY = PS_MIN(PS_MAX(Yo - 2.15*Smajor, 0), Ny - 1); + int maxY = PS_MIN(PS_MAX(Yo + 2.15*Smajor, 0), Ny - 1); + + float rMxx = 0.5 / PS_SQR(shape.sx); + float rMyy = 0.5 / PS_SQR(shape.sy); + float Sxy = shape.sxy; // factor of -1 is included to match the previous window function + // implementation. XXX: Is this correct? + + int ID = source->id; + float Io = source->peak->rawFlux; + for (int iy = minY; iy < maxY; iy++) { + for (int ix = minX; ix < maxX; ix++) { + + float dX = (ix + 0.5 - Xo); + float dY = (iy + 0.5 - Yo); + + float z = rMxx * PS_SQR(dX) + rMyy * PS_SQR(dY) + Sxy*dX*dY; + if (z > 2.311) continue; + + float f = Io*exp(-z); + + if (f < 2.0*skySigma) continue; + IDimage->data.S32[iy][ix] = ID; + } + } + + return true; +} + +bool psphotBlendSetSourceSatstar (psImage *IDimage, pmSource *source) { + + float Xo = source->satstar->Xo; + float Yo = source->satstar->Yo; + psVector *logRmodel = source->satstar->logRmodel; + psVector *logFmodel = source->satstar->logFmodel; + + float lPeak = logFmodel->data.F32[0]; + float fPeak = pow(10.0, lPeak); + float threshold = 0.1*fPeak; + float logThresh = log10(threshold); + + float radius = NAN; + + // what is the radius for a specific peak fraction? + for (int i = 1; i < logFmodel->n; i++) { + float logF = logFmodel->data.F32[i]; + if (!isfinite(logF)) continue; + + float flux = pow(10.0, logF); + if (flux > threshold) continue; + + float logF0 = logFmodel->data.F32[i - 1]; + float logR0 = logRmodel->data.F32[i - 1]; + + float logF1 = logFmodel->data.F32[i]; + float logR1 = logRmodel->data.F32[i]; + + float logR = InterpolateValues (logF0, logR0, logF1, logR1, logThresh); + radius = pow(10.0, logR); + break; + } + + if (!isfinite(radius)) { + for (int i = logFmodel->n - 1; i >= 0; i--) { + if (!isfinite(logFmodel->data.F32[i])) continue; + float logR = logRmodel->data.F32[i]; + radius = pow(10.0, logR); + break; + } + } + + if (!isfinite(radius)) return false; + + int Nx = IDimage->numCols; + int Ny = IDimage->numRows; + + // region to mask + int minX = PS_MIN(PS_MAX(Xo - radius, 0), Nx - 1); + int maxX = PS_MIN(PS_MAX(Xo + radius, 0), Nx - 1); + int minY = PS_MIN(PS_MAX(Yo - radius, 0), Ny - 1); + int maxY = PS_MIN(PS_MAX(Yo + radius, 0), Ny - 1); + + int ID = source->id; + + fprintf (stderr, "Xo,Yo: %f,%f; radius: %f\n", Xo, Yo, radius); + + for (int iy = minY; iy < maxY; iy++) { + for (int ix = minX; ix < maxX; ix++) { + + float dX = (ix - Xo); + float dY = (iy - Yo); + float R = hypot (dX, dY) ; + if (R > radius) continue; + + IDimage->data.S32[iy][ix] = ID; + } + } + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCheckStarDistribution.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCheckStarDistribution.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCheckStarDistribution.c (revision 41083) @@ -0,0 +1,105 @@ +# include "psphotInternal.h" + +// examine the x,y distribution of the psfstars and extend the selection if needed +// desired region and coverage are specified by pmPSFOptions +// even if stars->n is at the limit, we will add more sources to fill out the area + +bool psphotCheckStarDistribution (psArray *stars, psArray *sources, pmPSFOptions *options) { + + // count the number of PSFSTAR sources in each cell. for polynomial representations, we + // use an image (Norder + 1) x (Norder + 1) + int Nx = options->psfTrendNx; + int Ny = options->psfTrendNy; + if (options->psfTrendMode != PM_TREND_MAP) { + Nx ++; + Ny ++; + } + + // set up and image and an image binning structure to cover the field + psImage *nCell = psImageAlloc (Nx, Ny, PS_TYPE_S32); + psImageInit (nCell, 0); + + psImageBinning *binning = psImageBinningAlloc(); + binning->nXruff = Nx; + binning->nYruff = Ny; + binning->nXfine = options->psfFieldNx; + binning->nYfine = options->psfFieldNy; + + psImageBinningSetScale (binning, PS_IMAGE_BINNING_CENTER); + psImageBinningSetSkipByOffset (binning, options->psfFieldXo, options->psfFieldYo); + + // where are the PSF stars located? + for (int i = 0; i < stars->n; i++) { + + pmSource *source = stars->data[i]; + if (source->peak == NULL) continue; + if (!(source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR)) continue; + + int binX = psImageBinningGetRuffX (binning, source->peak->xf); + int binY = psImageBinningGetRuffY (binning, source->peak->yf); + + assert (binX >= 0); + assert (binY >= 0); + assert (binX < nCell->numCols); + assert (binY < nCell->numRows); + + nCell->data.S32[binY][binX] ++; + } + + // do any cells have too few PSFSTAR sources? + // we use 3 as a minimum (slightly arbitrary...) + for (int iy = 0; iy < nCell->numRows; iy++) { + for (int ix = 0; ix < nCell->numCols; ix++) { + if (nCell->data.S32[iy][ix] < 3) { + int nNew = psphotSupplementStars (stars, sources, binning, ix, iy); + if (nNew) { + psLogMsg ("pmObjects", 3, "added %d fainter PSF candidates to %d,%d\n", nNew, ix, iy); + } else { + psLogMsg ("pmObjects", 3, "tried to add to %d,%d, but no more sources are available\n", ix, iy); + } + } + } + } + + psFree (nCell); + psFree (binning); + return true; +} + +// select more possible PSF stars in the specified cell +// sources should be sorted by SN before calling this function +int psphotSupplementStars (psArray *stars, psArray *sources, psImageBinning *binning, int ix, int iy) { + + int nNew = 0; + + float Xs = psImageBinningGetFineX (binning, ix); + float Xe = psImageBinningGetFineX (binning, ix + 1); + + float Ys = psImageBinningGetFineY (binning, iy); + float Ye = psImageBinningGetFineY (binning, iy + 1); + + for (int i = 0; i < sources->n; i++) { + + // add sources which are marked as stars, in S/N order until... + pmSource *source = sources->data[i]; + if (source->peak == NULL) continue; + if (source->moments == NULL) continue; + if (source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + if (source->type != PM_SOURCE_TYPE_STAR) continue; + + float x = source->peak->xf; + float y = source->peak->yf; + + if (x < Xs) continue; + if (y < Ys) continue; + if (x > Xe) continue; + if (y > Ye) continue; + + source->tmpFlags |= PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; + psArrayAdd (stars, 200, source); + + nNew ++; + } + return nNew; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChipParams.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChipParams.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChipParams.c (revision 41083) @@ -0,0 +1,300 @@ +# include "psphotInternal.h" + +bool psphotChipParamsReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char *filerule, pmReadout *readout, psArray *sources); +bool psphotChipParams_Threaded (psThreadJob *job); +bool psphotChipParamsSource (pmSource *source, psArray *backmaps); + +typedef struct { + int index; + short chipNum; + psString name; + float C0x; + float C1x; + float C2x; + float C0y; + float C1y; + float C2y; + int xMin; + int xMax; + int yMin; + int yMax; +} psphotChipBackmap ; + +psArray * psphotChipParamsParseBackmaps(pmReadout *readout); + +bool psphotChipParams (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Chip Parameters ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + //MEH -- shouldn't do if input is stack... will need to be a config option for now + // -- return true method like for psphotExtendedSourceFits to be consistent... + if (!psMetadataLookupBool (&status, recipe, "CHIP_PARAMS")) { + psLogMsg ("psphot", PS_LOG_INFO, "stack -- skipping backward chip mapping \n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + if (!psphotChipParamsReadout (config, recipe, view, filerule, readout, sources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + + return true; +} + +bool psphotChipParamsReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char *filerule, pmReadout *readout, psArray *sources) { + + bool status = false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping chipParams"); + return true; + } + + psTimerStart ("psphot.chipparams"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + psArray *backmaps = psphotChipParamsParseBackmaps(readout); + if (!backmaps) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to parse chip backmaps for %s", filerule); + return false; + } + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_CHIP_PARAMS"); + + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, backmaps); // sources + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(backmaps); + return false; + } +# else + if (!psphotChipParams_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + // psFree(AnalysisRegion); + psFree(backmaps); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(backmaps); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + psFree (cellGroups); + psFree(backmaps); + + psLogMsg ("psphot.chip.params", PS_LOG_WARN, "measure chip paramters : %f sec for %ld objects\n", psTimerMark ("psphot.chipparams"), sources->n); + return true; +} + +bool psphotChipParams_Threaded (psThreadJob *job) { + + psArray *sources = job->args->data[0]; + psArray *backmaps = job->args->data[1]; + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + + psphotChipParamsSource(source, backmaps); + } + + return true; +} + + +bool psphotChipParamsSource (pmSource *source, psArray *backmaps) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + + float warp_x = source->peak->xf; + float warp_y = source->peak->yf; + + int chipForSource = -1; + for (int i=0; i < backmaps->n; i++) { + psphotChipBackmap *thisChip = backmaps->data[i]; + + float chip_x = thisChip->C0x + thisChip->C1x * warp_x + thisChip->C2x * warp_y; + float chip_y = thisChip->C0y + thisChip->C1y * warp_x + thisChip->C2y * warp_y; + if (chip_x >= thisChip->xMin && chip_x <= thisChip->xMax && + chip_y >= thisChip->yMin && chip_y <= thisChip->yMax) { + + chipForSource = thisChip->chipNum; + if (0) { + printf("source %8d on chip: %s %2d at %6.1f, %6.1f\n", source->chipNum, thisChip->name, + chipForSource, chip_x, chip_y); + } + source->chipNum = chipForSource; + source->chipX = chip_x; + source->chipY = chip_y; + // Since it is on this chip we are done for this source + break; + } + } + if (chipForSource < 0) return false; + + return true; +} + + +psArray * psphotChipParamsParseBackmaps(pmReadout *readout) { + + // XXX: I'm assuming the structure of the FPA here. Find a more usual way + // to find header + psMetadata *header = readout->parent->parent->parent->hdu->header; + if (!header) { + psError(PS_ERR_UNKNOWN, true, "Unable to find header for readout."); + return NULL; + } + + psArray *backmaps = psArrayAllocEmpty(6); + + // find the keywords that define the warp to chip transformations stored by + // pswarpUpdateMetadata() + + for (int i = 0; ; i++) { + bool mdok; + char keyword[16]; + + sprintf(keyword, "SRC_%04d", i); + psString chipName = psMetadataLookupStr(&mdok, header, keyword); + if (!chipName) { + break; + } + psphotChipBackmap *thisChip = psAlloc(sizeof(psphotChipBackmap)); + psArrayAdd(backmaps, 1, thisChip); + psFree(thisChip); + + thisChip->index = i; + thisChip->name = chipName; // not incrementing the reference this gets freed before header + + // XXX: If chipName begins with XY assume gpc1 style class_id and use number as ID + if (!strncmp(chipName, "XY", 2)) { + thisChip->chipNum = atoi(chipName+2); + } else { + // otherwise just use the index + thisChip->chipNum = thisChip->index; + } + + sprintf(keyword, "SEC_%04d", i); + psString chipRegionStr = psMetadataLookupStr(&mdok, header, keyword); + if (!chipRegionStr) { + psError(PS_ERR_UNKNOWN, true, "failed to find required keyword %s", keyword); + psFree(backmaps); + return NULL; + } + int nvals = sscanf(chipRegionStr, "[%d:%d,%d:%d]", &thisChip->xMin, &thisChip->xMax, &thisChip->yMin, &thisChip->yMax); + if (nvals != 4) { + psError(PS_ERR_UNKNOWN, true, "failed to parse %s: %s", keyword, chipRegionStr); + psFree(backmaps); + return NULL; + } + // convert from fits extent convention to zero offset array + #define COORD_SLACK 2 + thisChip->xMin -= COORD_SLACK; thisChip->xMax += COORD_SLACK; + thisChip->yMin -= COORD_SLACK; thisChip->yMax += COORD_SLACK; + + sprintf(keyword, "MPX_%04d", i); + psString mpx = psMetadataLookupStr(&mdok, header, keyword); + if (!mpx) { + psError(PS_ERR_UNKNOWN, true, "failed to find %s", keyword); + psFree(backmaps); + return NULL; + } + nvals = sscanf(mpx, "[%f,%f,%f]", &thisChip->C0x, &thisChip->C1x, &thisChip->C2x); + if (nvals != 3) { + psError(PS_ERR_UNKNOWN, true, "failed to parse %s: %s", keyword, mpx); + psFree(backmaps); + return NULL; + } + sprintf(keyword, "MPY_%04d", i); + psString mpy = psMetadataLookupStr(&mdok, header, keyword); + if (!mpy) { + psError(PS_ERR_UNKNOWN, true, "failed to find %s", keyword); + psFree(backmaps); + return NULL; + } + nvals = sscanf(mpy, "[%f,%f,%f]", &thisChip->C0y, &thisChip->C1y, &thisChip->C2y); + if (nvals != 3) { + psError(PS_ERR_UNKNOWN, true, "failed to parse %s: %s", keyword, mpx); + psFree(backmaps); + return NULL; + } + } + + if (!backmaps->n) { + psFree(backmaps); + return NULL; + } + + return backmaps; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChooseAnalysisOptions.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChooseAnalysisOptions.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChooseAnalysisOptions.c (revision 41083) @@ -0,0 +1,653 @@ +# include "psphotInternal.h" + +// choose which sources will be processed for the petrosian radii and/or galaxy/trail model fits +// a source for which we want galaxy models or petrosians gets + +// currently we also skip the following items (regardless of recipe values): +// TYPE_DEFECT, TYPE_SATURATED, MODE_DEFECT, MODE_SATSTAR, SATSTAR_PROFILE + +// RULES: +// * global override: +// ** PSPHOT.EXT.FIT.ALL.SOURCES +// ** PSPHOT.EXT.FIT.ALL.THRESH (density limit) + +// * select by Star/Galaxy? +// ** PSPHOT.EXT.NSIGMA.LIMIT (sets EXT_LIMIT bit -- star or galaxy) +// -- PSPHOT.FIT.EXT_LIMIT (bool : ignore or not EXT_LIMIT) + +// * select by S/N? +// ** EXTENDED_SOURCE_MODELS : SNLIM (per model value) + +// * select by Flux? +// * needs to depend on the filter +// * define a metadata block +// -- EXT.ANALYSIS.MAG.LIMITS (metadata block per filter) + +// * select by galactic latitude (global override)? +// * if (|b| > LIM) do not limit the number of sources by density +// -- GLAT_MAX + +// * de-select by density (global override)? +// ?? + +// for which sources do we want to run the extended source analysis (petrosian and/or galaxy fits)? +bool psphotChooseAnalysisOptions (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Choose Analysis Options ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotChooseAnalysisOptionsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on source size analysis for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +static bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos); + +// this function use an internal flag to mark sources which have already been measured +bool psphotChooseAnalysisOptionsReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + bool status; + + psTimerStart ("psphot.options"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmChip *chip = pmFPAviewThisChip(view, file->fpa); + psAssert (chip, "missing chip?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source size"); + return true; + } + + // we do the petrosian analysis for the same sources as the extended source fits IFF + // this recipe value is turned on (otherwise we only make a selection based on the mag limits) + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + + // Option to enable fitting of all objects with extended model. + bool extFitAll = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.FIT.ALL.SOURCES"); + assert (status); + + // Fitting everything is fine, but if the source density is high, we probably shouldn't. + float extFitAllThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.FIT.ALL.THRESH"); + assert (status); + + // Determine if this readout is above the threshold to ext fit all sources + if (extFitAll) { + float maskFrac = psMetadataLookupF32(&status,readout->analysis,"READOUT.MASK.FRAC"); + if (status) maskFrac = 0.0; + if (sources->n * (1.0 - maskFrac) > extFitAllThresh) { + extFitAll = false; + } + } + + // use EXT_LIMIT bit to select objects? + bool useEXT_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.NSIGMA.LIMIT.USE"); + assert (status); + + float SN_LIM = psMetadataLookupF32 (&status, recipe, "EXTENDED_SOURCE_SN_LIM"); + assert (status); + + // use GAL_LIMIT to select / skip objects? + psSphereRot *toGal = NULL; + float GAL_LIMIT = 0.0; + float GAL_LIMIT_BULGE = 0.0; + float GAL_LIMIT_SIGMA2 = 0.0; + bool useGAL_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.USE"); + assert (status); + if (useGAL_LIMIT) { + toGal = psSphereRotICRSToGalactic(); + GAL_LIMIT = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT"); + assert (status); + GAL_LIMIT = DEG_TO_RAD(GAL_LIMIT); + GAL_LIMIT_BULGE = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE"); + assert (status); + GAL_LIMIT_BULGE = DEG_TO_RAD(GAL_LIMIT_BULGE); + float GAL_LIMIT_SIGMA = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA"); + assert (status); + assert(GAL_LIMIT_SIGMA > 0); + GAL_LIMIT_SIGMA = DEG_TO_RAD(GAL_LIMIT_SIGMA); + GAL_LIMIT_SIGMA2 = GAL_LIMIT_SIGMA * GAL_LIMIT_SIGMA; + } + + // use MAG.LIMITS + psMetadata *magLimits = psMetadataLookupPtr (&status, recipe, "EXT.ANALYSIS.MAG.LIMITS"); + if (!status || !magLimits) { + psLogMsg ("psphot", PS_LOG_WARN, "EXT.ANALYSIS.MAG.LIMITS not found in the recipe, will use other criteria."); + magLimits = NULL; + } + + float petroFluxLim = NAN; + float extFitFluxLim = NAN; + + if (magLimits) { + float extFitMagLimDefault = NAN; + float extFitMagLim = NAN; + float petroMagLimDefault = NAN; + float petroMagLim = NAN; + + // match to the given filter + psString filterID = psMetadataLookupStr(&status, file->fpa->concepts, "FPA.FILTERID"); + psAssert (filterID, "missing FPA.FILTERID?"); + + psMetadataIterator *iter = psMetadataIteratorAlloc(magLimits, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + if (item->type != PS_DATA_METADATA) { + psAbort ("Invalid type for EXT.ANALYSIS.MAG.LIMITS: %s, not a metadata folder", item->name); + } + psString thisFilter = psMetadataLookupStr (&status, item->data.md, "FILTER.ID"); + psAssert(thisFilter, "missing FILTER.ID"); + + // find a matching filter or default to 'any' + if (!strcasecmp (thisFilter, "any")) { + psString petroMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(petroMagLimStr, "missing MAG.LIMIT.PETRO"); + petroMagLimDefault = atof (petroMagLimStr); + + psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(extFitMagLimStr, "missing MAG.LIMIT.EXTFIT"); + extFitMagLimDefault = atof (extFitMagLimStr); + } + + if (!strcasecmp (thisFilter, filterID)) { + psString petroMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(petroMagLimStr, "missing MAG.LIMIT.PETRO"); + petroMagLim = atof (petroMagLimStr); + + psString extFitMagLimStr = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(extFitMagLimStr, "missing MAG.LIMIT.EXTFIT"); + extFitMagLim = atof (extFitMagLimStr); + break; + } + } + psFree(iter); + if (!isfinite (petroMagLim)) petroMagLim = petroMagLimDefault; + if (!isfinite (extFitMagLim)) extFitMagLim = extFitMagLimDefault; + + // now I need to convert the mag limits into instrumental flux limits + // I need to get a zero point and exposure time for this image + + // select the exposure time + float exptime = psMetadataLookupF32(&status, file->fpa->concepts, "CELL.EXPOSURE"); + if (!status) { + exptime = psMetadataLookupF32(&status, file->fpa->concepts, "FPA.EXPOSURE"); + psAssert (status, "missing CELL.EXPOSURE and FPA.EXPOSURE?"); + } + + // select the exposure time + float zeropt = psMetadataLookupF32(&status, file->fpa->concepts, "FPA.ZP"); + psAssert (status, "missing FPA.ZP?"); + + petroFluxLim = exptime * pow (10.0, 0.4*(zeropt - petroMagLim)); + extFitFluxLim = exptime * pow (10.0, 0.4*(zeropt - extFitMagLim)); + } + + pmSourceTmpF clearBits = ~(PM_SOURCE_TMPF_EXT_FIT | PM_SOURCE_TMPF_PETRO); + for (psS32 i = 0 ; i < sources->n ; i++) { + + pmSource *source = (pmSource *) sources->data[i]; + + // clear the 2 relevant bits + source->tmpFlags &= clearBits; + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // skip sources without a psf model + if (source->modelPSF == NULL) continue; + + // Do the fits if the recipe requests we do extended source fits to everything + if (extFitAll) { + source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; + if (doPetrosian) source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + continue; + } + + if (useEXT_LIMIT) { + if (!(source->mode & PM_SOURCE_MODE_EXT_LIMIT)) { + // not extended so skip + continue; + } + } + + if (useGAL_LIMIT) { + psSphere ptGal, ptSky; + GetGalacticCoords (&ptGal, &ptSky, toGal, chip, source->peak->xf, source->peak->yf); + float l = ptGal.r; + float b_min = GAL_LIMIT + GAL_LIMIT_BULGE * exp(-0.5*(l*l/GAL_LIMIT_SIGMA2)); + if (fabs(ptGal.d) < b_min) continue; + // include an exception for low density skycells below the limit? + } + + // for petro and extFit, we will either use the mag limits or the S/N + if (doPetrosian) { + if (isfinite(petroFluxLim)) { + if (source->moments->KronFlux > petroFluxLim) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + } + } else if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + } + } + if (isfinite(extFitFluxLim)) { + if (source->moments->KronFlux > extFitFluxLim) { + source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; + } + } else { + if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { + source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; + } + } + } + + psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld sources: %f sec\n", sources->n, psTimerMark ("psphot.options")); + + psFree(toGal); + + return true; +} + +// this function use an internal flag to mark sources which have already been measured +bool psphotChooseAnalysisOptionsByObject(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects) +{ + bool status; + + psTimerStart ("psphot.options"); + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Choose Analysis Options (By Object) ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // S/N lim to perform radial aperture analysis + float SN_LIM_RADIAL = psMetadataLookupF32 (&status, recipe, "RADIAL_APERTURES_SN_LIM"); + + // we do the petrosian analysis for the same sources as the extended source fits IFF + // this recipe value is turned on (otherwise we only make a selection based on the mag limits) + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + + // Option to enable fitting of all objects with extended model. + bool extFitAll = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.FIT.ALL.SOURCES"); + assert (status); + + // Fitting everything is fine, but if the source density is high, we probably shouldn't. + float extFitAllThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.FIT.ALL.THRESH"); + assert (status); + + // use EXT_LIMIT bit to select objects? + bool useEXT_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.NSIGMA.LIMIT.USE"); + assert (status); + + float SN_LIM = psMetadataLookupF32 (&status, recipe, "EXTENDED_SOURCE_SN_LIM"); + assert (status); + + // use GAL_LIMIT to select / skip objects? + psSphereRot *toGal = NULL; + float GAL_LIMIT = 0.0; + float GAL_LIMIT_BULGE = 0.0; + float GAL_LIMIT_SIGMA2 = 0.0; + bool useGAL_LIMIT = psMetadataLookupBool(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.USE"); + assert (status); + if (useGAL_LIMIT) { + toGal = psSphereRotICRSToGalactic(); + GAL_LIMIT = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT"); + assert (status); + GAL_LIMIT = DEG_TO_RAD(GAL_LIMIT); + GAL_LIMIT_BULGE = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE"); + assert (status); + GAL_LIMIT_BULGE = DEG_TO_RAD(GAL_LIMIT_BULGE); + float GAL_LIMIT_SIGMA = psMetadataLookupF32(&status, recipe, "EXT.FIT.MIN.GAL.LIMIT.BULGE.SIGMA"); + assert (status); + assert(GAL_LIMIT_SIGMA > 0); + GAL_LIMIT_SIGMA = DEG_TO_RAD(GAL_LIMIT_SIGMA); + GAL_LIMIT_SIGMA2 = GAL_LIMIT_SIGMA * GAL_LIMIT_SIGMA; + } + + // use MAG.LIMITS + psMetadata *magLimits = psMetadataLookupPtr (&status, recipe, "EXT.ANALYSIS.MAG.LIMITS"); + if (!status || !magLimits) { + psLogMsg ("psphot", PS_LOG_WARN, "EXT.ANALYSIS.MAG.LIMITS not found in the recipe, will use other criteria."); + magLimits = NULL; + } + + int num = psphotFileruleCount(config, filerule); + + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // Find the filter for each of the inputs in fpa concepts + psArray *inputFilters = psArrayAlloc(num); + psArray *chips = psArrayAlloc(num); + psVector *zeropt = psVectorAlloc (inputFilters->n, PS_TYPE_F32); + psVector *exptime = psVectorAlloc (inputFilters->n, PS_TYPE_F32); + + // we will not use the chisq image to set the fitting limits + if (chisqNum >= 0) { + inputFilters->data[chisqNum] = psStringCopy("chisq"); + chips->data[chisqNum] = NULL; + zeropt->data.F32[chisqNum] = NAN; + exptime->data.F32[chisqNum] = NAN; + } + + // get the needed metadata for the non-chisq images + for (int i = 0 ; i < num; i++) { + if (i == chisqNum) continue; + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); + psAssert (file, "missing file?"); + + pmChip *chip = pmFPAviewThisChip(view, file->fpa); + psAssert (chip, "missing chip?"); + chips->data[i] = psMemIncrRefCounter (chip); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + float maskFrac = psMetadataLookupF32(&status,readout->analysis,"READOUT.MASK.FRAC"); + if (!status) maskFrac = 0.0; + if (sources->n * (1.0 - maskFrac) > extFitAllThresh) { + extFitAll = false; + } + + // select the filterID for this image + char *filterID = psMetadataLookupStr(&status, file->fpa->concepts, "FPA.FILTERID"); + psAssert (status, "missing FPA.FILTERID?"); + inputFilters->data[i] = psStringCopy (filterID); + + // select the exposure time for this image + exptime->data.F32[i] = psMetadataLookupF32(&status, file->fpa->concepts, "CELL.EXPOSURE"); + if (!status) { + exptime->data.F32[i] = psMetadataLookupF32(&status, file->fpa->concepts, "FPA.EXPOSURE"); + psAssert (status, "missing CELL.EXPOSURE and FPA.EXPOSURE?"); + } + + // select the zero point for this image + zeropt->data.F32[i] = psMetadataLookupF32(&status, file->fpa->concepts, "FPA.ZP"); + psAssert (status, "missing FPA.ZP?"); + } + + // find extFitFluxLim->data.F32[i] for i == image number + psVector *extFitFluxLim = NULL; + psVector *petroFluxLim = NULL; + if (magLimits) { + extFitFluxLim = psVectorAlloc (inputFilters->n, PS_TYPE_F32); + petroFluxLim = psVectorAlloc (inputFilters->n, PS_TYPE_F32); + psVectorInit (extFitFluxLim, NAN); + psVectorInit (petroFluxLim, NAN); + + float extFitMagLimDefault = NAN; + float petroMagLimDefault = NAN; + + // match mag limits (flux limits) to the filters + psMetadataIterator *iter = psMetadataIteratorAlloc(magLimits, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + if (item->type != PS_DATA_METADATA) { + psAbort ("Invalid type for EXT.ANALYSIS.MAG.LIMITS: %s, not a metadata folder", item->name); + } + psString thisFilter = psMetadataLookupStr (&status, item->data.md, "FILTER.ID"); + psAssert(thisFilter, "missing FILTER.ID"); + + // save the default value to assign to unset filters + if (!strcasecmp (thisFilter, "any")) { + psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(magString, "missing MAG.LIMIT.EXTFIT"); + extFitMagLimDefault = atof (magString); + + magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(magString, "missing MAG.LIMIT.PETRO"); + petroMagLimDefault = atof (magString); + continue; + } + + // not every entry in the metadata block needs to match to an image in our list + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; + if (!strcasecmp (thisFilter, inputFilters->data[i])) { + psString magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.PETRO"); + psAssert(magString, "missing MAG.LIMIT.PETRO"); + float magvalue = atof (magString); + petroFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - magvalue)); + + magString = psMetadataLookupStr (&status, item->data.md, "MAG.LIMIT.EXTFIT"); + psAssert(magString, "missing MAG.LIMIT.EXTFIT"); + magvalue = atof (magString); + extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - magvalue)); + break; + } + } + } + psFree(iter); + + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; + if (!isfinite(petroFluxLim->data.F32[i])) { + petroFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - petroMagLimDefault)); + } + if (!isfinite(extFitFluxLim->data.F32[i])) { + extFitFluxLim->data.F32[i] = exptime->data.F32[i] * pow (10.0, 0.4*(zeropt->data.F32[i] - extFitMagLimDefault)); + } + } + } + + pmSourceTmpF clearBits = ~(PM_SOURCE_TMPF_EXT_FIT | PM_SOURCE_TMPF_PETRO); + + for (int i = 0; i < objects->n; i++) { + pmPhotObj *object = objects->data[i]; + if (!object) continue; + if (!object->sources) continue; + + // we check each source for an object and keep the object if any source is valid + + bool doObjectRadial = false; + bool doObjectExtFit = false; + bool doObjectPetrosian = false; + for (int j = 0; !doObjectExtFit && !doObjectRadial && !doObjectPetrosian && (j < object->sources->n); j++) { + + pmSource *source = object->sources->data[j]; + if (!source) continue; + if (!source->peak) continue; + if (source->imageID < 0) continue; // skip sources which come from other images? + if (source->imageID >= num) continue; // skip sources which come from other images? + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // XXX should I fit all even if one of the detections matches the above? + + // check on radial aperture analysis (fewer options for now) + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + bool doSourceRadial = (source->moments->KronFlux > SN_LIM_RADIAL * source->moments->KronFluxErr); + doObjectRadial = doObjectRadial | doSourceRadial; + } else { + bool doSourceRadial = (sqrt(source->peak->detValue) > SN_LIM_RADIAL); + doObjectRadial = doObjectRadial | doSourceRadial; + } + + // Do the fits if the recipe requests we do extended source fits to everything + if (extFitAll) { + doObjectExtFit = true; + doObjectPetrosian = doPetrosian; + continue; + } + + if (useEXT_LIMIT) { + if (!(source->mode & PM_SOURCE_MODE_EXT_LIMIT)) { + continue; // not extended so skip + } + } + + int imageID = source->imageID; + + if (useGAL_LIMIT) { + psSphere ptGal, ptSky; + GetGalacticCoords (&ptGal, &ptSky, toGal, chips->data[imageID], source->peak->xf, source->peak->yf); + float l = ptGal.r; + float b_min = GAL_LIMIT + GAL_LIMIT_BULGE * exp(-0.5*(l*l/GAL_LIMIT_SIGMA2)); + if (fabs(ptGal.d) < b_min) continue; + } + + float fluxLim = NAN; + // for petro and extFit, we will either use the mag limits or the S/N + if (doPetrosian) { + fluxLim = petroFluxLim ? petroFluxLim->data.F32[imageID] : NAN; + if (isfinite(fluxLim)) { + if (source->moments->KronFlux > extFitFluxLim->data.F32[imageID]) { + doObjectPetrosian = true; + } + } else { + if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { + doObjectPetrosian = true; + } + } + } + fluxLim = extFitFluxLim ? extFitFluxLim->data.F32[imageID] : NAN; + // for petro and extFit, we will either use the mag limits or the S/N + if (isfinite(fluxLim)) { + if (source->moments->KronFlux > extFitFluxLim->data.F32[imageID]) { + doObjectExtFit = true; + } + } else { + if (source->moments->KronFlux > SN_LIM * source->moments->KronFluxErr) { + doObjectExtFit = true; + } + } + } + + for (int j = 0; j < object->sources->n; j++) { + + pmSource *source = object->sources->data[j]; + if (!source) continue; + if (!source->peak) continue; + + // clear the 2 relevant bits + source->tmpFlags &= clearBits; + + // skip PSF-like and non-astronomical objects?? + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // skip sources without a psf model + if (source->modelPSF == NULL) continue; + + if (doObjectExtFit) { + source->tmpFlags |= PM_SOURCE_TMPF_EXT_FIT; + } + if (doObjectPetrosian) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO; + } + + // Do the fits if the recipe requests we do extended source fits to everything + if (doObjectRadial) { + source->tmpFlags |= PM_SOURCE_TMPF_RADIAL_KEEP; + source->tmpFlags &= ~PM_SOURCE_TMPF_RADIAL_SKIP; + } else { + source->tmpFlags |= PM_SOURCE_TMPF_RADIAL_SKIP; + source->tmpFlags &= ~PM_SOURCE_TMPF_RADIAL_KEEP; + } + } + } + + psLogMsg ("psphot.options", PS_LOG_WARN, "choose analysis options for %ld objects: %f sec\n", objects->n, psTimerMark ("psphot.options")); + + psFree(exptime); + psFree(zeropt); + psFree(chips); + psFree(inputFilters); + psFree(petroFluxLim); + psFree(extFitFluxLim); + psFree(toGal); + + return true; +} + +static bool GetGalacticCoords (psSphere *ptGal, psSphere *ptSky, psSphereRot *toGal, pmChip *chip, float xPos, float yPos) { + + pmFPA *fpa = chip->parent; + + if (!chip->toFPA) goto escape; + if (!fpa->toTPA) goto escape; + if (!fpa->toSky) goto escape; + + psPlane ptCH, ptFP, ptTP; + + // calculate the astrometry for the coordinate of interest + ptCH.x = xPos; + ptCH.y = yPos; + psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH); + psPlaneTransformApply (&ptTP, fpa->toTPA, &ptFP); + psDeproject (ptSky, &ptTP, fpa->toSky); + psSphereRotApply (ptGal, toGal, ptSky); + + // psSphereRotApply insures that 0 < r < 2PI. We want -PI < b <= PI + if (ptGal->r > M_PI) { + ptGal->r -= 2.0 * M_PI; + } + + return true; + +escape: + // no astrometry calibration, give up + ptSky->r = NAN; + ptSky->d = NAN; + + ptGal->r = NAN; + ptGal->d = NAN; + + return false; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChoosePSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChoosePSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotChoosePSF.c (revision 41083) @@ -0,0 +1,788 @@ +# include "psphotInternal.h" +bool psphotPSFstatsSources (pmReadout *readout, psArray *sources, pmPSF *psf); + +// generate a PSF model for inputs without PSF models already loaded +bool psphotChoosePSF (pmConfig *config, const pmFPAview *view, const char *filerule, bool newSources) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Choose PSF ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotChoosePSFReadout (config, view, filerule, i, recipe, newSources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to choose a psf model for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// try PSF models and select best option +bool psphotChoosePSFReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool newSources) { + + bool status; + + psTimerStart ("psphot.choose.psf"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // do not generate a PSF if we already were supplied one + if (psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF")) { + psLogMsg ("psphot", PS_LOG_DETAIL, "psf model supplied for input file %d", index); + return true; + } + + if (psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psLogMsg ("psphot", PS_LOG_DETAIL, "skipping choose PSF for input file %d", index); + return true; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping PSF model"); + return false; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + psAssert (maskVal, "missing mask value?"); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + psAssert (markVal, "missing mark value?"); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // examine PSF sources in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // structure to store user options defining the psf + pmPSFOptions *options = pmPSFOptionsAlloc(); + + // load user options from the recipe. no need to check existence -- they are + // array to store candidate PSF stars + int NSTARS = psMetadataLookupS32 (&status, recipe, "PSF_MAX_NSTARS"); + assert (status); + + // supply the measured sky variance for optional constant errors (non-poissonian) + float SKY_SIG = psMetadataLookupF32 (&status, recipe, "SKY_SIG"); + assert (status); + + // use poissonian errors or local-sky errors + options->poissonErrorsPhotLMM = psMetadataLookupBool (&status, recipe, "POISSON.ERRORS.PHOT.LMM"); + assert (status); + + // use poissonian errors or local-sky errors + options->poissonErrorsPhotLin = psMetadataLookupBool (&status, recipe, "POISSON.ERRORS.PHOT.LIN"); + assert (status); + + // use poissonian errors or local-sky errors + options->poissonErrorsParams = psMetadataLookupBool (&status, recipe, "POISSON.ERRORS.PARAMS"); + assert (status); + + // how to model the PSF variations across the field + options->psfTrendMode = pmTrend2DModeFromString (psMetadataLookupStr (&status, recipe, "PSF.TREND.MODE")); + assert (status); + + options->psfTrendNx = psMetadataLookupS32 (&status, recipe, "PSF.TREND.NX"); + assert (status); + + options->psfTrendNy = psMetadataLookupS32 (&status, recipe, "PSF.TREND.NY"); + assert (status); + + // get the fixed PSF fit radius + // XXX check that PSF_FIT_RADIUS < SKY_OUTER_RADIUS + // options->radius = psMetadataLookupF32 (&status, recipe, "PSF_FIT_RADIUS"); + // assert (status); + + // values on readout->analysis if we have calculated a Gaussian window function, use that + // for both the PSF fit radius and the aperture radius (scaling SIGMA), otherwise base the value on the recipe value for MOMENTS_GAUSS_SIGMA + float gaussSigma = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + gaussSigma = psMetadataLookupF32 (&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float fitScale = psMetadataLookupF32(&status, recipe, "PSF_FIT_RADIUS_SCALE"); + float apScale = psMetadataLookupF32(&status, recipe, "PSF_APERTURE_SCALE"); + options->fitRadius = (int)(fitScale*gaussSigma); + options->apRadius = (int)(apScale*gaussSigma); + + // XXX use the same radii for standard analysis as for the PSF creation + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "PSF_FIT_RADIUS", PS_META_REPLACE, "fit radius", options->fitRadius); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "PSF_APERTURE", PS_META_REPLACE, "psf aperture", options->apRadius); + + // dimensions of the field for which the PSF is defined + options->psfFieldNx = readout->image->numCols; + options->psfFieldNy = readout->image->numRows; + options->psfFieldXo = readout->image->col0; + options->psfFieldYo = readout->image->row0; + + int fitIter = psMetadataLookupS32(&status, recipe, "PSF_FIT_ITER"); // Maximum number of fit iterations + if (!status || fitIter <= 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "PSF_FIT_ITER is not positive"); + return false; + } + float fitMinTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_MIN_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + fitMinTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "PSF_FIT_MIN_TOL (and PSF_FIT_TOL) not defined or positive"); + return false; + } + } + float fitMaxTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_MAX_TOL"); // Fit tolerance + if (!status || !isfinite(fitMaxTol) || fitMaxTol <= 0) { + fitMaxTol = 1.0; + } + float maxChisqDOF = psMetadataLookupF32 (&status, recipe, "PSF_FIT_MAX_CHISQ"); // Fit tolerance + + bool chisqConvergence = psMetadataLookupBool (&status, recipe, "LMM_FIT_CHISQ_CONVERGENCE"); // Fit tolerance + if (!status) { + // default to the old method (chisqConvergence) + chisqConvergence = true; + } + + int gainFactorMode = psMetadataLookupS32 (&status, recipe, "LMM_FIT_GAIN_FACTOR_MODE"); // Fit tolerance + if (!status) { + // default to the old method (chisqConvergence) + gainFactorMode = 0; + } + + // options which modify the behavior of the model fitting + options->fitOptions = pmSourceFitOptionsAlloc(); + options->fitOptions->nIter = fitIter; + options->fitOptions->minTol = fitMinTol; + options->fitOptions->maxTol = fitMaxTol; + options->fitOptions->maxChisqDOF = maxChisqDOF; + options->fitOptions->poissonErrors = options->poissonErrorsPhotLMM; + options->fitOptions->weight = PS_SQR(SKY_SIG); + options->fitOptions->mode = PM_SOURCE_FIT_PSF; + options->fitOptions->covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + + options->fitOptions->gainFactorMode = gainFactorMode; + options->fitOptions->chisqConvergence = chisqConvergence; + + psArray *stars = psArrayAllocEmpty (sources->n); + + // select the candidate PSF stars (pointers to original sources) + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR) { + // keep NSTARS PSF stars + if (stars->n < NSTARS) { + psArrayAdd (stars, 200, source); + } + } + } + + // check that the identified psf stars sufficiently cover the region; if not, extend the + // limits somewhat + psphotCheckStarDistribution (stars, sources, options); + + psLogMsg ("psphot.pspsf", PS_LOG_DETAIL, "selected %ld candidate PSF objects\n", stars->n); + + if (stars->n < 50) { + // ROBUST is too agressive if we only have a small number of PSF stars + options->stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + } else { + options->stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + } + + // get the list pointers for the PSF_MODEL entries + psArray *modelNames = NULL; + psMetadataItem *item = psMetadataLookup (recipe, "PSF_MODEL"); + if (item == NULL) psAbort("missing PSF_MODEL selection"); + + if (item->type == PS_DATA_STRING) { + modelNames = psArrayAlloc(1); + modelNames->data[0] = psStringCopy (item->data.V); + } else { + if (item->type != PS_DATA_METADATA_MULTI) psAbort("missing PSF_MODEL selection"); + modelNames = psListToArray (item->data.list); + } + + // generate a psf model using the first selection + if (stars->n == 0) { + psError(PSPHOT_ERR_PSF, false, "Failed to fit any models when choosing PSF"); + bool mdok; // Status of MD lookup + if (!psMetadataLookupBool(&mdok, recipe, PSPHOT_RECIPE_PSF_FAKE_ALLOW)) { + return false; + } + psErrorStackPrint(stderr, "Using guess PSF model"); + psErrorClear(); + + psFree(options); + + // no sources are used as PSF stars + + // XXX set sxx, etc from FWHM in recipe + pmPSF *psf = pmPSFBuildSimple (modelNames->data[0], 1.0, 1.0, 0.0, 1.0); + psf->fieldNx = readout->image->numCols; + psf->fieldNy = readout->image->numRows; + psFree (modelNames); + + bool status = true; + status &= psphotMakeFluxScale (readout->image, recipe, psf); + status &= psphotPSFstats (readout, psf); + if (!status) { + psError(PSPHOT_ERR_PSF, false, "Failed to fit any models when choosing PSF"); + psFree (psf); + return NULL; + } + + // XXX set DSX_MEAN, etc? + return psf; + } + + // set up an array to store the results + psArray *models = psArrayAlloc (modelNames->n); + + // try each model option listed in config + // pmPSFtryModel makes a local copy of the sources -- those points are not the same as those for 'sources' + for (int i = 0; i < modelNames->n; i++) { + char *modelName = modelNames->data[i]; + pmPSFtry *try = pmPSFtryModel (stars, modelName, options, maskVal, markVal); // Attempt at fit + if (!try) { + // No big deal --- we'll try another model + if (i < modelNames->n - 1) { + psErrorClear(); + } + continue; + } + models->data[i] = try; + } + + // select the best of the models + // here we are using the clippedStdev on the metric as the indicator + int bestN = -1; + float bestM = 0.0; + for (int i = 0; i < models->n; i++) { + pmPSFtry *try = models->data[i]; + if (try == NULL) { + psTrace ("psphot", 3, "PSF model %d is NULL", i); + continue; + } + float M = try->psf->dApResid; + if (bestN < 0 || M < bestM) { + bestM = M; + bestN = i; + } + } + + // use the best model: + if (bestN < 0) { + psFree (models); + psFree (stars); + + psErrorStackPrint (stderr, "Failed to fit any models when choosing PSF"); + psErrorClear(); + + bool mdok; // Status of MD lookup + if (!psMetadataLookupBool(&mdok, recipe, PSPHOT_RECIPE_PSF_FAKE_ALLOW)) { + psFree (modelNames); + psFree(options); + return NULL; + } + + // generate a psf model using the first selection + psLogMsg ("psphot.pspsf", PS_LOG_INFO, "Using guess PSF model"); + + // no sources are used as PSF stars + + // XXX set sxx, etc from FWHM in recipe + pmPSF *psf = pmPSFBuildSimple (modelNames->data[0], 1.0, 1.0, 0.0, 1.0); + psf->fieldNx = readout->image->numCols; + psf->fieldNy = readout->image->numRows; + psFree (modelNames); + + bool status = true; + status &= psphotMakeFluxScale (readout->image, recipe, psf); + status &= psphotPSFstats (readout, psf); + if (!status) { + psError(PSPHOT_ERR_PSF, false, "Failed to fit any models when choosing PSF"); + psFree (psf); + return NULL; + } + + // XXX set DSX_MEAN, etc? + return psf; + } + + psFree (modelNames); + psFree (stars); + + pmPSFtry *try = models->data[bestN]; + + // set the PSFSTAR flag for stars used for the PSF model + int nKeep = 0; + for (int i = 0; i < try->sources->n; i++) { + pmSource *source = try->sources->data[i]; + if (try->mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue; + source->mode |= PM_SOURCE_MODE_PSFSTAR; + + // this source was used: find the parent and set its PSFSTAR flag + pmSource *realSource = source->parent; + psAssert (realSource, "pmPSFtryAlloc should have set the parent pointers"); + realSource->mode |= PM_SOURCE_MODE_PSFSTAR; + nKeep ++; + } + psLogMsg ("psphot.pspsf", PS_LOG_DETAIL, "used %d of %ld candidate PSF objects\n", nKeep, try->sources->n); + + // build a PSF residual image + // we need to use the 'try' set because they have the associated models defined + if (!psphotMakeResiduals (try->sources, recipe, try->psf, maskVal)) { + psError(PSPHOT_ERR_PSF, false, "Unable to construct residual table for PSF"); + psFree (models); + psFree(options); + return NULL; + } + + // XXX does this work here? + psphotVisualShowPSFStars (recipe, try->psf, try->sources); + + // build the flux-to-magnitude conversion information + if (!psphotMakeFluxScale (readout->image, recipe, try->psf)) { + psError(PSPHOT_ERR_PSF, false, "Unable to construct flux scale for PSF"); + psFree (models); + psFree(options); + return NULL; + } + + // build curve-of-growth vector for this psf + if (!psphotMakeGrowthCurve (readout, recipe, try->psf, try->sources)) { + psError(PSPHOT_ERR_PSF, false, "Unable to construct flux scale for PSF"); + psFree (models); + psFree(options); + return NULL; + } + + // test dump of psf star data and psf-subtracted image + if (psTraceGetLevel("psphot.psfstars") > 5) { + psphotDumpPSFStars (readout, try, options->fitRadius, maskVal, markVal); + } + + // save only the best model; + pmPSF *psf = psMemIncrRefCounter(try->psf); + psFree (models); + + // if (!psphotPSFstats (readout, psf)) { + + if (!psphotPSFstatsSources (readout, sources, psf)) { + psError(PSPHOT_ERR_PSF, false, "cannot measure PSF shape terms"); + psFree(options); + psFree(psf); + return NULL; + } + + + + char *modelName = pmModelClassGetName (psf->type); + psLogMsg ("psphot.pspsf", PS_LOG_WARN, "select psf model: %f sec\n", psTimerMark ("psphot.choose.psf")); + psLogMsg ("psphot.pspsf", PS_LOG_INFO, "psf model %s, ApResid: %f +/- %f (%d x %d model)\n", modelName, psf->ApResid, psf->dApResid, psf->trendNx, psf->trendNy); + + psFree (options); + + // save PSF on readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot psf model", psf)) { + psError (PSPHOT_ERR_UNKNOWN, false, "problem saving sources on readout"); + return false; + } + psFree (psf); + + psphotVisualShowPSFModel (readout, psf); + + return true; +} + +// measure average parameters of the PSF model +bool psphotPSFstats (pmReadout *readout, pmPSF *psf) { + + psEllipseShape shape; + psEllipseAxes axes; + + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(psf, false); + + psImage *image = readout->image; + PS_ASSERT_PTR_NON_NULL(image, false); + + // XXX a minor hack: measure the values for a grid of points across the image, determine mean, UQ, LQ, etc: + psVector *fwhmMajor = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *fwhmMinor = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *psfExtra1 = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *psfExtra2 = psVectorAllocEmpty (100, PS_DATA_F32); + + for (float ix = -0.4; ix <= +0.4; ix += 0.1) { + for (float iy = -0.4; iy <= +0.4; iy += 0.1) { + + // use the center of the center pixel of the image + float xc = ix*image->numCols + 0.5*image->numCols + image->col0 + 0.5; + float yc = iy*image->numRows + 0.5*image->numRows + image->row0 + 0.5; + + // create modelPSF from this model + pmModel *modelPSF = pmModelFromPSFforXY (psf, xc, yc, 1.0); + if (!modelPSF) { + fprintf (stderr, "?"); + continue; + } + + // get the model full-width at half-max + float FWHM_MAJOR = 2*modelPSF->class->modelRadius (modelPSF->params, 0.5); + + // XXX make sure this is consistent with the re-definition of PM_PAR_SXX + shape.sx = modelPSF->params->data.F32[PM_PAR_SXX]; + shape.sy = modelPSF->params->data.F32[PM_PAR_SYY]; + shape.sxy = modelPSF->params->data.F32[PM_PAR_SXY]; + axes = psEllipseShapeToAxes (shape, 20.0); + + float FWHM_MINOR = FWHM_MAJOR * (axes.minor / axes.major); + if (!isfinite(FWHM_MAJOR) || !isfinite(FWHM_MINOR)) { + fprintf (stderr, "!"); + psFree (modelPSF); + continue; + } + psVectorAppend (fwhmMajor, FWHM_MAJOR); + psVectorAppend (fwhmMinor, FWHM_MINOR); + + if (modelPSF->params->n > 7) { + psVectorAppend (psfExtra1, modelPSF->params->data.F32[7]); + } + if (modelPSF->params->n > 8) { + psVectorAppend (psfExtra2, modelPSF->params->data.F32[8]); + } + psFree (modelPSF); + + // float fwhmtest = pmPSFtoFWHM(psf, xc, yc); + // fprintf (stderr, "fwhm: %f, %f : %f\n", FWHM_MAJOR, FWHM_MINOR, fwhmtest); + } + } + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_SAMPLE_QUARTILE); + if (!psVectorStats (stats, fwhmMajor, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MAJOR"); + goto escape; + } + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MAJ", PS_META_REPLACE, "PSF FWHM Major axis (mean)", stats->sampleMean); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_SG", PS_META_REPLACE, "PSF FWHM Major axis (sigma)", stats->sampleStdev); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_LQ", PS_META_REPLACE, "PSF FWHM Major axis (lower quartile)", stats->sampleLQ); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_UQ", PS_META_REPLACE, "PSF FWHM Major axis (upper quartile)", stats->sampleUQ); + + float fwhmMaj = stats->sampleMean; // FWHM on major axis + + if (!psVectorStats (stats, fwhmMinor, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MINOR"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MIN", PS_META_REPLACE, "PSF FWHM Minor axis (mean)", stats->sampleMean); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_SG", PS_META_REPLACE, "PSF FWHM Minor axis (sigma)", stats->sampleStdev); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_LQ", PS_META_REPLACE, "PSF FWHM Minor axis (lower quartile)", stats->sampleLQ); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_UQ", PS_META_REPLACE, "PSF FWHM Minor axis (upper quartile)", stats->sampleUQ); + + float fwhmMin = stats->sampleMean; // FWHM on minor axis + + if (readout->parent) { + + // we now have 2 possible measurements of the seeing : the PSF model based version and + // the Moments based value we need to define a definitive "CHIP.SEEING" value. This + // value is used by the PSF-matching programs (ppSub and ppStack). The moments-based + // value is probably more representative of the value needed by those tools (it is also + // more stable?) + bool status = false; + float fwhmMajorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1"); + float fwhmMinorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1"); + + pmChip *chip = readout->parent->parent; // Parent chip + psAssert(chip, "Cell should be attached to a chip."); + psMetadataItem *item = psMetadataLookup(chip->concepts, "CHIP.SEEING"); // Item with chip + item->data.F32 = 0.5 * (fwhmMajorMoments + fwhmMinorMoments); + + psLogMsg ("psphot", PS_LOG_DETAIL, "fwhm (psf): %f,%f (moments): %f,%f", fwhmMaj, fwhmMin, fwhmMajorMoments, fwhmMinorMoments); + } + + if (psfExtra1->n) { + if (!psVectorStats (stats, psfExtra1, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 1"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT1", PS_META_REPLACE, "PSF extra param 1", stats->sampleMean); + psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 1: %f +/- %f", stats->sampleMean, stats->sampleStdev); + } + + if (psfExtra2->n) { + if (!psVectorStats (stats, psfExtra2, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 2"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT2", PS_META_REPLACE, "PSF extra param 2", stats->sampleMean); + psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 2: %f +/- %f", stats->sampleMean, stats->sampleStdev); + } + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE", PS_META_REPLACE, "PSF angle", axes.theta); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", psf->nPSFstars); + + // psLogMsg ("psphot", PS_LOG_DETAIL, "PSF angle: %f, nstars: %d", axes.theta, psf->nPSFstars); + + char *psfModelName = pmModelClassGetName(psf->type); + psMetadataAddStr(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "PSF Model Name", psfModelName); + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSF_OK", PS_META_REPLACE, "Valid PSF Model?", true); + + int nParams = pmModelClassParameterCount(psf->type); + psMetadataAddS32(readout->analysis, PS_LIST_TAIL, "PSF_NPAR", PS_META_REPLACE, "Number of PSF parameters", nParams); + + psFree (fwhmMajor); + psFree (fwhmMinor); + psFree (psfExtra1); + psFree (psfExtra2); + psFree (stats); + return true; + +escape: + psFree (fwhmMajor); + psFree (fwhmMinor); + psFree (psfExtra1); + psFree (psfExtra2); + psFree (stats); + return false; +} + +// measure average parameters of the PSF model +bool psphotPSFstatsSources (pmReadout *readout, psArray *sources, pmPSF *psf) { + + psEllipseShape shape; + psEllipseAxes axes; + + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(psf, false); + + psImage *image = readout->image; + PS_ASSERT_PTR_NON_NULL(image, false); + + // XXX a minor hack: measure the values for a grid of points across the image, determine mean, UQ, LQ, etc: + psVector *fwhmMajor = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *fwhmMinor = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *psfExtra1 = psVectorAllocEmpty (100, PS_DATA_F32); + psVector *psfExtra2 = psVectorAllocEmpty (100, PS_DATA_F32); + + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + + float xc = source->peak->xf; + float yc = source->peak->yf; + + // create modelPSF from this model + pmModel *modelPSF = pmModelFromPSFforXY (psf, xc, yc, 1.0); + if (!modelPSF) { + fprintf (stderr, "?"); + continue; + } + + // get the model full-width at half-max + float FWHM_MAJOR = 2*modelPSF->class->modelRadius (modelPSF->params, 0.5); + + // XXX make sure this is consistent with the re-definition of PM_PAR_SXX + shape.sx = modelPSF->params->data.F32[PM_PAR_SXX]; + shape.sy = modelPSF->params->data.F32[PM_PAR_SYY]; + shape.sxy = modelPSF->params->data.F32[PM_PAR_SXY]; + axes = psEllipseShapeToAxes (shape, 20.0); + + float FWHM_MINOR = FWHM_MAJOR * (axes.minor / axes.major); + if (!isfinite(FWHM_MAJOR) || !isfinite(FWHM_MINOR)) { + fprintf (stderr, "!"); + psFree (modelPSF); + continue; + } + psVectorAppend (fwhmMajor, FWHM_MAJOR); + psVectorAppend (fwhmMinor, FWHM_MINOR); + + if (modelPSF->params->n > 7) { + psVectorAppend (psfExtra1, modelPSF->params->data.F32[7]); + } + if (modelPSF->params->n > 8) { + psVectorAppend (psfExtra2, modelPSF->params->data.F32[8]); + } + psFree (modelPSF); + + // float fwhmtest = pmPSFtoFWHM(psf, xc, yc); + // fprintf (stderr, "fwhm: %f, %f : %f\n", FWHM_MAJOR, FWHM_MINOR, fwhmtest); + } + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_SAMPLE_QUARTILE); + if (!psVectorStats (stats, fwhmMajor, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MAJOR"); + goto escape; + } + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MAJ", PS_META_REPLACE, "PSF FWHM Major axis (mean)", stats->sampleMean); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_SG", PS_META_REPLACE, "PSF FWHM Major axis (sigma)", stats->sampleStdev); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_LQ", PS_META_REPLACE, "PSF FWHM Major axis (lower quartile)", stats->sampleLQ); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_UQ", PS_META_REPLACE, "PSF FWHM Major axis (upper quartile)", stats->sampleUQ); + + float fwhmMaj = stats->sampleMean; // FWHM on major axis + + if (!psVectorStats (stats, fwhmMinor, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MINOR"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MIN", PS_META_REPLACE, "PSF FWHM Minor axis (mean)", stats->sampleMean); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_SG", PS_META_REPLACE, "PSF FWHM Minor axis (sigma)", stats->sampleStdev); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_LQ", PS_META_REPLACE, "PSF FWHM Minor axis (lower quartile)", stats->sampleLQ); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_UQ", PS_META_REPLACE, "PSF FWHM Minor axis (upper quartile)", stats->sampleUQ); + + float fwhmMin = stats->sampleMean; // FWHM on minor axis + + if (readout->parent) { + + // we now have 2 possible measurements of the seeing : the PSF model based version and + // the Moments based value we need to define a definitive "CHIP.SEEING" value. This + // value is used by the PSF-matching programs (ppSub and ppStack). The moments-based + // value is probably more representative of the value needed by those tools (it is also + // more stable?) + bool status = false; + float fwhmMajorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1"); + float fwhmMinorMoments = psMetadataLookupF32(&status, readout->analysis, "IQ_FW1"); + + pmChip *chip = readout->parent->parent; // Parent chip + psAssert(chip, "Cell should be attached to a chip."); + psMetadataItem *item = psMetadataLookup(chip->concepts, "CHIP.SEEING"); // Item with chip + item->data.F32 = 0.5 * (fwhmMajorMoments + fwhmMinorMoments); + + psLogMsg ("psphot", PS_LOG_DETAIL, "fwhm (psf): %f,%f (moments): %f,%f", fwhmMaj, fwhmMin, fwhmMajorMoments, fwhmMinorMoments); + } + + if (psfExtra1->n) { + if (!psVectorStats (stats, psfExtra1, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 1"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT1", PS_META_REPLACE, "PSF extra param 1", stats->sampleMean); + psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 1: %f +/- %f", stats->sampleMean, stats->sampleStdev); + } + + if (psfExtra2->n) { + if (!psVectorStats (stats, psfExtra2, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for PSF EXTRA 2"); + goto escape; + } + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PSF_EXT2", PS_META_REPLACE, "PSF extra param 2", stats->sampleMean); + psLogMsg ("psphot", PS_LOG_DETAIL, "PSF extra parameter 2: %f +/- %f", stats->sampleMean, stats->sampleStdev); + } + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE", PS_META_REPLACE, "PSF angle", axes.theta); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", psf->nPSFstars); + + // psLogMsg ("psphot", PS_LOG_DETAIL, "PSF angle: %f, nstars: %d", axes.theta, psf->nPSFstars); + + char *psfModelName = pmModelClassGetName(psf->type); + psMetadataAddStr(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "PSF Model Name", psfModelName); + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSF_OK", PS_META_REPLACE, "Valid PSF Model?", true); + + int nParams = pmModelClassParameterCount(psf->type); + psMetadataAddS32(readout->analysis, PS_LIST_TAIL, "PSF_NPAR", PS_META_REPLACE, "Number of PSF parameters", nParams); + + psFree (fwhmMajor); + psFree (fwhmMinor); + psFree (psfExtra1); + psFree (psfExtra2); + psFree (stats); + return true; + +escape: + psFree (fwhmMajor); + psFree (fwhmMinor); + psFree (psfExtra1); + psFree (psfExtra2); + psFree (stats); + return false; +} + +// determine approximate PSF shape parameters based on the moments +bool psphotMomentsStats (pmReadout *readout, psArray *sources) { + + // without the PSF model, we can only rely on the source->moments + // as a measure of the FWHM values + + int FWHM_N = 0; + double FWHM_X = 0.0; + double FWHM_Y = 0.0; + double FWHM_T = 0.0; + + psEllipseMoments moments; + psEllipseAxes axes; + + PS_ASSERT_PTR_NON_NULL(readout, false); + PS_ASSERT_PTR_NON_NULL(sources, false); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!source) continue; + if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + if (!source->moments) { + continue; + } + + moments.x2 = source->moments->Mxx; + moments.y2 = source->moments->Myy; + moments.xy = source->moments->Mxy; + + // limit axis ratio < 20.0 + axes = psEllipseMomentsToAxes (moments, 20.0); + + FWHM_X += axes.major * 2.35; + FWHM_Y += axes.minor * 2.35; + FWHM_T += axes.theta; + FWHM_N ++; + } + + FWHM_X /= FWHM_N; + FWHM_Y /= FWHM_N; + FWHM_T /= FWHM_N; + + if (readout->parent) { + pmChip *chip = readout->parent->parent; // Parent chip + psAssert(chip, "Cell should be attached to a chip."); + psMetadataItem *item = psMetadataLookup(chip->concepts, "CHIP.SEEING"); // Item with chip + item->data.F32 = 0.5 * (FWHM_X + FWHM_Y); + } + + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MAJ", PS_META_REPLACE, "PSF FWHM Major axis (mean)", FWHM_X); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_SG", PS_META_REPLACE, "PSF FWHM Major axis (sigma)", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_LQ", PS_META_REPLACE, "PSF FWHM Major axis (lower quartile)", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MJ_UQ", PS_META_REPLACE, "PSF FWHM Major axis (upper quartile)", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FWHM_MIN", PS_META_REPLACE, "PSF FWHM Minor axis (mean)", FWHM_Y); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_SG", PS_META_REPLACE, "PSF FWHM Minor axis (sigma)", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_LQ", PS_META_REPLACE, "PSF FWHM Minor axis (lower quartile)", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "FW_MN_UQ", PS_META_REPLACE, "PSF FWHM Minor axis (upper quartile)", 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "ANGLE", PS_META_REPLACE, "PSF angle", FWHM_T); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "NPSFSTAR", PS_META_REPLACE, "Number of stars used to make PSF", 0); + psMetadataAddStr(readout->analysis, PS_LIST_TAIL, "PSFMODEL", PS_META_REPLACE, "PSF Model Name", "NONE"); + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSF_OK", PS_META_REPLACE, "Valid PSF Model?", false); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCleanup.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCleanup.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCleanup.c (revision 41083) @@ -0,0 +1,64 @@ +# include "psphotStandAlone.h" + +void psphotCleanup (pmConfig *config) { + + // Dump configuration if requested + bool status; + psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG"); + if (dump_file) { + (void)pmConfigCamerasCull(config,NULL); + (void)pmConfigRecipesCull(config,NULL); + pmConfigDump(config, dump_file); + } + + // if the program has stats write it out + psphotStatsFile *statsFile = psphotStatsFileGet(); + if (statsFile) { + psphotStatsFileSave(config, statsFile); + } + + psFree (config); + + psphotVisualClose(); + psMemCheckCorruption (stderr, true); + pmModelClassCleanup (); + pmSourceFitSetDone (); + pmConceptsDone (); + pmConfigDone (); + pmVisualCleanup (); + psLibFinalize(); +#if (PS_TRACE_ON) + // don't display memory leaks unless trace is on. + // fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "psphot"); + fprintf (stderr, "found %d leaks at %s\n", psMemCheckLeaks2 (0, NULL, stdout, false, 500), "psphot"); +#endif + return; +} + +psExit psphotGetExitStatus (void) { + + // gcc -Wswitch complains here if err is declared as type psErrorCode + // the collection of ps*ErrorCode values are enums defined separately for + // each module (psphot, pswarp, etc). the lowest type, psErrorCode is only the base set and does + // not include the possible psphot values + + // for now, to get around this, we just use an int for the switch + + // psErrorCode err = psErrorCodeLast (); + int err = psErrorCodeLast (); + switch (err) { + case PS_ERR_NONE: + return PS_EXIT_SUCCESS; + case PSPHOT_ERR_SYS: + return PS_EXIT_SYS_ERROR; + case PSPHOT_ERR_CONFIG: + return PS_EXIT_CONFIG_ERROR; + case PSPHOT_ERR_PROG: + return PS_EXIT_PROG_ERROR; + case PSPHOT_ERR_DATA: + return PS_EXIT_DATA_ERROR; + default: + return PS_EXIT_UNKNOWN_ERROR; + } + return PS_EXIT_UNKNOWN_ERROR; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCullPeaks.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCullPeaks.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotCullPeaks.c (revision 41083) @@ -0,0 +1,119 @@ +# include "psphotInternal.h" + +/* + * Cull a set of peaks contained in a psArray of pmFootprints + */ +psErrorCode +psphotCullPeaks(const pmReadout *readout, // the image wherein lives the footprint + const pmReadout *signifR, // smoothed image and significance + const psMetadata *recipe, + psArray *footprints) { // array of pmFootprints + + bool status = false; + + float nsigma_delta = psMetadataLookupF32(&status, recipe, "FOOTPRINT_CULL_NSIGMA_DELTA"); + if (!status) { + nsigma_delta = 0; // min. + } + float nsigma_min = psMetadataLookupF32(&status, recipe, "FOOTPRINT_CULL_NSIGMA_MIN"); + if (!status) { + nsigma_min = 0; + } + float fPadding = psMetadataLookupF32(&status, recipe, "FOOTPRINT_CULL_NSIGMA_PAD"); + if (!status) { + fPadding = 0; + } + const float skyStdev = psMetadataLookupF32(&status, readout->analysis, "SKY_STDEV"); + psAssert (status, "cannot find SKY_STDEV in readout->analysis"); + + // the sky has been smoothed, so we need to scale down the raw sky stdev by this amount: + float scaleFactor = psMetadataLookupF32(&status, readout->analysis, "SIGNIFICANCE_SCALE_FACTOR"); + if (!status) scaleFactor = 1.0; + + const float MIN_THRESHOLD = nsigma_min*skyStdev / sqrt(scaleFactor); + + // for saturated stars, we should be somewhat more agressive about culling: instead of + // letting the height of the intervening col (saddle point) be set by the S/N of the image + // pixel, it should be set to a fraction of the saturation value. example for a + // near-saturation pixel: + // flux = 40k, sigma = 200 + // nsigma_delta = 4.0, nsigma_min = 1.0, fPadding = 0.01, + // fStdev = 0.005, stdev_pad = 0.011*40k = 400 + // threshold = 40k - 4*400 = 38400 + // this gives too tight of a tolerance on the bright stars + // in practice, we should make the threshold much lower. + // below I am using 0.05 * saturation (eg, 2000 DN above sky in a GPC1 image) + + float SATURATION = NAN; + + // do not completely trust the values in the header... + float CELL_SATURATION = psMetadataLookupF32 (&status, readout->parent->concepts, "CELL.SATURATION"); + float MIN_SATURATION = psMetadataLookupF32 (&status, recipe, "DEBLEND_MIN_SATURATION"); + if (!status || !isfinite(MIN_SATURATION)) { + MIN_SATURATION = 40000.0; + } + if (!isfinite(CELL_SATURATION)) { + SATURATION = MIN_SATURATION; + } else { + SATURATION = PS_MAX(MIN_SATURATION, CELL_SATURATION); + } + float SAT_TEST_LEVEL = 0.50*SATURATION; + float SAT_THRESHOLD = 0.05*SATURATION; + +# if (PM_PEAKS_CULL_WITH_SMOOTHED_IMAGE) + psLogMsg ("psphot", PS_LOG_INFO, "Culling peaks from footprints using the smoothed image"); + bool setNaNsToZero = psMetadataLookupF32 (&status, recipe, "FOOTPRINT_SET_NANS_TO_ZERO"); + if (setNaNsToZero) { + // Set NaN pixels to zero so that they do not considered in the footprint analysis + // XXX: Currently the caller does nothing further with the significance readout so + // modifiying the image does no harm + // XXX: Note: signifR->weight is not used by pmFootprintCullPeaks so we don't + // need to touch it + psImageClipNaN(signifR->image, 0); + } + + +# else + psLogMsg ("psphot", PS_LOG_INFO, "Culling peaks from footprints using the raw (unsmoothed) image"); +# endif + + for (int i = 0; i < footprints->n; i++) { + pmFootprint *fp = footprints->data[i]; + if (fp->peaks == NULL) continue; + if (fp->peaks->n == 0) continue; + + if (fp->npix > 30000) { + psLogMsg("psphot.cull.footprints", PS_LOG_WARN, "big footprint: %f %f to %f %f (%d pix)\n", fp->bbox.x0, fp->bbox.y0, fp->bbox.x1, fp->bbox.y1, fp->npix); + } + psTimerStart ("psphot.cull.footprints"); + + pmPeak *brightPeak = fp->peaks->data[0]; + float max_threshold = SAT_TEST_LEVEL; + if (brightPeak->rawFlux > SAT_TEST_LEVEL) { + max_threshold = SAT_THRESHOLD; + brightPeak->type = PM_PEAK_SUSPECT_SATURATION; + } + +# if (PM_PEAKS_CULL_WITH_SMOOTHED_IMAGE) + // New (post r30869) style of culling using the smoothed image and variance (S/N) + // if we cull using the significance image, then the definition of variance is different (thus the bool in arg 8) + if (pmFootprintCullPeaks(signifR->image, signifR->variance, fp, nsigma_delta, fPadding, MIN_THRESHOLD, max_threshold, true) != PS_ERR_NONE) { + return psError(PS_ERR_UNKNOWN, false, "Culling pmFootprint %d", fp->id); + } +# else + // Old (pre r30869) style of culling using the raw image and variance + if (pmFootprintCullPeaks(readout->image, readout->variance, fp, nsigma_delta, fPadding, MIN_THRESHOLD, max_threshold, false) != PS_ERR_NONE) { + return psError(PS_ERR_UNKNOWN, false, "Culling pmFootprint %d", fp->id); + } +# endif + + + float dtime = psTimerMark ("psphot.cull.footprints"); + if (dtime > 1.0) { + psLogMsg("psphot.cull.footprints", PS_LOG_WARN, "slow cull for %d (%f sec)\n", i, dtime); + } + } + return PS_ERR_NONE; +} + +// psLogMsg ("psphot", PS_LOG_INFO, "%ld peaks, %ld total footprints: %f sec\n", detections->peaks->n, detections->footprints->n, Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDeblendSatstars.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDeblendSatstars.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDeblendSatstars.c (revision 41083) @@ -0,0 +1,1109 @@ +# include "psphotInternal.h" + +typedef struct { + float min; + float max; + float lower20; + float upper20; + int Npts; +} QuickStats; + +bool psImageQuickStats (QuickStats *stats, psImage *image, psRegion region); +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X); +bool psphotVisualScaleImage (int kapaFD, psImage *inImage, psImage *inMask, const char *name, float factor, int channel); + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotDeblendSatstars (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = false; + + int num = psphotFileruleCount(config, filerule); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full extended source non-linear fits? + if (!psMetadataLookupBool (&status, recipe, "SUBTRACT_SATSTAR_PROFILE")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping extended source fits\n"); + return true; + } + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotDeblendSatstarsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on saturated star deblend analysis for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +/** this function does 3 things: + + 1) choose likely saturated stars + - mode | SATSTAR + - if moments->Peak > 0.5*SATURATION, examine region of peak pixels (ensure we go down to 0.1*SAT) for SAT mask + + 2) adjust the window for saturated objects + + 3) measure the radial profile + + 4) subtract the radial profile + + There is also support code for calculation of magnitudes and add/subtract the profile (a la pmSourceOp) + + TBD: + + * recenter the profile (based on cross-correlation / convolution with 1D profile) + **/ + +bool psphotDeblendSatstarsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int fileIndex, psMetadata *recipe) { + + int N; + pmSource *source; + bool status; + + psTimerStart ("psphot.deblend.sat"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, fileIndex); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping satstar blend"); + return true; + } + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskSat = psMetadataLookupImageMask(&status, recipe, "MASK.SAT"); // Mask value for bad pixels + assert (maskSat); + + float SATURATION = NAN; + { + // XXX do we need to set this differently from the value used to mark saturated pixels? + pmCell *cell = readout->parent; + + // do not completely trust the values in the header... + float CELL_SATURATION = psMetadataLookupF32 (&status, cell->concepts, "CELL.SATURATION"); + float MIN_SATURATION = psMetadataLookupF32 (&status, recipe, "DEBLEND_MIN_SATURATION"); + if (!status || !isfinite(MIN_SATURATION)) { + MIN_SATURATION = 40000.0; + } + if (!isfinite(CELL_SATURATION)) { + SATURATION = MIN_SATURATION; + } else { + SATURATION = PS_MAX(MIN_SATURATION, CELL_SATURATION); + } + } + + // source analysis is done in peak order (brightest first) + // we use an index for this so the spatial sorting is kept + psVector *SN = psVectorAlloc (sources->n, PS_DATA_F32); + for (int i = 0; i < SN->n; i++) { + source = sources->data[i]; + SN->data.F32[i] = source->moments->SN; + } + psVector *index = psVectorSortIndex (NULL, SN); + // this results in an index of increasing SN + + // psphotVisualPlotRadialProfiles (recipe, sources, PM_SOURCE_MODE_SATSTAR); + + int BIG_RADIUS = 250; + int BIG_SIGMA = BIG_RADIUS / 4.0; + + // int display = psphotKapaChannel (1); + // psphotVisualScaleImage (display, (psImage *) source->pixels->parent, NULL, "image", 1.0, 0); + + int Nsatstar = 0; + + // examine sources in decreasing SN order + for (int i = sources->n - 1; i >= 0; i--) { + N = index->data.U32[i]; + source = sources->data[N]; + + if (!source->pixels) continue; +#ifndef DEBLEND_PASS2_SOURCES + // XXX: Need to define this if we want to rerun the satstar stuff again on the pass2 sources + // This is needed for psphotStack in -updatemode + if ((source->mode2 & PM_SOURCE_MODE2_PASS1_SRC) == 0) continue; +#endif + + bool isSat = source->mode & PM_SOURCE_MODE_SATSTAR; + + // for fairly bright stars, check the pixels near the peak again + if (source->moments->Peak > 0.5*SATURATION) { + // pmSourceRoughClass does this analysis, but uses a small (5x5) window. + // here we use a larger window since stacks can have some funny features + psRegion inner; + QuickStats stats; + // grow out the search radius until we have lower20 < 0.1*SATURATION + for (int radius = 2; radius < 30; radius ++) { + inner = psRegionForSquare (source->peak->x, source->peak->y, radius); + inner = psRegionForImage (source->maskView, inner); + psImageQuickStats (&stats, readout->image, inner); + if ((stats.Npts > 1) && (stats.lower20 < 0.1*SATURATION)) break; + } + int Nsatpix = psImageCountPixelMask (source->maskView, inner, maskSat); + // fprintf (stderr, "test object: %d,%d : %d vs %d : %f - %f - %f - %f\n", + // source->peak->x, source->peak->y, Nsatpix, stats.Npts, + // stats.min, stats.lower20, stats.upper20, stats.max); + if (Nsatpix > 1) isSat = true; + } + if (!isSat) continue; + + // For saturated stars, choose a much larger box NOTE this is slightly sleazy, but + // only slightly: pmSourceRedefinePixels uses the readout to pass the pointers to + // the parent image data. I guess the API could be simplified: we could recover + // this from the source in the function + + pmReadout tmpReadout; + tmpReadout.image = (psImage *)source->pixels->parent; + tmpReadout.mask = (psImage *)source->maskView->parent; + tmpReadout.variance = (psImage *)source->variance->parent; + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + pmSourceRedefinePixels (source, &tmpReadout, source->peak->x, source->peak->y, BIG_RADIUS + 2); + + psTrace ("psphot", 4, "retrying moments for %d, %d\n", source->peak->x, source->peak->y); + status = pmSourceMoments (source, BIG_RADIUS, BIG_SIGMA, 0.0, 5.0, maskVal); + source->mode |= PM_SOURCE_MODE_BIG_RADIUS; + + if (!psphotSatstarProfileModel (source, maskVal)) continue; + + source->mode |= PM_SOURCE_MODE_SATSTAR; // yes, this source IS saturated + source->mode2 |= PM_SOURCE_MODE2_SATSTAR_PROFILE; // and we have in fact subtracted the profile + + Nsatstar ++; + + // XXX visualize, model, and subtract + // if (!psphotVisualRadialProfileSatstar (source, maskVal)) { + // break; + // } + + // generate radial profile, store on the source structure + } + // show the image after object have been subtracted + // psphotVisualScaleImage (display, (psImage *) source->pixels->parent, NULL, "image", 1.0, 1); + + psFree (SN); + psFree (index); + + psLogMsg ("psphot", PS_LOG_INFO, "deblend %d satstars: %f sec\n", Nsatstar, psTimerMark ("psphot.deblend.sat")); + return true; +} + +bool psphotDeblendSatstarsReadoutOld (pmConfig *config, const pmFPAview *view, const char *filerule, int fileIndex) { + + int N; + pmSource *source; + bool status; + + psTimerStart ("psphot.deblend.sat"); + + int Nblend = 0; + float SAT_MIN_RADIUS = 5.0; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, fileIndex); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping satstar blend"); + return true; + } + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + pmCell *cell = readout->parent; + + float SATURATION = NAN; + + // do not completely trust the values in the header... + float CELL_SATURATION = psMetadataLookupF32 (&status, cell->concepts, "CELL.SATURATION"); + float MIN_SATURATION = psMetadataLookupF32 (&status, recipe, "DEBLEND_MIN_SATURATION"); + if (!status || !isfinite(MIN_SATURATION)) { + MIN_SATURATION = 40000.0; + } + if (!isfinite(CELL_SATURATION)) { + SATURATION = MIN_SATURATION; + } else { + SATURATION = PS_MAX(MIN_SATURATION, CELL_SATURATION); + } + float SAT_TEST_LEVEL = 0.5*SATURATION; + + // we need sources spatially-sorted to find overlaps + sources = psArraySort (sources, pmSourceSortByY); + + // source analysis is done in peak order (brightest first) + // we use an index for this so the spatial sorting is kept + psVector *SN = psVectorAlloc (sources->n, PS_DATA_F32); + for (int i = 0; i < SN->n; i++) { + source = sources->data[i]; + SN->data.F32[i] = source->peak->rawFlux; + } + psVector *index = psVectorSortIndex (NULL, SN); + // this results in an index of increasing SN + + // examine sources in decreasing SN order + for (int i = sources->n - 1; i >= 0; i--) { + N = index->data.U32[i]; + source = sources->data[N]; + + // XXX filter? if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + if (source->mode & PM_SOURCE_MODE_BLEND) continue; + if (source->peak->rawFlux < SAT_TEST_LEVEL) continue; + + // save these for reference below + int xPeak = source->peak->x; + int yPeak = source->peak->y; + + // generate a basic contour (set of x,y coordinates at-or-below flux level) + psArray *contour = pmSourceContour (source->pixels, xPeak, yPeak, SAT_TEST_LEVEL); + if (contour == NULL) continue; + + // contour consists of a set of X,Y coords giving the boundary + psVector *xVec = contour->data[0]; + psVector *yVec = contour->data[1]; + if (xVec->n < 5) { + psFree(contour); + continue; + } + + // find the center of the contour (let's just use mid[x,y]) + int xMin = xVec->data.F32[0]; + int xMax = xVec->data.F32[0]; + int yMin = yVec->data.F32[0]; + int yMax = yVec->data.F32[0]; + for (int j = 0; j < xVec->n; j++) { + xMin = PS_MIN (xMin, xVec->data.F32[j]); + xMax = PS_MAX (xMax, xVec->data.F32[j]); + yMin = PS_MIN (yMin, yVec->data.F32[j]); + yMax = PS_MAX (yMax, yVec->data.F32[j]); + } + int xCenter = 0.5*(xMin + xMax); + int yCenter = 0.5*(yMin + yMax); + psFree (contour); + + psAssert (xCenter >= source->pixels->col0, "invalid shift in object center"); + psAssert (xCenter < source->pixels->col0 + source->pixels->numCols, "invalid shift in object center"); + psAssert (yCenter >= source->pixels->row0, "invalid shift in object center"); + psAssert (yCenter < source->pixels->row0 + source->pixels->numRows, "invalid shift in object center"); + + // reset the peak for this source to the value of the center pixel + source->peak->x = xCenter; + source->peak->y = yCenter; + source->peak->xf = xCenter; + source->peak->yf = yCenter; + + // temporary array for overlapping objects we find + psArray *overlap = psArrayAllocEmpty (100); + + // search backwards for overlapping sources + for (int j = N - 1; j >= 0; j--) { + pmSource *testSource = sources->data[j]; + if (testSource->peak->x < source->pixels->col0) continue; + if (testSource->peak->x >= source->pixels->col0 + source->pixels->numCols) continue; + if (testSource->peak->y < source->pixels->row0) break; + if (testSource->peak->y >= source->pixels->row0 + source->pixels->numRows) { + fprintf (stderr, "warning: invalid condition\n"); + continue; + } + psArrayAdd (overlap, 100, testSource); + } + + // search forwards for overlapping sources + for (int j = N + 1; j < sources->n; j++) { + pmSource *testSource = sources->data[j]; + if (testSource->peak->x < source->pixels->col0) continue; + if (testSource->peak->x >= source->pixels->col0 + source->pixels->numCols) continue; + if (testSource->peak->y < source->pixels->row0) { + fprintf (stderr, "warning: invalid condition\n"); + continue; + } + if (testSource->peak->y >= source->pixels->row0 + source->pixels->numRows) break; + psArrayAdd (overlap, 100, testSource); + } + + if (overlap->n == 0) { + psFree (overlap); + continue; + } + + // now find the contour which is at 0.5*SAT_TEST_LEVEL (xPeak, yPeak is valid high pixel) + contour = pmSourceContour (source->pixels, xPeak, yPeak, 0.5*SAT_TEST_LEVEL); + if (contour == NULL) { + psFree (overlap); + continue; + } + + // any peaks within this contour should be dropped (mark as blend, drop after loop is done) + // also drop any peaks which are too close to this peak + psVector *xv = contour->data[0]; + psVector *yv = contour->data[1]; + for (int k = 0; k < overlap->n; k++) { + pmSource *testSource = overlap->data[k]; + float radius = hypot((testSource->peak->x - xCenter), (testSource->peak->y - yCenter)); + if (radius < SAT_MIN_RADIUS) { + testSource->mode |= PM_SOURCE_MODE_BLEND; + Nblend ++; + continue; + } + for (int j = 0; j < xv->n; j+=2) { + if (fabs(yv->data.F32[j] - testSource->peak->y) > 0.5) continue; + if (xv->data.F32[j+0] > testSource->peak->x) break; + if (xv->data.F32[j+1] < testSource->peak->x) break; + testSource->mode |= PM_SOURCE_MODE_BLEND; + Nblend ++; + j = xv->n; // skip rest of contour + } + } + psFree (overlap); + psFree (contour); + } + + // drop the sources marked as BLEND + for (int i = 0; i < sources->n;) { + pmSource *source = sources->data[i]; + + if (!(source->mode & PM_SOURCE_MODE_BLEND)) { + i++; + continue; + } + + // shuffle the remaining sources forward + for (int j = i; j < sources->n - 1; j++) { + sources->data[j] = sources->data[j+1]; + } + psFree (source); + sources->n --; + } + + psFree (SN); + psFree (index); + + psLogMsg ("psphot", PS_LOG_INFO, "found %d satstar blend peaks, leaving %ld sources: %f sec\n", Nblend, sources->n, psTimerMark ("psphot.deblend.sat")); + return true; +} + +// create a model for the radial profile of a saturated star (is this actually more generic?) +bool psphotSatstarProfileModel (pmSource *source, psImageMaskType maskVal) { + + // XXX user define somewhere? + float Rmax = 320.0; + + // XXX is this ever the case?? + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + int nPts = source->pixels->numRows * source->pixels->numCols; + psVector *logR = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *flux = psVectorAllocEmpty (nPts, PS_TYPE_F32); + + int ng = 0; + + // choose the best center for this profile + float Xo = NAN; + float Yo = NAN; + + if (source->modelPSF) { + // XXX do we ever have a PSF model for a SATSTAR? + Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + } else { + Xo = source->moments->Mx; + Yo = source->moments->My; + } + + float Xc = Xo - source->pixels->col0 - 0.5; + float Yc = Yo - source->pixels->row0 - 0.5; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + if ((source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal)) continue; + // XXX do this faster by generating R^2 and returning 0.5*log10(R^2)? + logR->data.F32[ng] = log10(hypot (ix - Xc, iy - Yc)); + flux->data.F32[ng] = source->pixels->data.F32[iy][ix]; + ng++; + } + } + logR->n = ng; + flux->n = ng; + + // XXX do something sensible here if there are no pixels + + // measure the radial profile + psVector *logFmodel = NULL; + psVector *logRmodel = NULL; + psphotSatstarProfileCreate (source, &logRmodel, &logFmodel, logR, flux, Rmax); + + // XXX do something sensible here if the profile is crap + + // replace an existing profile + psFree (source->satstar); + source->satstar = pmSourceSatstarAlloc(); + source->satstar->Xo = Xo; + source->satstar->Yo = Yo; + source->satstar->Rmax = Rmax; + source->satstar->logFmodel = logFmodel; + source->satstar->logRmodel = logRmodel; + + // subtract the profile (false => subtract) + psphotSatstarProfileOp (source, maskVal, 1.0, 0, false); + + psFree (logR); + psFree (flux); + + return true; +} + +// Take logR + flux and generate radial bins logRmodelOut, logFmodelOut +bool psphotSatstarProfileCreate (pmSource *source, psVector **logRmodelOut, psVector **logFmodelOut, psVector *logR, psVector *flux, float Rmax) { + + // we have log(radius) & log(flux). find the median flux in log radial bins + + float logRmax = log10(Rmax); + float logRdel = 0.1; // XXX user-define? + + psVector *logRmodel = psVectorAlloc (1 + (int) (logRmax / logRdel), PS_TYPE_F32); + psVector *logFmodel = psVectorAlloc (1 + (int) (logRmax / logRdel), PS_TYPE_F32); + + pmSourceRadialProfileSortPair (logR, flux); + + psStats *fluxStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); + psVector *fluxVals = psVectorAllocEmpty (100, PS_TYPE_F32); + + int bin = 0; + for (int i = 0; i < logRmodel->n; i++) { + + // bin (i) has log radius range (i*logRdel : (i+1)*logRdel) + float lRmin = logRdel*(i + 0); + float lRmax = logRdel*(i + 1); + + // reset the flux vector + fluxVals->n = 0; + psStatsInit (fluxStats); + + while (bin < logR->n && logR->data.F32[bin] < lRmax) { + if (isfinite(flux->data.F32[bin])) { + psVectorAppend (fluxVals, flux->data.F32[bin]); + } + bin ++; + } + + // we have the set of fluxes for this bin; find the median values + + float Rmin = pow(10.0, lRmin); + float Rmax = pow(10.0, lRmax); + + float Rmean = (2.0/3.0) * (pow(Rmax, 3.0) - pow(Rmin, 3.0)) / (PS_SQR(Rmax) - PS_SQR(Rmin)); + logRmodel->data.F32[i] = log10(Rmean); + + float Area = M_PI * (PS_SQR(Rmax) - PS_SQR(Rmin)); + if (fluxVals->n < 0.25*Area) { + logFmodel->data.F32[i] = NAN; + continue; + } + + psVectorStats (fluxStats, fluxVals, NULL, NULL, 0); + if (fluxStats->robustMedian > 0.0) { + logFmodel->data.F32[i] = log10(fluxStats->robustMedian); + } else { + logFmodel->data.F32[i] = -3.0; + } + // fprintf (stderr, "R: %f, F: %f +/- %f\n", Rmean, fluxStats->robustMedian, fluxStats->robustStdev); + } + + // now how do i use this to subtract a model?? + *logRmodelOut = logRmodel; + *logFmodelOut = logFmodel; + + // need to free stuff + psFree (fluxStats); + psFree (fluxVals); + + return true; +} + +bool psphotSatstarProfileOp (pmSource *source, psImageMaskType maskVal, float FACTOR, pmModelOpMode mode, bool add) { + + int alt; + float logRdel = 0.1; + + pmSourceSatstar *satstar = source->satstar ? source->satstar : (source->parent ? source->parent->satstar : NULL); + psAssert (satstar, "null satstar"); + float Xc = satstar->Xo - source->pixels->col0 - 0.5; + float Yc = satstar->Yo - source->pixels->row0 - 0.5; + psVector *logRmodel = satstar->logRmodel; + psVector *logFmodel = satstar->logFmodel; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + if ((source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal)) continue; + + float radius = hypot (ix - Xc, iy - Yc) ; + float logR = log10(radius); + + int bin = (int)(logR / logRdel); + + // we are going to interpolate if possible, or extrapolate if necessary + if (bin >= logRmodel->n - 1) bin = logRmodel->n - 1; + if (bin < 0) bin = 0; + + // interpolate to the current radial position + // XXX BIG HACK : skip nan bins for now + + float logF0 = logFmodel->data.F32[bin]; + float logR0 = logRmodel->data.F32[bin]; + if (!isfinite(logF0)) continue; + + // interpolate between closest two bins if possible, extrapolate on ends + if (logR < logR0) { + alt = (bin > 0) ? bin - 1 : bin + 1; + } else { + alt = (bin < logRmodel->n - 1) ? bin + 1 : bin - 1; + } + + float logF1 = logFmodel->data.F32[alt]; + float logR1 = logRmodel->data.F32[alt]; + if (!isfinite(logF1)) continue; + + // XXX use linear flux, not logFlux + float logF = InterpolateValues (logR0, logF0, logR1, logF1, logR); + float flux = pow (10.0, logF); + + if (mode & PM_MODEL_OP_NOISE) { + if (add) { + source->variance->data.F32[iy][ix] += FACTOR * flux; + } else { + source->variance->data.F32[iy][ix] -= FACTOR * flux; + } + } else { + if (add) { + source->pixels->data.F32[iy][ix] += flux; + } else { + source->pixels->data.F32[iy][ix] -= flux; + } + } + } + } + return true; +} + +// Take logR + flux and generate radial bins logRmodelOut, logFmodelOut +bool psphotSatstarPhotometry (pmSource *source) { + + if (!source->satstar) return false; + + psVector *logRmodel = source->satstar->logRmodel; + psVector *logFmodel = source->satstar->logFmodel; + + float fluxTotal = 0.0; + float logRdel = 0.1; // XXX user-define (or carry in satstar) + + // integrate flux in radial bins + for (int i = 0; i < logRmodel->n; i++) { + // just add up the mean flux in each bin and multiply by the area of the bin + + float logF = logFmodel->data.F32[i]; + if (!isfinite(logF)) continue; + float flux = pow(10.0, logF); // this is the mean flux per pixel (ie, surface brightness) + + // bin (i) has log radius range (i*logRdel : (i+1)*logRdel) + float lRmin = logRdel*(i + 0); + float lRmax = logRdel*(i + 1); + + float Rmin = pow(10.0, lRmin); + float Rmax = pow(10.0, lRmax); + + float Area = M_PI * (PS_SQR(Rmax) - PS_SQR(Rmin)); + + fluxTotal += flux * Area; + } + + source->psfMag = -2.5*log10(fluxTotal); + source->psfMagErr = 0.05; + + // XXX I have no idea of a realistic error on the photometry here. the bottom line is that + // the error is totally dominated by the loss of charge due to saturation. I am not + // modeling the profile in any real detail other than to follow the radial bins. + + source->extMag = NAN; + source->apMag = NAN; + source->apMagRaw = NAN; + source->apFlux = fluxTotal; + source->apFluxErr = 0.05*fluxTotal; + + return true; +} + +# if (0) +static bool skipDisplay = false; +bool psphotVisualRadialProfileSatstar (pmSource *source, psImageMaskType maskVal) { + + int kapaImage = -1; + int kapaGraph = -1; + Graphdata graphdata; + + if (!pmVisualTestLevel("psphot.satstar", 3)) { + skipDisplay = true; + } + + float Rmax = 320.0; + + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + int nPts = source->pixels->numRows * source->pixels->numCols; + psVector *rg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *Rg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *fg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *Fg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *rb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *Rb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *fb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + + int ng = 0; + int nb = 0; + + float Xo = NAN; + float Yo = NAN; + + if (source->modelPSF) { + Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS] - source->pixels->col0; + Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS] - source->pixels->row0; + } else { + Xo = source->moments->Mx - source->pixels->col0; + Yo = source->moments->My - source->pixels->row0; + } + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + if ((source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal)) { + rb->data.F32[nb] = hypot (ix + 0.5 - Xo, iy + 0.5 - Yo) ; + // rb->data.F32[nb] = hypot (ix - Xo, iy - Yo) ; + Rb->data.F32[nb] = log10(rb->data.F32[nb]); + fb->data.F32[nb] = log10(source->pixels->data.F32[iy][ix]); + nb++; + } else { + rg->data.F32[ng] = hypot (ix + 0.5 - Xo, iy + 0.5 - Yo) ; + // rg->data.F32[ng] = hypot (ix - Xo, iy - Yo) ; + Rg->data.F32[ng] = log10(rg->data.F32[ng]); + Fg->data.F32[ng] = source->pixels->data.F32[iy][ix]; + fg->data.F32[ng] = log10(Fg->data.F32[ng]); + ng++; + } + } + } + rg->n = ng; + Rg->n = ng; + fg->n = ng; + Fg->n = ng; + rb->n = nb; + Rb->n = nb; + fb->n = nb; + + if (!skipDisplay) { + kapaImage = psphotKapaChannel (1); + if (kapaImage == -1) return false; + + kapaGraph = psphotKapaChannel (2); + if (kapaGraph == -1) return false; + + KapaSection section; // put the positive profile in one and the residuals in another? + + // first section : mag vs CR nSigma + section.dx = 1.0; + section.dy = 0.5; + section.x = 0.0; + section.y = 0.0; + section.bg = -1; + section.name = NULL; + psStringAppend (§ion.name, "linlog"); + KapaSetSection (kapaGraph, §ion); + psFree (section.name); + + // first section : mag vs CR nSigma + section.dx = 1.0; + section.dy = 0.5; + section.x = 0.0; + section.y = 0.5; + section.bg = -1; + section.name = NULL; + psStringAppend (§ion.name, "loglog"); + KapaSetSection (kapaGraph, §ion); + psFree (section.name); + + KapaInitGraph (&graphdata); + + // ** linlog ** + KapaSelectSection (kapaGraph, "linlog"); + + // examine sources to set data range + graphdata.xmin = -0.05; + graphdata.xmax = Rmax + 0.05; + graphdata.ymin = -0.05; + graphdata.ymax = +8.05; + KapaSetLimits (kapaGraph, &graphdata); + + KapaSetFont (kapaGraph, "helvetica", 14); + KapaBox (kapaGraph, &graphdata); + KapaSendLabel (kapaGraph, "radius (pixels)", KAPA_LABEL_XM); + KapaSendLabel (kapaGraph, "log flux (counts)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (kapaGraph, ng, &graphdata); + KapaPlotVector (kapaGraph, ng, rg->data.F32, "x"); + KapaPlotVector (kapaGraph, ng, fg->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (kapaGraph, nb, &graphdata); + KapaPlotVector (kapaGraph, nb, rb->data.F32, "x"); + KapaPlotVector (kapaGraph, nb, fb->data.F32, "y"); + + // ** loglog ** + KapaSelectSection (kapaGraph, "loglog"); + + // examine sources to set data range + graphdata.xmin = -1.51; + graphdata.xmax = log10(Rmax) + 0.02; + graphdata.ymin = -5.05; + graphdata.ymax = +8.05; + graphdata.color = KapaColorByName ("black"); + KapaSetLimits (kapaGraph, &graphdata); + + KapaSetFont (kapaGraph, "helvetica", 14); + KapaBox (kapaGraph, &graphdata); + KapaSendLabel (kapaGraph, "log radius (pixels)", KAPA_LABEL_XM); + KapaSendLabel (kapaGraph, "log flux (counts)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (kapaGraph, ng, &graphdata); + KapaPlotVector (kapaGraph, ng, Rg->data.F32, "x"); + KapaPlotVector (kapaGraph, ng, fg->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (kapaGraph, nb, &graphdata); + KapaPlotVector (kapaGraph, nb, Rb->data.F32, "x"); + KapaPlotVector (kapaGraph, nb, fb->data.F32, "y"); + } + + // measure the radial profile + psVector *logFmodel = NULL; + psVector *logRmodel = NULL; + psphotTestRadialModel (source, &logRmodel, &logFmodel, Rg, Fg, Rmax); + + if (!skipDisplay) { + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 2; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (kapaGraph, logRmodel->n, &graphdata); + KapaPlotVector (kapaGraph, logRmodel->n, logRmodel->data.F32, "x"); + KapaPlotVector (kapaGraph, logRmodel->n, logFmodel->data.F32, "y"); + } + + // subtract the model from the images + psphotTestRadialModelSub (source, logRmodel, logFmodel, Xo, Yo, Rmax, maskVal); + + psFree (logRmodel); + psFree (logFmodel); + + if (!skipDisplay && source->modelPSF) { + // generate model profiles (major and minor axis): + // create a model with theta = 0.0 so major and minor axes are equiv to x and y: + psEllipseShape rawShape, rotShape; + + rawShape.sx = source->modelPSF->params->data.F32[PM_PAR_SXX] / M_SQRT2; + rawShape.sy = source->modelPSF->params->data.F32[PM_PAR_SYY] / M_SQRT2; + rawShape.sxy = source->modelPSF->params->data.F32[PM_PAR_SXY]; + + psEllipseAxes axes = psEllipseShapeToAxes (rawShape, 20.0); + + axes.theta = 0.0; + + rotShape = psEllipseAxesToShape (axes); + + psVector *params = psVectorAlloc(source->modelPSF->params->n, PS_TYPE_F32); + for (int i = 0; i < source->modelPSF->params->n; i++) { + params->data.F32[i] = source->modelPSF->params->data.F32[i]; + } + params->data.F32[PM_PAR_SXX] = rotShape.sx * M_SQRT2; + params->data.F32[PM_PAR_SYY] = rotShape.sy * M_SQRT2; + params->data.F32[PM_PAR_SXY] = rotShape.sxy; + params->data.F32[PM_PAR_XPOS] = 0.0; + params->data.F32[PM_PAR_YPOS] = 0.0; + + psVector *rmod = psVectorAlloc(Rmax*10, PS_TYPE_F32); + psVector *fmaj = psVectorAlloc(Rmax*10, PS_TYPE_F32); + psVector *fmin = psVectorAlloc(Rmax*10, PS_TYPE_F32); + + psVector *coord = psVectorAlloc(2, PS_TYPE_F32); + + float r = 0.0; + for (int i = 0; i < rmod->n; i++) { + r = i*0.1; + rmod->data.F32[i] = r; + + coord->data.F32[1] = r; + coord->data.F32[0] = 0.0; + fmaj->data.F32[i] = log10(source->modelPSF->modelFunc (NULL, params, coord)); + + coord->data.F32[0] = r; + coord->data.F32[1] = 0.0; + fmin->data.F32[i] = log10(source->modelPSF->modelFunc (NULL, params, coord)); + } + psFree (coord); + psFree (params); + + float FWHM_MAJOR = 2.0*source->modelPSF->modelRadius (source->modelPSF->params, 0.5*source->modelPSF->params->data.F32[PM_PAR_I0]); + float FWHM_MINOR = FWHM_MAJOR * (axes.minor / axes.major); + if (FWHM_MAJOR < FWHM_MINOR) PS_SWAP (FWHM_MAJOR, FWHM_MINOR); + + psEllipseMoments emoments; + emoments.x2 = source->moments->Mxx; + emoments.xy = source->moments->Mxy; + emoments.y2 = source->moments->Myy; + axes = psEllipseMomentsToAxes (emoments, 20.0); + float MOMENTS_MAJOR = 2.355*axes.major; + float MOMENTS_MINOR = 2.355*axes.minor; + + float logHM = log10(0.5*source->modelPSF->params->data.F32[PM_PAR_I0]); + + // reset source Add/Sub state to recorded + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + // ** linlog ** + KapaSelectSection (kapaGraph, "linlog"); + KapaGetGraphData (kapaGraph, &graphdata); + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (kapaGraph, rmod->n, &graphdata); + KapaPlotVector (kapaGraph, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (kapaGraph, rmod->n, fmin->data.F32, "y"); + plotline (kapaGraph, &graphdata, graphdata.xmin, logHM, graphdata.xmax, logHM); + plotline (kapaGraph, &graphdata, 0.5*FWHM_MINOR, graphdata.ymin, 0.5*FWHM_MINOR, graphdata.ymax); + graphdata.ltype = 1; + plotline (kapaGraph, &graphdata, 0.5*MOMENTS_MINOR, graphdata.ymin, 0.5*MOMENTS_MINOR, graphdata.ymax); + graphdata.ltype = 0; + + graphdata.color = KapaColorByName ("green"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (kapaGraph, rmod->n, &graphdata); + KapaPlotVector (kapaGraph, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (kapaGraph, rmod->n, fmaj->data.F32, "y"); + plotline (kapaGraph, &graphdata, 0.5*FWHM_MAJOR, graphdata.ymin, 0.5*FWHM_MAJOR, graphdata.ymax); + graphdata.ltype = 1; + plotline (kapaGraph, &graphdata, 0.5*MOMENTS_MAJOR, graphdata.ymin, 0.5*MOMENTS_MAJOR, graphdata.ymax); + graphdata.ltype = 0; + + for (int i = 0; i < rmod->n; i++) { + rmod->data.F32[i] = log10(rmod->data.F32[i]); + } + + // ** loglog ** + KapaSelectSection (kapaGraph, "loglog"); + KapaGetGraphData (kapaGraph, &graphdata); + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (kapaGraph, rmod->n, &graphdata); + KapaPlotVector (kapaGraph, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (kapaGraph, rmod->n, fmin->data.F32, "y"); + + graphdata.color = KapaColorByName ("green"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (kapaGraph, rmod->n, &graphdata); + KapaPlotVector (kapaGraph, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (kapaGraph, rmod->n, fmaj->data.F32, "y"); + + psFree (rmod); + psFree (fmin); + psFree (fmaj); + } + + if (!skipDisplay) { + KiiCenter (kapaImage, source->peak->xf, source->peak->yf, 1); + psphotVisualScaleImage (kapaImage, (psImage *) source->pixels->parent, NULL, "image", 1.0, 1); + KiiOverlay overlay; + overlay.x = source->peak->xf; + overlay.y = source->peak->yf; + overlay.dx = 5; + overlay.dy = 5; + overlay.angle = 0.0; + overlay.type = KiiOverlayTypeByName ("circle"); + KiiLoadOverlay (kapaImage, &overlay, 1, "red"); + overlay.x = source->moments->Mx; + overlay.y = source->moments->My; + overlay.dx = 8; + overlay.dy = 8; + overlay.angle = 0.0; + overlay.type = KiiOverlayTypeByName ("circle"); + KiiLoadOverlay (kapaImage, &overlay, 1, "blue"); + } + + psFree (rg); + psFree (Rg); + psFree (fg); + psFree (Fg); + psFree (rb); + psFree (Rb); + psFree (fb); + + if (skipDisplay) return true; + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[q]uit satstar? [e]rase and continue? [o]verplot and continue? [s]kip rest of stars? : "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + if (key[0] == 'e') { + KapaClearPlots (kapaGraph); + } + if (key[0] == 'q') { + return false; + } + if (key[0] == 's') { + skipDisplay = true; + } + return true; +} +# endif + +// only valid for F32 +bool psImageQuickStats (QuickStats *stats, psImage *image, psRegion region) { + + psAssert (image->type.type == PS_TYPE_F32, "unsupported image type"); + + int Npix = (region.y1 - region.y0) * (region.x1 - region.x0); + psVector *tmp = psVectorAllocEmpty (Npix, PS_TYPE_F32); + + int bin = 0; + for (int iy = region.y0; iy < region.y1; iy++) { + for (int ix = region.x0; ix < region.x1; ix++) { + if (!isfinite(image->data.F32[iy][ix])) continue; + tmp->data.F32[bin] = image->data.F32[iy][ix]; + bin ++; + } + } + tmp->n = bin; + + if (bin < 1) { + stats->min = NAN; + stats->lower20 = NAN; + stats->upper20 = NAN; + stats->max = NAN; + stats->Npts = 0; + psFree (tmp); + return true; + } + + + psVectorSortInPlace (tmp); + + int N20 = 0.2*tmp->n; + int N80 = 0.8*tmp->n; + int Nmax = tmp->n - 1; + + stats->min = tmp->data.F32[0]; + stats->lower20 = tmp->data.F32[N20]; + stats->upper20 = tmp->data.F32[N80]; + stats->max = tmp->data.F32[Nmax]; + stats->Npts = bin; + psFree (tmp); + + return true; +} + +// the return state indicates if any sources were actually subtracted +bool psphotAddOrSubSatstarsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int fileIndex, psMetadata *recipe, bool add) { + + bool status; + bool modified = false; + + psTimerStart ("psphot.deblend.sat"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, fileIndex); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + // if no work to do, should just return false + if (!sources) return false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping satstar blend"); + return false; + } + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // examine sources in decreasing SN order + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + if (!(source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE)) continue; + + // tell the calling function that we modified the image + modified = true; + + if (!psphotSatstarProfileOp (source, maskVal, 1.0, 0, add)) continue; + } + + psLogMsg ("psphot", PS_LOG_DETAIL, "satstar op: %f sec\n", psTimerMark ("psphot.deblend.sat")); + return modified; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDefineFiles.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDefineFiles.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDefineFiles.c (revision 41083) @@ -0,0 +1,215 @@ +# include "psphotInternal.h" + +bool psphotINpsphotStack = false; + +// List of output files +static const char *outputFiles[] = { "PSPHOT.OUTPUT", "PSPHOT.RESID", "PSPHOT.BACKMDL", + "PSPHOT.BACKMDL.STDEV", "PSPHOT.BACKGND", "PSPHOT.BACKSUB", + "PSPHOT.PSF.SAVE", "SOURCE.PLOT.MOMENTS", "SOURCE.PLOT.PSFMODEL", + "SOURCE.PLOT.APRESID", NULL }; + +// XXX we need to be able to distinguish several cases: +// 1) the particular output data was not requested +// 2) the particular output data was requested but was not generated (skipped) +// 3) the particular output data was requested but was not generated (for valid failure) +// 4) the particular output data was requested but was not generated (surprising failure) + +// define the needed / desired I/O files +bool psphotDefineFiles (pmConfig *config, pmFPAfile *input) { + + bool status = false; + + // select recipe options supplied on command line + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + + // the output sources are carried on the input->fpa structures + pmFPAfile *outsources = pmFPAfileDefineOutputFromFile (config, input, "PSPHOT.OUTPUT"); + if (!outsources) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.OUTPUT"); + return false; + } + outsources->save = true; + + // if we are choosing not to save the output detections, mark this file inactive so it will + // not be written, but will keep the detections in memory for other functions (eg, astrometry) + if (!psMetadataLookupBool (NULL, recipe, "SAVE.OUTPUT")) { + pmFPAfileActivate (config->files, false, "PSPHOT.OUTPUT"); + outsources->save = false; + } + + // optionally save the residual image + if (psMetadataLookupBool(NULL, recipe, "SAVE.RESID")) { + pmFPAfile *output = pmFPAfileDefineOutputFromFile (config, input, "PSPHOT.RESID"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.RESID"); + return false; + } + output->save = true; + } + // optionally save the background model (small FITS image) + if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKMDL")) { + int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); + int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); + pmFPAfile *output = pmFPAfileDefineFromFile (config, input, DX, DY, "PSPHOT.BACKMDL"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKMDL"); + return false; + } + output->save = true; + } + // optionally save the background model's standard deviation (small FITS image) + if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKMDL.STDEV")) { + int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); + int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); + pmFPAfile *output = pmFPAfileDefineFromFile (config, input, DX, DY, "PSPHOT.BACKMDL.STDEV"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKMDL.STDEV"); + return false; + } + output->save = true; + } + // optionally save the full background image + if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKGND")) { + pmFPAfile *output = pmFPAfileDefineFromFile (config, input, 1, 1, "PSPHOT.BACKGND"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.BACKGND"); + return false; + } + output->save = true; + } + // optionally save the background-subtracted image + if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKSUB")) { + pmFPAfile *output = pmFPAfileDefineFromFile (config, input, 1, 1, "PSPHOT.BACKSUB"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT."); + return false; + } + output->save = true; + } +// // optionally save the smoothed variance model (small FITS image) +// if (psMetadataLookupBool(NULL, recipe, "SAVE.BACKMDL")) { +// int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); +// int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); +// pmFPAfile *output = pmFPAfileDefineFromFile (config, input, DX, DY, "PSPHOT.VARMDL"); +// if (!output) { +// psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.VARMDL"); +// return false; +// } +// output->save = true; +// } +// // optionally save the smoothed variance model's standard deviation (small FITS image) +// if (psMetadataLookupBool(NULL, recipe, "SAVE.VARMDL.STDEV")) { +// int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); +// int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); +// pmFPAfile *output = pmFPAfileDefineFromFile (config, input, DX, DY, "PSPHOT.VARMDL.STDEV"); +// if (!output) { +// psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.VARMDL.STDEV"); +// return false; +// } +// output->save = true; +// } + // optionally save the PSF Model + if (psMetadataLookupBool(NULL, recipe, "SAVE.PSF")) { + pmFPAfile *output = pmFPAfileDefineOutputFromFile (config, input, "PSPHOT.PSF.SAVE"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.PSF.SAVE"); + return false; + } + output->save = true; + } + + // optionally save output plots + // allow specific plots only + if (psMetadataLookupBool(NULL, recipe, "SAVE.PLOTS")) { + pmFPAfile *output = NULL; + output = pmFPAfileDefineOutputFromFile (config, input, "SOURCE.PLOT.MOMENTS"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for SOURCE.PLOT.MOMENTS"); + return false; + } + output->save = true; + output = pmFPAfileDefineOutputFromFile (config, input, "SOURCE.PLOT.PSFMODEL"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for SOURCE.PLOT.PSFMODEL"); + return false; + } + output->save = true; + output = pmFPAfileDefineOutputFromFile (config, input, "SOURCE.PLOT.APRESID"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for SOURCE.PLOT.APRESID"); + return false; + } + output->save = true; + } + + if (psMetadataLookupPtr(NULL, config->arguments, "SRC")) { + if (!pmFPAfileDefineFromArgs (&status, config, "PSPHOT.INPUT.CMF", "SRC")) { + psError(PSPHOT_ERR_CONFIG, false, "Failed to find/build PSPHOT.INPUT.CMF"); + return status; + } + } + + if (psMetadataLookupPtr(NULL, config->arguments, "FORCE")) { + if (!pmFPAfileDefineFromArgs (&status, config, "PSPHOT.INPUT.CFF", "FORCE")) { + psError(PSPHOT_ERR_CONFIG, false, "Failed to find/build PSPHOT.INPUT.CFF"); + return status; + } + } + + if (psMetadataLookupPtr(NULL, config->arguments, "SRCTEXT")) { + // XXX cannot use pmFPAfileDefineFromArgs: this is explicitly a FITS-based I/O function + // supply the attach the + if (!psphotLoadSRCTEXT(input->fpa, config)) { + psError(PSPHOT_ERR_CONFIG, false, "Failed to load PSPHOT.INPUT.TEXT"); + return status; + } + } + + if (psMetadataLookupPtr(NULL, config->arguments, "PSPHOT.PSF")) { + pmFPAfileBindFromArgs(&status, input, config, "PSPHOT.PSF.LOAD", "PSPHOT.PSF"); + if (!status) { + psError(PSPHOT_ERR_CONFIG, false, "Failed to find/build PSPHOT.PSF.LOAD"); + return status; + } + } + + // XXX add in example PSF image thumbnails + // pmFPAfileConstruct (config->files, format, config->camera, "PSPHOT.PSF_SAMPLE"); + + return true; +} + +void psphotFilesActivate(pmConfig *config, bool state) +{ + for (int i = 0; outputFiles[i]; i++) { + if (!pmFPAfileActivate(config->files, state, outputFiles[i])) { + psErrorClear(); + } + } + + return; +} + +// psphotGetFilerule +// Since psphotStack processes multipe FPAs at a time it has a different file rule structure than regular psphot. +// For the background output files we define a function psphotGetFilerule which given a base psphot file rule +// returns the corresponding psphotStack rule *if* the program is psphotStack. That is indicated by a global +// boolean which defaults to false, and psphotStack only sets to true + +const char *psphotGetFilerule(const char *psphotRule) { + const char *rule = psphotRule; + if (psphotINpsphotStack) { + if (!strcmp(psphotRule, "PSPHOT.BACKMDL")) { + rule = "PSPHOT.STACK.BACKMDL"; + } else if (!strcmp(psphotRule, "PSPHOT.BACKMDL.STDEV")) { + rule = "PSPHOT.STACK.BACKMDL.STDEV"; + } else if (!strcmp(psphotRule, "PSPHOT.BACKSUB")) { + rule = "PSPHOT.STACK.BACKSUB"; + } else if (!strcmp(psphotRule, "PSPHOT.BACKGND")) { + rule = "PSPHOT.STACK.BACKGND"; + } else { + psAssert(0, "unsupported file rule %s", psphotRule); + } + } + return rule; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDiagnosticPlots.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDiagnosticPlots.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotDiagnosticPlots.c (revision 41083) @@ -0,0 +1,203 @@ +# include "psphotInternal.h" + +# if (HAVE_KAPA) + +static int kapa_fd = -1; + +int psphotKapaOpen () +{ + char kapa[64]; + + strcpy (kapa, "kapa"); + + if (kapa_fd == -1) { + kapa_fd = KapaOpenNamedSocket (kapa, "psphot"); + } + return kapa_fd; +} + +bool psphotKapaClose () +{ + + if (kapa_fd == -1) + return true; + KapaClose (kapa_fd); + kapa_fd = -1; + return true; +} + +bool psphotImageBackgroundCellHistogram (psVector *values, float mean, float sigma, int ix, int iy) +{ + + KapaSection section; + Graphdata graphdata; + + int kapa = pmKapaOpen (true); + if (kapa == -1) { + psError(PS_ERR_UNKNOWN, true, "failure to open kapa"); + return false; + } + + psStats *stats = psStatsAlloc (PS_STAT_MAX | PS_STAT_MIN); + if (!psVectorStats (stats, values, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for histogram"); + return false; + } + + psHistogram *histogram = psHistogramAlloc (stats->min, stats->max, 1000); + psVectorHistogram (histogram, values, NULL, NULL, 0); + + KapaInitGraph (&graphdata); + KapaClearPlots (kapa); + + // plot 1 is the full range + section.x = 0.0; + section.y = 0.0; + section.dx = 1.0; + section.dy = 0.5; + section.name = strcreate ("bottom"); + KapaSetSection (kapa, §ion); + free (section.name); + + // set limits based on data values + graphdata.xmin = stats->min; + graphdata.xmax = stats->max; + + psStatsInit (stats); + stats->options = PS_STAT_MAX | PS_STAT_MIN; + if (!psVectorStats (stats, histogram->nums, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for histogram"); + psFree(histogram); + return false; + } + + // scale the plot to hold the histogram + graphdata.ymin = stats->min - 0.05*(stats->max - stats->min); + graphdata.ymax = stats->max + 0.05*(stats->max - stats->min); + + KapaSetLimits (kapa, &graphdata); + KapaSetFont (kapa, "helvetica", 14); + KapaBox (kapa, &graphdata); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 1; + KapaPrepPlot (kapa, histogram->nums->n, &graphdata); + KapaPlotVector (kapa, histogram->nums->n, histogram->bounds->data.F32, "x"); + KapaPlotVector (kapa, histogram->nums->n, histogram->nums->data.F32, "y"); + psFree (histogram); + + // plot 2 is the +/- 10 sigma + section.x = 0.0; + section.y = 0.5; + section.dx = 1.0; + section.dy = 0.5; + section.name = strcreate ("top"); + KapaSetSection (kapa, §ion); + free (section.name); + + // +/- 10 sigma + graphdata.xmin = mean - 10.0*sigma; + graphdata.xmax = mean + 10.0*sigma; + + histogram = psHistogramAlloc (graphdata.xmin, graphdata.xmax, 100); + psVectorHistogram (histogram, values, NULL, NULL, 0); + + psStatsInit (stats); + stats->options = PS_STAT_MAX | PS_STAT_MIN; + if (!psVectorStats (stats, histogram->nums, NULL, NULL, 0)) { + psError(PS_ERR_UNKNOWN, false, "failure to measure stats for histogram"); + psFree(histogram); + return false; + } + + // scale the plot to hold the histogram + graphdata.ymin = stats->min - 0.05*(stats->max - stats->min); + graphdata.ymax = stats->max + 0.05*(stats->max - stats->min); + + KapaSetLimits (kapa, &graphdata); + KapaSetFont (kapa, "helvetica", 14); + KapaBox (kapa, &graphdata); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 1; + KapaPrepPlot (kapa, histogram->nums->n, &graphdata); + KapaPlotVector (kapa, histogram->nums->n, histogram->bounds->data.F32, "x"); + KapaPlotVector (kapa, histogram->nums->n, histogram->nums->data.F32, "y"); + + char line[128]; + sprintf (line, "sky: %f +/- %f, cell %d,%d", mean, sigma, ix, iy); + KapaSendLabel (kapa, line, KAPA_LABEL_XP); + + // pause until user types 'return' + fprintf(stdout, "press return"); + if (!fgets(line, 64, stdin)) { + // This is just to avoid a compiler warning on some systems; it's not really necessary + psWarning("Couldn't read anything."); + } + + psFree (stats); + psFree (histogram); + + return true; +} + +bool psphotDiagnosticPlots (const pmConfig *config, const char *name, ...) { + + va_list argPtr; + bool status; + + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + psMetadata *plots = psMetadataLookupPtr (&status, recipe, "DIAGNOSTIC.PLOTS"); + assert (plots); + + // do we want the requested plot? + if (!psMetadataLookupBool (&status, plots, name)) return false; + + // Get the variable list parameters to pass to allocation function + va_start(argPtr, name); + + if (!strcmp(name, "IMAGE.BACKGROUND.CELL.HISTOGRAM")) { + + int ix = va_arg(argPtr, psS32); + int iy = va_arg(argPtr, psS32); + float mean = va_arg(argPtr, double); + float sigma = va_arg(argPtr, double); + psVector *values = va_arg(argPtr, psPtr); + + int xPlot = psMetadataLookupS32 (&status, plots, "IMAGE.BACKGROUND.CELL.HISTOGRAM.X"); + assert (status); + int yPlot = psMetadataLookupS32 (&status, plots, "IMAGE.BACKGROUND.CELL.HISTOGRAM.Y"); + assert (status); + + bool gotX = (xPlot < 0) || (xPlot == ix); + bool gotY = (yPlot < 0) || (yPlot == iy); + + if (gotX && gotY) { + psphotImageBackgroundCellHistogram (values, mean, sigma, ix, iy); + goto done; + } + } + + // Clean up stack after variable arguement has been used +done: + va_end(argPtr); + return true; +} + +# else + +int psphotKapaOpen () { return -1; } +bool psphotKapaClose () { return true; } +bool psphotImageBackgroundCellHistogram (psVector *values, float mean, float sigma, int ix, int iy) +{ + return true; +} +bool psphotDiagnosticPlots (pmConfig *config, char *name, ...) { return true; } + +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEfficiency.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEfficiency.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEfficiency.c (revision 41083) @@ -0,0 +1,623 @@ +# include "psphotInternal.h" + +#define MODEL_MASK (PM_MODEL_STATUS_NONCONVERGE | PM_MODEL_STATUS_OFFIMAGE | \ + PM_MODEL_STATUS_BADARGS | PM_MODEL_STATUS_LIMITS) // Mask to apply to models + +# define TESTING 0 + +// Calculate the limiting magnitude for an image +// +// We limit ourselves to calculating the peak flux in the smoothed image, which should be close (modulo +// non-Gaussian PSF) to the limiting flux in the un-smoothed original image. +static bool effLimit(float *magLim, // Limiting magntiude, to return + int *radius, // Radius for fake sources, to return + float *minFlux, // Minimum flux for fake sources, to return + float *norm, // Normalisation of PSF (conversion: peak --> integrated flux) + float *covarFactor,// Covariance factor + const pmReadout *ro, // Readout of interest + pmPSF *psf, // Point-spread function + float thresh, // Threshold for source identification + float smoothSigma, // Gaussian smoothing sigma + float smoothNsigma, // Smoothing limit + psImageMaskType maskVal // Value to mask + ) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PM_ASSERT_READOUT_VARIANCE(ro, false); + assert(magLim); + assert(radius); + assert(minFlux); + assert(norm); + assert(covarFactor); + + // apply the amplitude of kernel^2 to the covarFactor (why?) + // XXX this is simply undoing the scale scalculate in psImageCovarianceCalculateFactor + // if this is the right solution, make this extra calculation optional (and explain...) + psKernel *kernel = psImageSmoothKernel(smoothSigma, smoothNsigma); // Kernel used for smoothing + double sum2 = 0.0; // Sum of kernel squared + for (int y = kernel->yMin; y <= kernel->yMax; y++) { + for (int x = kernel->xMin; x <= kernel->xMax; x++) { + sum2 += PS_SQR(kernel->kernel[y][x]); + } + } + *covarFactor = sum2 * psImageCovarianceCalculateFactor(kernel, ro->covariance); + psFree(kernel); + + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics for variance + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(stats, NULL, ro->variance, ro->mask, maskVal, rng) || + !isfinite(stats->robustMedian)) { + psError(PSPHOT_ERR_DATA, false, "Unable to determine mean variance"); + psFree(stats); + psFree(rng); + return false; + } + psFree(rng); + float meanVar = stats->robustMedian; // Mean variance + psFree(stats); + +#ifndef USE_SINGLE_POINT_FOR_MODEL + // Need to normalise out difference between Gaussian and real PSF + int sizeX = ro->variance->numCols / 16; + int sizeY = ro->variance->numRows / 16; + int numPoints = 0; + float sum = 0; + for (int y = sizeY * 0.5 ; y < ro->variance->numRows; y += sizeY) { + for (int x = sizeX * 0.5 ; x < ro->variance->numCols; x += sizeX) { + pmModel *normModel = pmModelFromPSFforXY(psf, (float) x, (float) y, 1.0); // model for normalization + + if (!normModel || (normModel->flags & MODEL_MASK)) { + psFree(normModel); + continue; + } + float flux = normModel->class->modelFlux(normModel->params); // Total flux for peak of 1.0 + psFree(normModel); + if (!isfinite(flux)) { + continue; + } + numPoints++; + sum += flux; + } + } + if (!isfinite(sum) || numPoints == 0) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate PSF model for any of %d points.", sizeX * sizeY); + return false; + } + *norm = sum / numPoints; +#else + // This can fail for readout that has few good pixels. It's better to sample many points. + pmModel *normModel = pmModelFromPSFforXY(psf, 0.5 * ro->variance->numCols, + 0.5 * ro->variance->numRows, 1.0); // Model for normalisation + psFree(normModel); + if (!normModel || (normModel->flags & MODEL_MASK)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate PSF model."); + psFree(normModel); + return false; + } + *norm = normModel->modelFlux(normModel->params); // Total flux for peak of 1.0 + psFree(normModel); +#endif + + // The signal-to-noise of the smoothed image is: S/N ~ I_smooth / sqrt(variance * factor) + // since the variance factor tells us the variance in the smoothed image. Now, the trick is working + // out what the intensity in the smoothed image is, and how it is related to the flux. We are + // convolving Io G_* with G_PSF/2pi.w^2, where G_* is the approximately-Gaussian PSF of the star, + // G_PSF is the pretty-close-matching Gaussian of the convolution kernel, Io is the peak flux in the + // unsmoothed image, and w is the width of the Gaussian. Now, a normalised 2D Gaussian convolved + // with itself has a normalisation of 1/2pi(w^2+w^2) = 1/4pi.w^2. Therefore: + // I_smooth = Flux / 2*norm. + float peakLim = thresh * sqrtf(meanVar * *covarFactor); // Limiting peak value in smoothed image + float fluxLim = 2.0 * *norm * peakLim; // Limiting flux in original + *magLim = -2.5 * log10f(fluxLim); + psTrace("psphot.fake", 1, "Covar Factor: %f\n", *covarFactor); + psTrace("psphot.fake", 1, "Limiting peak: %f\n", peakLim); + psTrace("psphot.fake", 1, "Limiting flux: %f\n", fluxLim); + psTrace("psphot.fake", 1, "Limiting mag: %f\n", *magLim); + psLogMsg("psphot", PS_LOG_INFO, + "Detection efficiency:\n" + " Mean variance: %f * %f\n" + " Threshold: %f\n" + " Normalisation: %f\n" + " Limiting magnitude: %f\n", + meanVar, *covarFactor, thresh, *norm, *magLim); + + *radius = smoothSigma * smoothNsigma; + + *minFlux = 0.1 * sqrtf(meanVar); + + return true; +} + +/// Generate a fake image and add it in to the existing readout +static pmReadout *effGenerate(psImage **xSrc, psImage **ySrc, // Positions of sources + const pmReadout *ro, // Readout of interest + const pmPSF *psf, // Point-spread function + const psVector *magOffsets, // Magnitude offsets for fake sources + int numSources, // Number of fake sources for each bin + float refMag, // Reference magnitude + int radius, // Radius for fake sources + float minFlux // Minimum flux for fake sources + ) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PM_ASSERT_READOUT_IMAGE(ro, false); + PS_ASSERT_VECTOR_NON_NULL(magOffsets, false); + PS_ASSERT_VECTOR_TYPE(magOffsets, PS_TYPE_F32, false); + assert(xSrc); + assert(ySrc); + + int numBins = magOffsets->n; // Number of bins + int numCols = ro->image->numCols, numRows = ro->image->numRows; // Size of image + + *xSrc = psImageRecycle(*xSrc, numSources, numBins, PS_TYPE_F32); + *ySrc = psImageRecycle(*ySrc, numSources, numBins, PS_TYPE_F32); + + psVector *xAll = psVectorAlloc(numBins * numSources, PS_TYPE_F32); + psVector *yAll = psVectorAlloc(numBins * numSources, PS_TYPE_F32); + psVector *magAll = psVectorAlloc(numBins * numSources, PS_TYPE_F32); + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + for (int i = 0, index = 0; i < numBins; i++) { + float mag = refMag + magOffsets->data.F32[i]; // Instrumental magnitude of sources + + for (int j = 0; j < numSources; j++, index++) { + xAll->data.F32[index] = psRandomUniform(rng) * numCols; + yAll->data.F32[index] = psRandomUniform(rng) * numRows; + magAll->data.F32[index] = mag; + } + memcpy((*xSrc)->data.F32[i], &xAll->data.F32[i * numSources], + numSources * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + memcpy((*ySrc)->data.F32[i], &yAll->data.F32[i * numSources], + numSources * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + } + psFree(rng); + + bool oldThreads = pmReadoutFakeThreads(true); // Old threading status + + pmReadout *fakeRO = pmReadoutAlloc(NULL); // Fake readout + if (!pmReadoutFakeFromVectors(fakeRO, numCols, numRows, xAll, yAll, magAll, + NULL, NULL, psf, minFlux, radius, false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate fake image"); + psFree(fakeRO); + psFree(magAll); + psFree(xAll); + psFree(yAll); + return NULL; + } + psFree(magAll); + psFree(xAll); + psFree(yAll); + + pmReadoutFakeThreads(oldThreads); + + psBinaryOp(ro->image, ro->image, "+", fakeRO->image); + + // return the readout so we can subtract it later + return fakeRO; +} + +bool psphotEfficiency (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Efficiency ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotEfficiencyReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure detection efficiency for %s entry %d", filerule, i); + psErrorStackPrint(stderr, " "); + psErrorClear(); + // send message to log as well + psLogMsg ("psphot", PS_LOG_WARN, "failed to measure detection efficiency for %s entry %d", filerule, i); + } + } + return true; +} + +// Determine detection efficiency +bool psphotEfficiencyReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + bool status = true; + + psTimerStart("psphot.fake"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *realSources = detections->allSources; + psAssert (realSources, "missing sources?"); + + // XXX do we need to skip this step if we do not have a psf? + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + PM_ASSERT_READOUT_NON_NULL(readout, false); + PM_ASSERT_READOUT_IMAGE(readout, false); + PS_ASSERT_PTR_NON_NULL(psf, false); + PS_ASSERT_METADATA_NON_NULL(recipe, false); + + // Collect recipe information + float smoothNsigma = psMetadataLookupF32(&status, recipe, "PEAKS_SMOOTH_NSIGMA"); // Smoothing limit + psAssert (status && isfinite(smoothNsigma), "Unable to find PEAKS_SMOOTH_NSIGMA in recipe (or invalid value)"); + + float thresh = psMetadataLookupF32(&status, recipe, "PEAKS_NSIGMA_LIMIT_2"); + psAssert (status && isfinite(thresh), "Unable to find PEAKS_NSIGMA_LIMIT_2 in recipe (or invalid value)"); + + psVector *magOffsets = psMetadataLookupVector(&status, recipe, "EFF.MAG"); // Magnitude offsets + psAssert (status, "Unable to find EFF.MAG F32 vector in recipe"); + psAssert (magOffsets->type.type == PS_TYPE_F32, "Unable to find EFF.MAG F32 vector in recipe"); + + int numSources = psMetadataLookupS32(&status, recipe, "EFF.NUM"); // Number of sources for each bin + psAssert (status && (numSources > 0), "Unable to find EFF.NUM in recipe (or invalid value)"); + + float minGauss = psMetadataLookupF32(&status, recipe, "PEAKS_MIN_GAUSS"); // Minimum valid fraction of kernel + psAssert (status && isfinite(minGauss), "PEAKS_MIN_GAUSS is not set in recipe (or invalid)"); + + // find the PSF size information (why is this not part of the psf structure?) + float fwhmMajor = psMetadataLookupF32(NULL, readout->analysis, "FWHM_MAJ"); // PSF size in x + float fwhmMinor = psMetadataLookupF32(NULL, readout->analysis, "FWHM_MIN"); // PSF size in y + if (!isfinite(fwhmMajor) || !isfinite(fwhmMinor) || fwhmMajor == 0.0 || fwhmMinor == 0.0) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find FWHM_MAJ and FWHM_MIN in readout->analysis"); + return false; + } + + float smoothSigma = 0.5*(fwhmMajor + fwhmMinor) / (2.0*sqrtf(2.0*log(2.0))); // Gaussian smoothing sigma + int numCols = readout->image->numCols, numRows = readout->image->numRows; // Size of image + int numBins = magOffsets->n; // Number of bins + + psImageMaskType maskVal = psMetadataLookupImageMask(NULL, recipe, "MASK.PSPHOT"); // Value to mask + + // remove all sources, adding noise for subtracted sources + psphotRemoveAllSourcesByArray(realSources, recipe); + +#if TESTING + { + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + readout->image->data.F32[y][x] = psRandomGaussian(rng); + readout->variance->data.F32[y][x] = 1.0; + readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0; + } + } + psFree(readout->covariance); + readout->covariance = NULL; + psFree(rng); + } +#endif + + + + float magLim; // Guess at limiting magnitude + int radius; // Radius for fake sources + float minFlux; // Minimum flux for fake sources + float norm; // Normalisation of PSF + float covarFactor; // Covariance factor + if (!effLimit(&magLim, &radius, &minFlux, &norm, &covarFactor, readout, + psf, thresh, smoothSigma, smoothNsigma, maskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine limits for image"); + return false; + } + +#if TESTING + psphotSaveImage(NULL, readout->image, "orig_image.fits"); + psphotSaveImage(NULL, readout->variance, "orig_variance.fits"); + psphotSaveImage(NULL, readout->mask, "orig_mask.fits"); +#endif + + psImage *xFake = NULL, *yFake = NULL; // Coordinates of sources, each bin in a row + pmReadout *fakeRO = effGenerate(&xFake, &yFake, readout, psf, magOffsets, numSources, magLim, radius, minFlux); + if (!fakeRO) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate fake sources"); + psFree(xFake); + psFree(yFake); + return false; + } + +#if TESTING + psphotSaveImage(NULL, readout->image, "fake_image.fits"); + psphotSaveImage(NULL, readout->variance, "fake_variance.fits"); + psphotSaveImage(NULL, readout->mask, "fake_mask.fits"); +#endif + + // XXX Could speed this up significantly by only convolving the central pixels of each fake source + psVector *significance = NULL; // Significance image + { + int num = numSources * numBins; // Total number of sources + // Pixel coordinates of sources + psVector *xPix = psVectorAlloc(num, PS_TYPE_S32); + psVector *yPix = psVectorAlloc(num, PS_TYPE_S32); + for (int i = 0, index = 0; i < numBins; i++) { + for (int j = 0; j < numSources; j++, index++) { + float x = xFake->data.F32[i][j]; + float y = yFake->data.F32[i][j]; + xPix->data.S32[index] = PS_MIN(x + 0.5, numCols - 1); + yPix->data.S32[index] = PS_MIN(y + 0.5, numRows - 1); + } + } + + psVector *convImage = psImageSmoothMaskPixels(readout->image, readout->mask, maskVal, xPix, yPix, + smoothSigma, smoothNsigma, minGauss); // Convolved image + if (!convImage) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to smooth image pixels"); + psFree(xPix); + psFree(yPix); + psFree(xFake); + psFree(yFake); + return false; + } + psVector *convVar = psImageSmoothMaskPixels(readout->variance, readout->mask, maskVal, xPix, yPix, + smoothSigma, smoothNsigma, minGauss); // Convolved varianc + if (!convVar) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to smooth variance pixels"); + psFree(xPix); + psFree(yPix); + psFree(xFake); + psFree(yFake); + return false; + } + + float factor = 1.0 / covarFactor; // Correction for covariance + + const psImage *mask = readout->mask; // Mask for readout + for (int i = 0; i < num; i++) { + float imageVal = convImage->data.F32[i]; // Image value + float varVal = convVar->data.F32[i]; // Variance value + int x = xPix->data.S32[i], y = yPix->data.S32[i]; // Coordinates + if (imageVal < 0 || varVal <= 0 || (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal)) { + convImage->data.F32[i] = 0.0; + } else { + convImage->data.F32[i] = factor * PS_SQR(imageVal) / varVal; + } + } + + psFree(xPix); + psFree(yPix); + + significance = convImage; + psFree(convVar); + } + + thresh *= thresh; // "Significance" is actually significance-squared + + psVector *count = psVectorAlloc(numBins, PS_TYPE_S32); // Number of sources found in each bin + psArray *fakeSources = psArrayAlloc(numSources); // Fake sources in each bin + psArray *fakeSourcesAll = psArrayAllocEmpty(numSources * numBins); // All fake sources + for (int i = 0, index = 0; i < numBins; i++) { + int numFound = 0; // Number found + + // Determine extraction size + float mag = magLim + magOffsets->data.F32[i]; // Magnitude for bin + float peak = powf(10.0, -0.4 * mag) / norm; // Peak flux +#ifndef USE_SINGLE_POINT_FOR_MODEL + int sizeX = numCols / 16; + int sizeY = numRows / 16; + int numPoints = 0; + float sum = 0; + for (int y = sizeY * 0.5 ; y < numRows; y += sizeY) { + for (int x = sizeX * 0.5 ; x < numCols; x += sizeX) { + // Need to normalise out difference between Gaussian and real PSF + pmModel *model = pmModelFromPSFforXY(psf, (float) x, (float) y, peak); + + if (!model || (model->flags & MODEL_MASK)) { + psFree(model); + continue; + } + float sourceRadius = PS_MAX(radius, model->class->modelRadius(model->params, minFlux)); // Radius for source + psFree(model); + if (!isfinite(sourceRadius)) { + continue; + } + numPoints++; + sum += sourceRadius; + } + } + if (!isfinite(sum) || numPoints == 0) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate PSF model for any of %d points.", sizeX * sizeY); + return false; + } + float sourceRadius = sum / numPoints; +#else + // This old way can fail for heavily masked images + pmModel *model = pmModelFromPSFforXY(psf, numCols / 2.0, numRows / 2.0, peak); // Model for source + if (!model || (model->flags & MODEL_MASK)) { + psError(PS_ERR_UNKNOWN, true, "Unable to generate model for bin %d", i); + psFree(model); + return false; + } + float sourceRadius = PS_MAX(radius, model->modelRadius(model->params, minFlux)); // Radius for source + psFree(model); +#endif + + psArray *sources = psArrayAllocEmpty(numSources); // Sources in this bin + for (int j = 0; j < numSources; j++, index++) { + // Coordinates of interest + float sig = significance->data.F32[index]; // Significance of pixel + if (sig > thresh) { + pmSource *source = pmSourceAlloc(); // Fake source + float x = xFake->data.F32[i][j]; + float y = yFake->data.F32[i][j]; + source->peak = pmPeakAlloc(x, y, sig, PM_PEAK_LONE); + if (!pmSourceDefinePixels(source, readout, x, y, sourceRadius)) { + psErrorClear(); + continue; + } + source->peak->xf = x; + source->peak->yf = y; + source->modelPSF = pmModelFromPSFforXY(psf, x, y, 2.0 * sqrtf(sig)); + if (source->modelPSF) { + source->type = PM_SOURCE_TYPE_STAR; + + source->modelPSF->fitRadius = sourceRadius; + source->apRadius = sourceRadius; + + numFound++; + psArrayAdd(sources, sources->n, source); + psArrayAdd(fakeSourcesAll, fakeSourcesAll->n, source); + } else { + psLogMsg("psphot", PS_LOG_WARN, "pmModelFromPSFforXY failed for sig: %f x: %6.1f y: %6.1f\n", + sig, x, y); + } + psFree(source); + } + } + fakeSources->data[i] = sources; + count->data.S32[i] = numFound; + } + psFree(xFake); + psFree(yFake); + psFree(significance); + + // psphotFitSourcesLinearReadout subtracts the model fits + if (!psphotFitSourcesLinearReadout(recipe, readout, fakeSourcesAll, psf, true, PM_SOURCE_PHOTFIT_CONST, false)) { + psError(PS_ERR_UNKNOWN, false, "Unable to perform linear fit on fake sources."); + psFree(fakeSources); + psFree(count); + return false; + } + + // Disable aperture corrections (save current value) + pmTrend2D *apTrend = psf->ApTrend; // Aperture trend + psf->ApTrend = NULL; + + // measure the magnitudes and fluxes for the sources + if (!psphotMagnitudesReadout(config, recipe, view, readout, fakeSourcesAll, psf, index)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure magnitudes of fake sources."); + psFree(fakeSources); + psFree(count); + psf->ApTrend = apTrend; // Casting away const! + return false; + } + + // replace the subtracted model fits + for (int i = 0; i < fakeSourcesAll->n; i++) { + pmSource *source = fakeSourcesAll->data[i]; + + // replace other sources? + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) continue; + + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + psFree(fakeSourcesAll); + + // Replace aperture corrections + psf->ApTrend = apTrend; + + psVector *magDiffMean = psVectorAlloc(numBins, PS_TYPE_F32); // Mean difference in magnitude for each bin + psVector *magDiffStdev = psVectorAlloc(numBins, PS_TYPE_F32); // Stdev of diff in magnitude for each bin + psVector *magErrMean = psVectorAlloc(numBins, PS_TYPE_F32); // Mean error in magnitude for each bin + psVector *magDiff = psVectorAlloc(numSources, PS_TYPE_F32); // Magnitude differences + psVector *magErr = psVectorAlloc(numSources, PS_TYPE_F32); // Magnitude errors + psVector *magMask = psVectorAlloc(numSources, PS_TYPE_VECTOR_MASK); // Mask for magnitude errors + psStats *magStats = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | + PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics + for (int i = 0; i < numBins; i++) { + psStatsInit(magStats); + psVectorInit(magMask, 0); + +#if TESTING + psString name = NULL; + psStringAppend(&name, "fake_%d.dat", i); + FILE *file = fopen(name, "w"); + psFree(name); +#endif + + float magRef = magLim + magOffsets->data.F32[i]; // Reference magnitude + psArray *sources = fakeSources->data[i]; // Sources in bin + for (int j = 0; j < sources->n; j++) { + pmSource *source = sources->data[j]; // Source of interest + if (!source || !isfinite(source->psfMag)) { + magMask->data.PS_TYPE_VECTOR_MASK_DATA[j] = 0xFF; + continue; + } + +#if TESTING + fprintf(file, "%f %f %f %f %f %f %f\n", source->peak->xf, source->peak->yf, + source->modelPSF->params->data.F32[PM_PAR_XPOS], + source->modelPSF->params->data.F32[PM_PAR_YPOS], + magRef, source->psfMag, source->psfMagErr); +#endif + magDiff->data.F32[j] = source->psfMag - magRef; + magErr->data.F32[j] = source->psfMagErr; + } + magDiff->n = sources->n; + magMask->n = sources->n; + magErr->n = sources->n; + if (!psVectorStats(magStats, magDiff, NULL, magMask, 0xFF)) { + // Probably because we don't have enough sources + psErrorClear(); + } + + if (isfinite(magStats->robustMedian) && isfinite(magStats->robustStdev)) { + magDiffMean->data.F32[i] = magStats->robustMedian; + magDiffStdev->data.F32[i] = magStats->robustStdev; + } else { + magDiffMean->data.F32[i] = magStats->sampleMean; + magDiffStdev->data.F32[i] = magStats->sampleStdev; + } + + if (!psVectorStats(magStats, magErr, NULL, magMask, 0xFF)) { + // Probably because we don't have enough sources + psErrorClear(); + } + + if (isfinite(magStats->robustMedian)) { + magErrMean->data.F32[i] = magStats->robustMedian; + } else { + magErrMean->data.F32[i] = magStats->sampleMean; + } + +#if TESTING + fclose(file); +#endif + } + + psFree(magStats); + psFree(magDiff); + psFree(magMask); + psFree(magErr); + + psFree(fakeSources); + + // subtract the faked sources from the original image + psBinaryOp(readout->image, readout->image, "-", fakeRO->image); + psFree(fakeRO); + + pmDetEff *de = pmDetEffAlloc(magLim, numSources, numBins); // Detection efficiency + de->magOffsets = psVectorCopy(NULL, magOffsets, PS_TYPE_F32); + de->counts = count; + de->magDiffMean = magDiffMean; + de->magDiffStdev = magDiffStdev; + de->magErrMean = magErrMean; + + psMetadataAddPtr(readout->analysis, PS_LIST_TAIL, PM_DETEFF_ANALYSIS, PS_META_REPLACE | PS_DATA_UNKNOWN, + "Detection efficiency", de); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "DETEFF.MAGREF", + PS_META_REPLACE, "Magnitude reference", magLim); + + psFree(de); + + psLogMsg("psphot", PS_LOG_WARN, "Detection efficiency: %lf sec\n", psTimerClear("psphot.fake")); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEllipticalContour.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEllipticalContour.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEllipticalContour.c (revision 41083) @@ -0,0 +1,172 @@ +# include "psphotInternal.h" + +// model parameters +enum {PAR_PHI, PAR_EPSILON, PAR_RMIN}; +psF32 psphotEllipticalContourFunc (psVector *deriv, const psVector *params, const psVector *coord); + +bool psphotEllipticalContour (pmSource *source) { + + psAssert (source, "missing source"); + psAssert (source->extpars, "missing extpars"); + psAssert (source->extpars->radFlux, "missing radFlux"); + + pmSourceRadialFlux *profile = source->extpars->radFlux; + pmSourceExtendedPars *extpars = source->extpars; + + // use LMM to fit theta vs radius to an ellipse + psVector *theta = profile->theta; + psVector *radius = profile->isophotalRadii; + + // find Rmin and Rmax for the initial guess + float Rmin = radius->data.F32[0]; + float Rmax = radius->data.F32[0]; + + // arrays to hold the data to be fitted + // we fit x and y vs theta in separate passes. + psArray *x = psArrayAllocEmpty(2*radius->n); + psVector *y = psVectorAllocEmpty(2*radius->n, PS_TYPE_F32); + psVector *yErr = psVectorAllocEmpty(2*radius->n, PS_TYPE_F32); + + int n = 0; + for (int i = 0; i < radius->n; i++) { + if (!isfinite(radius->data.F32[i])) continue; + + psVector *coord = NULL; + + // Rx coordinate value + coord = psVectorAlloc (2, PS_TYPE_F32); + coord->data.F32[1] = 0.0; + coord->data.F32[0] = theta->data.F32[i]; + x->data[n] = coord; + y->data.F32[n] = radius->data.F32[i]*cos(theta->data.F32[i]); + yErr->data.F32[n] = 1000.0; + n++; + + // Ry coordinate value + coord = psVectorAlloc (2, PS_TYPE_F32); + coord->data.F32[1] = 1.0; + coord->data.F32[0] = theta->data.F32[i]; + x->data[n] = coord; + y->data.F32[n] = radius->data.F32[i]*sin(theta->data.F32[i]); + yErr->data.F32[n] = 1000.0; + n++; + + // check the radius range + Rmin = MIN (Rmin, radius->data.F32[i]); + Rmax = MAX (Rmax, radius->data.F32[i]); + } + x->n = n; + y->n = n; + yErr->n = n; + + if (n < 4) { + psFree (x); + psFree (y); + psFree (yErr); + source->mode2 |= PM_SOURCE_MODE2_ECONTOUR_FEW_PTS; + return false; + } + + psVector *params = psVectorAlloc (3, PS_TYPE_F32); + + // psTraceSetLevel ("psLib.math.psMinimizeLMChi2", 7); + + // create the minimization constraints + psMinConstraint *constraint = psMinConstraintAlloc(); + + // XXX for now, no parameter masks, skip checkLimits + // XXX might help to add a limit to the angle or re-parameterize the ellipse in terms of Rxx, Ryy, Rxy + // constraint->checkLimits = psastroModelBoresiteLimits; + + params->data.F32[PAR_PHI] = 0.0; + params->data.F32[PAR_EPSILON] = Rmin / Rmax; + params->data.F32[PAR_RMIN] = Rmin; + + psMinimization *myMin = psMinimizationAlloc (25, 0.01, 1.00); + psImage *covar = psImageAlloc (params->n, params->n, PS_TYPE_F32); + + // XXX skip the weights for now + psMinimizeLMChi2(myMin, covar, params, constraint, x, y, yErr, psphotEllipticalContourFunc); + + /// XXX rationalize? if epsilon > 1, flip major and minor axes (rotate by 90 degrees) + if (params->data.F32[PAR_EPSILON] < 1.0) { + extpars->axes.major = params->data.F32[PAR_RMIN] / params->data.F32[PAR_EPSILON]; + extpars->axes.minor = params->data.F32[PAR_RMIN]; + extpars->axes.theta = params->data.F32[PAR_PHI]; + } else { + extpars->axes.major = params->data.F32[PAR_RMIN]; + extpars->axes.minor = params->data.F32[PAR_RMIN] / params->data.F32[PAR_EPSILON]; + extpars->axes.theta = params->data.F32[PAR_PHI] + 0.5*M_PI; + } + + psTrace ("psphot", 4, "# fitted values:\n"); + psTrace ("psphot", 4, "Phi: %f\n", extpars->axes.theta*PS_DEG_RAD); + psTrace ("psphot", 4, "Rmaj: %f\n", extpars->axes.major); + psTrace ("psphot", 4, "Rmin: %f\n", extpars->axes.minor); + + // show the results + // psphotPetrosianVisualEllipticalContour (profile, extpars); + + psFree (x); + psFree (y); + psFree (yErr); + psFree (params); + psFree (covar); + psFree (myMin); + psFree (constraint); + + return true; +} + +/** + * the full chisq is built of two associated sums over coordinates: + * chisq = sum ((Rx_obs - Rx_fit(t))^2 + (Ry_obs - Ry_fit(t))^2) + * we use split this into a 2x long vector and use coord[1] to distinguish the X and Y terms: + * coord[0] = measured X or measured Y + * coord[1] = 0 or 1 + */ +psF32 psphotEllipticalContourFunc (psVector *deriv, const psVector *params, const psVector *coord) { + + psF32 *par = params->data.F32; + + float alpha = coord->data.F32[0]; + + float cs_alpha = cos(alpha); + float sn_alpha = sin(alpha); + + float cs_phi = cos(alpha - par[PAR_PHI]); + float sn_phi = sin(alpha - par[PAR_PHI]); + + float r = 1.0 / sqrt(SQ(sn_phi) + SQ(par[PAR_EPSILON]*cs_phi)); + float r3 = pow(r, 3.0); + float drdE = -0.5 * r3 * SQ(cs_phi) * 2.0 * par[PAR_EPSILON]; + float drdP = -0.5 * r3 * (SQ(par[PAR_EPSILON]) - 1) * 2.0 * cs_phi * sn_phi; + + // value is X + if (coord->data.F32[1] < 0.5) { + float value = par[PAR_RMIN]*cs_alpha*r; + + if (deriv) { + psF32 *dPAR = deriv->data.F32; + dPAR[PAR_RMIN] = r*cs_alpha; + dPAR[PAR_EPSILON] = par[PAR_RMIN]*cs_alpha*drdE; + dPAR[PAR_PHI] = 4.0*par[PAR_RMIN]*cs_alpha*drdP; + } + return (value); + } + + // value is Y + if (coord->data.F32[1] > 0.5) { + float value = par[PAR_RMIN]*sn_alpha*r; + + if (deriv) { + psF32 *dPAR = deriv->data.F32; + dPAR[PAR_RMIN] = r*sn_alpha; + dPAR[PAR_EPSILON] = par[PAR_RMIN]*sn_alpha*drdE; + dPAR[PAR_PHI] = 4.0*par[PAR_RMIN]*sn_alpha*drdP; + } + return (value); + } + + psAbort ("programming error: invalid coordinate"); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEllipticalProfile.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEllipticalProfile.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEllipticalProfile.c (revision 41083) @@ -0,0 +1,90 @@ +# include "psphotInternal.h" + +bool psphotEllipticalProfile (pmSource *source, bool RAW_RADIUS) { + + psAssert (source, "missing source"); + psAssert (source->extpars, "missing extpars"); + psAssert (source->pixels, "missing pixels"); + + pmSourceExtendedPars *extpars = source->extpars; + + if (!source->extpars->ellipticalFlux) { + source->extpars->ellipticalFlux = pmSourceEllipticalFluxAlloc(); + } + pmSourceEllipticalFlux *profile = source->extpars->ellipticalFlux; + + profile->radiusElliptical = psVectorAllocEmpty(100, PS_TYPE_F32); + profile->fluxElliptical = psVectorAllocEmpty(100, PS_TYPE_F32); + + psVector *radius = profile->radiusElliptical; + psVector *flux = profile->fluxElliptical; + + // the psEllipse functions use z = 0.5(x/Sxx)^2 + 0.5(y/Syy)^2 + x y Sxy + // which are converted to z = 0.5(x/a)^2 + 0.5(y/b)^2 + // we have major and minor axes of a specific ellipse with r^2 = (x/A)^2 + (y/B)^2 + // a = A / sqrt(2) + + // we have the shape parameters of the elliptical contour at the reference isophote. + // use the axis ratio (major/minor) to rescale the radial profile so that 1 pixel + // along the major axis is 1 pixel, and a smaller amount on the minor axis + + psEllipseAxes axes; + if (RAW_RADIUS) { + // force circular profile + axes.major = M_SQRT1_2; + axes.minor = M_SQRT1_2; + } else { + axes.major = M_SQRT1_2; + axes.minor = M_SQRT1_2 * (extpars->axes.minor / extpars->axes.major); + } + + // axes.major = 1.0; + // axes.minor = extpars->axes.minor / extpars->axes.major; + + axes.theta = extpars->axes.theta; + psEllipseShape shape = psEllipseAxesToShape (axes); + + float Sxx = shape.sx; + float Sxy = shape.sxy; + float Syy = shape.sy; + + // XXX drop these two vectors? + psVector *radiusRaw = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *fluxRaw = psVectorAllocEmpty(100, PS_TYPE_F32); + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + + // 0.5 PIX: get radius as a function of pixel coord + float x = ix + 0.5 - source->peak->xf + source->pixels->col0; + float y = iy + 0.5 - source->peak->yf + source->pixels->row0; + + float r2 = 0.5*PS_SQR(x/Sxx) + 0.5*PS_SQR(y/Syy) + x*y*Sxy; + float Rraw = hypot(x, y); + + psVectorAppend(radius, sqrt(r2)); + psVectorAppend(flux, source->pixels->data.F32[iy][ix]); + + psVectorAppend(radiusRaw, Rraw); + psVectorAppend(fluxRaw, source->pixels->data.F32[iy][ix]); + } + } + + // psVector *radiusRaw = psVectorAllocEmpty(100, PS_TYPE_F32); + // psVector *fluxRaw = psVectorAllocEmpty(100, PS_TYPE_F32); + // for (int i = 0; i < profile->radii->n; i++) { + // psVector *r = profile->radii->data[i]; + // psVector *f = profile->fluxes->data[i]; + // for (int j = 0; j < r->n; j++) { + // psVectorAppend(radiusRaw, r->data.F32[j]); + // psVectorAppend(fluxRaw, f->data.F32[j]); + // } + // } + + psphotPetrosianVisualProfileRadii (radius, flux, radiusRaw, fluxRaw, source->peak->rawFlux, 0.0); + // psphotPetrosianVisualProfileByAngle (radius, flux); + + psFree (radiusRaw); + psFree (fluxRaw); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.c.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.c.in (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.c.in (revision 41083) @@ -0,0 +1,25 @@ +/* + * The line + { PSPHOT_ERR_$X{ErrorCode}, "$X{ErrorDescription}"}, + * (without the Xs) + * will be replaced by values from errorCodes.dat + */ +#include "pslib.h" +#include "psphotErrorCodes.h" + +void psphotErrorRegister(void) +{ + static psErrorDescription errors[] = { + { PSPHOT_ERR_BASE, "First value we use; lower values belong to psLib" }, + { PSPHOT_ERR_${ErrorCode}, "${ErrorDescription}"}, + }; + static int nerror = PSPHOT_ERR_NERROR - PSPHOT_ERR_BASE; // number of values in enum + + for (int i = 0; i < nerror; i++) { + psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription)); + *tmp = errors[i]; + psErrorRegister(tmp, 1); + psFree(tmp); /* it's on the internal list */ + } + nerror = 0; // don't register more than once +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.dat =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.dat (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.dat (revision 41083) @@ -0,0 +1,19 @@ +# +# This file is used to generate psphotErrorClasses.h +# +BASE = 3000 First value we use +UNKNOWN Unknown PM error code +NOT_IMPLEMENTED Desired feature is not yet implemented +FITS Problem in FITS I/O +FITS_WCS Error interpreting FITS WCS information +PHOTOM Problem in photometry +PSF Problem in PSF +APERTURE Problem with aperture photometry +SKY Problem in sky determination +IO Problem in data I/O +# these errors correspond to standard exit conditions +ARGUMENTS Incorrect arguments +SYS System error +CONFIG Problem in configure files +PROG Programming error +DATA invalid data Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.h.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.h.in (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotErrorCodes.h.in (revision 41083) @@ -0,0 +1,18 @@ +#if !defined(PSPHOT_ERROR_CODES_H) +#define PSPHOT_ERROR_CODES_H +/* + * The line + * PSPHOT_ERR_$X{ErrorCode}, + * (without the X) + * + * will be replaced by values from errorCodes.dat + */ +typedef enum { + PSPHOT_ERR_BASE = 3000, + PSPHOT_ERR_${ErrorCode}, + PSPHOT_ERR_NERROR +} psphotErrorCode; + +void psphotErrorRegister(void); + +#endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEvalFLT.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEvalFLT.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEvalFLT.c (revision 41083) @@ -0,0 +1,64 @@ +# include "psphotInternal.h" + +bool psphotEvalEXT (pmSource *source, pmModel *model) +{ + int keep; + + // do we actually have a valid EXT model? + if (model == NULL) { + source->mode &= ~PM_SOURCE_MODE_FITTED; + psTrace ("psphot", 5, "no model fitted?\n"); + return false; + } + + // was the model actually fitted? + if (!(model->flags & PM_MODEL_STATUS_FITTED)) { + source->mode &= ~PM_SOURCE_MODE_FITTED; + psTrace ("psphot", 5, "no model fitted?\n"); + return false; + } + + // did the model fit fail for one or another reason? + if (model->flags & (PM_MODEL_STATUS_BADARGS | + PM_MODEL_STATUS_NONCONVERGE | + PM_MODEL_STATUS_OFFIMAGE)) { + source->mode |= PM_SOURCE_MODE_FAIL; + psLogMsg ("psphot", 5, "EXT fail fit\n"); + psTrace ("psphot", 5, "EXT fail fit\n"); + + if (model->flags & PM_MODEL_STATUS_OFFIMAGE) { + psTrace ("psphot", 5, "off image\n"); + } + if (model->flags & PM_MODEL_STATUS_BADARGS) { + psTrace ("psphot", 5, "bad args\n"); + } + if (model->flags & PM_MODEL_STATUS_NONCONVERGE) { + psTrace ("psphot", 5, "non converge\n"); + } + return false; + } + + // unless we prove otherwise, this object is extended + source->type = PM_SOURCE_TYPE_EXTENDED; + + // the following source->mode information pertains to modelEXT: + source->mode |= PM_SOURCE_MODE_EXTMODEL; + + // if the object has a fitted peak below 0, the fit did not converge cleanly + // XXX this limit is fairly arbitrary, and must be > the value is the model limits + if (model->params->data.F32[PM_PAR_I0] <= 0.02) { + source->mode |= PM_SOURCE_MODE_FAIL; + psTrace ("psphot", 5, "model central intensity ~ zero\n"); + return false; + } + + keep = model->class->modelFitStatus(model); + if (keep) return true; + + // poor-quality fit; only keep if nothing else works... + psLogMsg ("psphot", 5, "EXT poor fit\n"); + psTrace ("psphot", 5, "EXT poor fit\n"); + + source->mode |= PM_SOURCE_MODE_POOR; + return false; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEvalPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEvalPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotEvalPSF.c (revision 41083) @@ -0,0 +1,200 @@ +# include "psphotInternal.h" + +// given a pmSource which has been fitted using modelPSF, evaluate the resulting fit: did +// the fit succeed? is this object saturated? return status is TRUE for a successful PSF +// fit, FALSE otherwise. + +// NOTE : 2008.10.12 EAM : This function used to make a measurement of the consistency of +// the source shape with the PSF. This feature has been superceded with the +// much-better-defined psphotSourceSize.c function + +static float SATURATION; +static float PSF_MIN_SN; +static float PSF_MAX_CHI; + +bool psphotInitLimitsPSF (psMetadata *recipe, pmReadout *readout) { + + bool status; + + // XXX do we need to set this differently from the value used to mark saturated pixels? + pmCell *cell = readout->parent; + + // do not completely trust the values in the header... + float CELL_SATURATION = psMetadataLookupF32 (&status, cell->concepts, "CELL.SATURATION"); + float MIN_SATURATION = psMetadataLookupF32 (&status, recipe, "DEBLEND_MIN_SATURATION"); + if (!status || !isfinite(MIN_SATURATION)) { + MIN_SATURATION = 40000.0; + } + if (!isfinite(CELL_SATURATION)) { + SATURATION = MIN_SATURATION; + } else { + SATURATION = PS_MAX(MIN_SATURATION, CELL_SATURATION); + } + + PSF_MIN_SN = psMetadataLookupF32 (&status, recipe, "PSF_MIN_SN"); + PSF_MAX_CHI = psMetadataLookupF32 (&status, recipe, "PSF_MAX_CHI"); + + return true; +} + +// examine the model->status, fit parameters, etc and decide if the model succeeded +// set the source->type and source->mode appropriately +bool psphotEvalPSF (pmSource *source, pmModel *model) { + + int keep; + float SN, Chi; + + // do we actually have a valid PSF model? + if (model == NULL) { + source->mode &= ~PM_SOURCE_MODE_FITTED; + return false; + } + + // was the model actually fitted? + if (!(model->flags & PM_MODEL_STATUS_FITTED)) { + source->mode &= ~PM_SOURCE_MODE_FITTED; + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + psTrace ("psphot", 5, "satstar was not fitted"); + } + return false; + } + // did the model fit fail for one or another reason? + if (model->flags & (PM_MODEL_STATUS_BADARGS | + PM_MODEL_STATUS_NONCONVERGE | + PM_MODEL_STATUS_OFFIMAGE)) { + source->mode |= PM_SOURCE_MODE_FAIL; + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + psTrace ("psphot", 5, "satstar failed fit"); + } + return false; + } + + // unless we prove otherwise, this object is a star. + source->type = PM_SOURCE_TYPE_STAR; + + // the following source->mode information pertains to modelPSF: + source->mode |= PM_SOURCE_MODE_PSFMODEL; + + // if the object has fitted peak above saturation, label as SATSTAR + // this is a valid PSF object, but ignore the other quality tests + // remember: fit does not use saturated pixels (masked) + // XXX no extended object can saturate and stay extended... + + // XXX this test is wrong : it should add back in the local sky from the model to test + // against saturation the local sky is saved in source->sky after psphotMagnitudes is called + if (model->params->data.F32[PM_PAR_I0] >= SATURATION) { + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + psLogMsg ("psphot", 5, "PSFSTAR marked SATSTAR\n"); + } + source->mode |= PM_SOURCE_MODE_SATSTAR; + return true; + } + + // if the object has a fitted peak below 0.02, the source is not viable + // perhaps the fit did not converge cleanly + if (model->params->data.F32[PM_PAR_I0] <= 0.02) { + source->mode |= PM_SOURCE_MODE_FAIL; + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + psTrace ("psphot", 5, "satstar failed fit (peak below 0)"); + } + return false; + } + + // if the source was predicted to be a SATSTAR, but it fitted below saturation, + // make a note to the user. + // XXX do not do this until we have a local measure of the sky in place + if (false && (source->mode & PM_SOURCE_MODE_SATSTAR)) { + psLogMsg ("psphot", 5, "SATSTAR marked normal (fitted peak below saturation)\n"); + source->mode &= ~PM_SOURCE_MODE_SATSTAR; + } + + SN = model->params->data.F32[PM_PAR_I0]/model->dparams->data.F32[PM_PAR_I0]; + Chi = model->chisqNorm / model->nDOF; + + // assign PM_SOURCE_MODE_GOODSTAR to bright objects within PSF region of dparams[] + keep = TRUE; + keep &= (SN > PSF_MIN_SN); + keep &= (Chi < PSF_MAX_CHI); + + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + psTrace ("psphot", 5, "satstar fit results: %f, %f %d : SN: %f Chisq: %f\n", + model->params->data.F32[PM_PAR_XPOS], model->params->data.F32[PM_PAR_YPOS], keep, SN, Chi); + } + + if (keep) { + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + psTrace ("psphot", 7, "PSFSTAR kept (%f, %f : SN: %f Chisq: %f)\n", + model->params->data.F32[PM_PAR_XPOS], model->params->data.F32[PM_PAR_YPOS], SN, Chi); + } + return true; + } + + // this source is not a star, warn if it was a PSFSTAR + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + psTrace ("psphot", 5, "PSFSTAR demoted based on fit quality (%f, %f : SN: %f Chisq: %f)\n", + model->params->data.F32[PM_PAR_XPOS], model->params->data.F32[PM_PAR_YPOS], SN, Chi); + } else { + psTrace ("psphot", 5, "fails PSF fit (%f, %f : SN: %f Chisq: %f)\n", + model->params->data.F32[PM_PAR_XPOS], model->params->data.F32[PM_PAR_YPOS], SN, Chi); + } + + // poor-quality fit; only keep if nothing else works... + source->mode |= PM_SOURCE_MODE_POOR; + return false; +} + +// examine the model->status, fit parameters, etc and decide if the model succeeded +// set the source->type and source->mode appropriately +bool psphotEvalDBL (pmSource *source, pmModel *model) { + + // do we actually have a valid PSF model? + if (model == NULL) { + source->mode &= ~PM_SOURCE_MODE_FITTED; + return false; + } + + // was the model actually fitted? + if (!(model->flags & PM_MODEL_STATUS_FITTED)) { + source->mode &= ~PM_SOURCE_MODE_FITTED; + return false; + } + // did the model fit fail for one or another reason? + if (model->flags & (PM_MODEL_STATUS_BADARGS | + PM_MODEL_STATUS_NONCONVERGE | + PM_MODEL_STATUS_OFFIMAGE)) { + source->mode |= PM_SOURCE_MODE_FAIL; + return false; + } + + // unless we prove otherwise, this object is a star. + source->type = PM_SOURCE_TYPE_STAR; + + // the following source->mode information pertains to modelPSF: + source->mode |= PM_SOURCE_MODE_PSFMODEL; + + // if the object has fitted peak above saturation, label as SATSTAR + // this is a valid PSF object, but ignore the other quality tests + // remember: fit does not use saturated pixels (masked) + // XXX no extended object can saturate and stay extended... + if (model->params->data.F32[PM_PAR_I0] >= SATURATION) { + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + psLogMsg ("psphot", 5, "PSFSTAR marked SATSTAR\n"); + } + source->mode |= PM_SOURCE_MODE_SATSTAR; + return true; + } + + // if the object has a fitted peak below 0, the fit did not converge cleanly + if (model->params->data.F32[PM_PAR_I0] <= 0.02) { + source->mode |= PM_SOURCE_MODE_FAIL; + return false; + } + + // if the source was predicted to be a SATSTAR, but it fitted below saturation, + // make a note to the user + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + psLogMsg ("psphot", 5, "SATSTAR marked normal (fitted peak below saturation)\n"); + source->mode &= ~PM_SOURCE_MODE_SATSTAR; + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceAnalysis.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceAnalysis.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceAnalysis.c (revision 41083) @@ -0,0 +1,295 @@ +# include "psphotInternal.h" +void psphotRadialProfileShowSkips (); + +// measure the petrosian parameters for the sources + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotExtendedSourceAnalysis (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Extended Source Analysis (Petrosians) ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + bool doAnnuli = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ANNULI"); + + // measure petrosians? + if (!doPetrosian && !doAnnuli) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping extended source measurements\n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotExtendedSourceAnalysisReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on measure extended source aperture-like parameters for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +/*** for the moment, this test code : it is not thread safe ***/ +static int Nall = 0; +static int Nskip1 = 0; +static int Nskip2 = 0; +static int Nskip3 = 0; +static int Nskip4 = 0; +static int Nskip5 = 0; +static int Nskip6 = 0; + +# define SKIP(VALUE) { VALUE++; continue; } + +// aperture-like measurements for extended sources +bool psphotExtendedSourceAnalysisReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + int Next = 0; + int Npetro = 0; + int Nannuli = 0; + + psTimerStart ("psphot.extended"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source size"); + return true; + } + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // get the sky noise from the background analysis; if this is missing, get the user-supplied value + float skynoise = psMetadataLookupF32 (&status, readout->analysis, "SKY_STDEV"); + if (!status) { + skynoise = psMetadataLookupF32 (&status, recipe, "SKY.NOISE"); + psWarning ("failed to get sky noise level from background analysis; defaulting to user supplied value of %f\n", skynoise); + } + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion *AnalysisRegion = psRegionAlloc(0,0,0,0); + *AnalysisRegion = psRegionForImage (readout->image, psRegionFromString (region)); + if (psRegionIsNaN (*AnalysisRegion)) psAbort("analysis region mis-defined"); + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_EXTENDED_ANALYSIS"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, AnalysisRegion); + psArrayAdd(job->args, 1, recipe); + + PS_ARRAY_ADD_SCALAR(job->args, skynoise, PS_TYPE_F32); + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Next + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Npetro + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nannuli + +// set this to 0 to run without threading +# if (0) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } +# else + if (!psphotExtendedSourceAnalysis_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } + psScalar *scalar = NULL; + scalar = job->args->data[5]; + Next += scalar->data.S32; + scalar = job->args->data[6]; + Npetro += scalar->data.S32; + scalar = job->args->data[7]; + Nannuli += scalar->data.S32; + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[5]; + Next += scalar->data.S32; + scalar = job->args->data[6]; + Npetro += scalar->data.S32; + scalar = job->args->data[7]; + Nannuli += scalar->data.S32; + } + psFree(job); + } + } + psFree (cellGroups); + psFree(AnalysisRegion); + + psLogMsg ("psphot", PS_LOG_WARN, "extended source analysis: %f sec for %d objects\n", psTimerMark ("psphot.extended"), Next); + psLogMsg ("psphot", PS_LOG_INFO, " %d petrosian\n", Npetro); + psLogMsg ("psphot", PS_LOG_INFO, " %d annuli\n", Nannuli); + +# if (PS_TRACE_ON) + fprintf (stderr, "ext analysis skipped @ 1 : %d\n", Nskip1); + fprintf (stderr, "ext analysis skipped @ 2 : %d\n", Nskip2); + fprintf (stderr, "ext analysis skipped @ 3 : %d\n", Nskip3); + fprintf (stderr, "ext analysis skipped @ 4 : %d\n", Nskip4); + fprintf (stderr, "ext analysis skipped @ 5 : %d\n", Nskip5); + fprintf (stderr, "ext analysis skipped @ 6 : %d\n", Nskip6); +#endif + + psphotRadialProfileShowSkips (); + + psphotVisualShowResidualImage (readout, false); + + psphotVisualShowPetrosians (sources); + + return true; +} + +bool psphotExtendedSourceAnalysis_Threaded (psThreadJob *job) { + + bool status; + + int Next = 0; + int Npetro = 0; + int Nannuli = 0; + + // arguments: readout, sources, models, region, psfSize, maskVal, markVal + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psRegion *region = job->args->data[2]; + psMetadata *recipe = job->args->data[3]; + + float skynoise = PS_SCALAR_VALUE(job->args->data[4],F32); + + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // choose the sources of interest + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + Nall ++; + + // rules for measuring petrosian parameters for specific objects are set in + // psphotChooseAnalysisOptions.c + if (!(source->tmpFlags & PM_SOURCE_TMPF_PETRO)) SKIP (Nskip1); + + // limit selection by analysis region (XXX move this into psphotChooseAnalysisOption?) + if (source->peak->x < region->x0) SKIP (Nskip2); + if (source->peak->y < region->y0) SKIP (Nskip3); + if (source->peak->x > region->x1) SKIP (Nskip4); + if (source->peak->y > region->y1) SKIP (Nskip5); + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + Next ++; + + // force source image to be a bit larger... + float radius = source->peak->xf - source->pixels->col0; + radius = PS_MAX (radius, source->peak->yf - source->pixels->row0); + radius = PS_MAX (radius, source->pixels->numRows - source->peak->yf + source->pixels->row0); + radius = PS_MAX (radius, source->pixels->numCols - source->peak->xf + source->pixels->col0); + pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 1.5*radius); + + // measure the radial profile + if (!psphotRadialProfile (source, recipe, skynoise, maskVal)) { + // re-subtract the object, leave local sky + psTrace ("psphot", 5, "failed to extract radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + SKIP (Nskip6); + } + + Nannuli ++; + source->mode |= PM_SOURCE_MODE_RADIAL_FLUX; + + // Petrosian Mags + if (doPetrosian) { + if (!psphotPetrosian (source, recipe, skynoise, maskVal)) { + psTrace ("psphot", 5, "FAILED petrosian flux & radius for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + } else { + psTrace ("psphot", 5, "measured petrosian flux & radius for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + Npetro ++; + source->mode |= PM_SOURCE_MODE_EXTENDED_STATS; + } + } + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + if (source->extpars) { + psFree(source->extpars->radFlux); + psFree(source->extpars->ellipticalFlux); + } + } + + psScalar *scalar = NULL; + + // change the value of a scalar on the array (wrap this and put it in psArray.h) + scalar = job->args->data[5]; + scalar->data.S32 = Next; + + scalar = job->args->data[6]; + scalar->data.S32 = Npetro; + + scalar = job->args->data[7]; + scalar->data.S32 = Nannuli; + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceAnalysisByObject.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceAnalysisByObject.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceAnalysisByObject.c (revision 41083) @@ -0,0 +1,161 @@ +# include "psphotInternal.h" + +// XXX option to choose a consistent elliptical contour +// XXX SDSS uses the r-band petrosian radius to measure petrosian fluxes in all bands + +// aperture-like measurements for extended sources +bool psphotExtendedSourceAnalysisByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule) { + + bool status; + int Next = 0; + int Npetro = 0; + int Nannuli = 0; + + psTimerStart ("psphot.extended"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full non-linear fits / extended source analysis? + if (!psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ANALYSIS")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping extended source measurements\n"); + return true; + } + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // XXX require petrosian analysis for non-linear fits? + + // XXX temporary user-supplied systematic sky noise measurement (derive from background model) + float skynoise = psMetadataLookupF32 (&status, recipe, "SKY.NOISE"); + + // S/N limit to perform full non-linear fits + float SN_LIM = psMetadataLookupF32 (&status, recipe, "EXTENDED_SOURCE_SN_LIM"); + + // which extended source analyses should we perform? + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + bool doAnnuli = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ANNULI"); + + // number of images used to define sources + int nImages = psphotFileruleCount(config, filerule); + + // generate look-up arrays for readouts + psArray *readouts = psArrayAlloc(nImages); + for (int i = 0; i < nImages; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + psLogMsg("psphot", PS_LOG_INFO, "petrosians for image %d", i); + psphotVisualShowImage(readout); + + readouts->data[i] = psMemIncrRefCounter(readout); + } + + // source analysis is done in S/N order (brightest first) + objects = psArraySort (objects, pmPhotObjSortByFlux); + + // process the objects in order. + for (int i = 0; i < objects->n; i++) { + pmPhotObj *object = objects->data[i]; + if (!object) continue; + if (!object->sources) continue; + + // we need to decide for an object if we are going to measure all sources or not + // simple rule : if *any* of the sources would be measured, measure the object + + // choose the sources of interest + bool measureSource = false; + for (int j = 0; !measureSource && (j < object->sources->n); j++) { + + pmSource *source = object->sources->data[j]; + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_STAR) continue; + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + if (!(source->mode & PM_SOURCE_MODE_EXT_LIMIT)) continue; + + // limit selection to some SN limit + assert (source->peak); // how can a source not have a peak? + if (sqrt(source->peak->detValue) < SN_LIM) continue; + measureSource = true; + } + if (!measureSource) continue; + + // choose the sources of interest + for (int j = 0; j < object->sources->n; j++) { + + pmSource *source = object->sources->data[j]; + psAssert (source, "programming error"); // all entries in object->sources must exist, right? + psAssert (source->peak, "programming error"); // how can a source not have a peak? + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + Next ++; + + int index = source->imageID; + pmReadout *readout = readouts->data[index]; + + // force source image to be a bit larger... + float radius = source->peak->xf - source->pixels->col0; + radius = PS_MAX (radius, source->peak->yf - source->pixels->row0); + radius = PS_MAX (radius, source->pixels->numRows - source->peak->yf + source->pixels->row0); + radius = PS_MAX (radius, source->pixels->numCols - source->peak->xf + source->pixels->col0); + pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 1.5*radius); + + // if we request any of these measurements, we require the radial profile + if (doPetrosian || doAnnuli) { + if (!psphotRadialProfile (source, recipe, skynoise, maskVal)) { + // all measurements below require the radial profile; skip them all + // re-subtract the object, leave local sky + psTrace ("psphot", 5, "FAILED radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + continue; + } else { + psTrace ("psphot", 5, "measured radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + Nannuli ++; + source->mode |= PM_SOURCE_MODE_RADIAL_FLUX; + } + } + + // Petrosian Mags + if (doPetrosian) { + if (!psphotPetrosian (source, recipe, skynoise, maskVal)) { + psTrace ("psphot", 5, "FAILED petrosian flux & radius for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + } else { + psTrace ("psphot", 5, "measured petrosian flux & radius for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + Npetro ++; + source->mode |= PM_SOURCE_MODE_EXTENDED_STATS; + } + } + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + if (source->extpars) { + psFree(source->extpars->radFlux); + psFree(source->extpars->ellipticalFlux); + psFree(source->extpars->petProfile); + } + } + } + + psLogMsg ("psphot", PS_LOG_INFO, "extended source analysis: %f sec for %d objects\n", psTimerMark ("psphot.extended"), Next); + psLogMsg ("psphot", PS_LOG_INFO, " %d petrosian\n", Npetro); + psLogMsg ("psphot", PS_LOG_INFO, " %d annuli\n", Nannuli); + + psFree(readouts); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceFits.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceFits.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotExtendedSourceFits.c (revision 41083) @@ -0,0 +1,742 @@ +# include "psphotInternal.h" +bool psphotSetWindowTrail (float *fitRadius, float *windowRadius, pmReadout *readout, pmSource *source, psImageMaskType markVal, float newRadius); + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotExtendedSourceFits (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Extended Source Fits ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full extended source non-linear fits? + if (!psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping extended source fits\n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotExtendedSourceFitsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on to fit extended sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// non-linear model fitting for extended sources +bool psphotExtendedSourceFitsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + int Next = 0; + int Nconvolve = 0; + int NconvolvePass = 0; + int Nplain = 0; + int NplainPass = 0; + int Nfaint = 0; + int Nfail = 0; + + psphotSersicModelClassInit(); + + psTimerStart ("psphot.extended"); + + psphotFitInitExtended(); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + psLogMsg("psphot", PS_LOG_INFO, "extended source fits for image %d", index); + psphotVisualShowImage(readout); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source size"); + psphotSersicModelClassCleanup(); + return true; + } + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + // do not thread if we are trying to study the fitting process + if (psTraceGetLevel ("psphot.psphotFitEXT") >= 6) { + nThreads = 0; + } + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + assert (markVal); + + // source fitting parameters for extended source fits + int fitIter = psMetadataLookupS32(&status, recipe, "EXT_FIT_ITER"); // Max number of fit iterations + assert (status && fitIter > 0); + + float fitMinTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MIN_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + fitMinTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_TOL"); // Fit tolerance + if (!status || !isfinite(fitMinTol) || fitMinTol <= 0) { + psAbort("PSF_FIT_MIN_TOL (and PSF_FIT_TOL) not defined or positive"); + } + } + + float fitMaxTol = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_TOL"); // Fit tolerance + if (!status || !isfinite(fitMaxTol) || fitMaxTol <= 0) { + fitMaxTol = 1.0; + } + + float fitNsigmaConv = psMetadataLookupF32 (&status, recipe, "EXT_FIT_NSIGMA_CONV"); // number of sigma for the convolution + if (!status || !isfinite(fitNsigmaConv) || fitNsigmaConv <= 0) { + fitNsigmaConv = 5.0; + } + + bool chisqConvergence = psMetadataLookupBool (&status, recipe, "LMM_FIT_CHISQ_CONVERGENCE"); // Fit tolerance + if (!status) { + // default to the old method (chisqConvergence) + chisqConvergence = true; + } + + int gainFactorMode = psMetadataLookupS32 (&status, recipe, "LMM_FIT_GAIN_FACTOR_MODE"); // Fit tolerance + if (!status) { + // default to the old method (chisqConvergence) + gainFactorMode = 0; + } + + // perform full extended source non-linear fits? + bool isInteractive = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS_INTERACTIVE"); + if (!status) isInteractive = false; + + // Define source fitting parameters for extended source fits + pmSourceFitOptions *fitOptions = pmSourceFitOptionsAlloc(); + fitOptions->mode = PM_SOURCE_FIT_EXT_AND_SKY; + fitOptions->saveCovariance = true; // XXX make this a user option? + fitOptions->covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + fitOptions->nIter = fitIter; + fitOptions->minTol = fitMinTol; + fitOptions->maxTol = fitMaxTol; + fitOptions->nsigma = fitNsigmaConv; + + fitOptions->gainFactorMode = gainFactorMode; + fitOptions->chisqConvergence = chisqConvergence; + fitOptions->isInteractive = isInteractive; + + // use poissonian errors or local-sky errors + fitOptions->poissonErrors = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS_POISSON"); + if (!status) fitOptions->poissonErrors = true; + + // Save pointer to fitOptions for possible use later reinstantiating the pcm models on the output readout + // and during radial apertures measurements. + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PCM_FIT_OPTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "pcm fit options", + fitOptions); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // select the collection of desired models + psMetadata *allModels = psMetadataLookupMetadata (&status, recipe, "EXTENDED_SOURCE_MODELS"); + if (!status) { + psWarning ("extended source model fits requested but model model is missing (EXTENDED_SOURCE_MODELS)\n"); + psFree (fitOptions); + psphotSersicModelClassCleanup(); + return true; + } + if (allModels->list->n == 0) { + psWarning ("extended source model fits requested but no models are specified\n"); + psFree (fitOptions); + psphotSersicModelClassCleanup(); + return true; + } + + psMetadata *models = NULL; + char *modelSelection = psMetadataLookupStr (&status, recipe, "EXTENDED_SOURCE_MODELS_SELECTION"); + if (!modelSelection || !status) { + models = psMemIncrRefCounter(allModels); + goto got_models; + } + + if (!strcasecmp(modelSelection, "ALL")) { + models = psMemIncrRefCounter(allModels); + goto got_models; + } + + if (!strcasecmp(modelSelection, "NONE")) { + psWarning ("extended source model fits requested but no models are selected (EXTENDED_SOURCE_MODELS_SELECTION = NONE)\n"); + psFree (fitOptions); + psphotSersicModelClassCleanup(); + return true; + } + + psArray *selection = psStringSplitArray (modelSelection, ",", false); + if (selection->n == 0) { + psWarning ("extended source model fits requested but model selection string is empty (EXTENDED_SOURCE_MODELS_SELECTION)\n"); + psFree (fitOptions); + psFree (selection); + psphotSersicModelClassCleanup(); + return true; + } + models = psMetadataAlloc(); + for (int i = 0; i < selection->n; i++) { + psMetadata *model = psMetadataLookupMetadata (&status, allModels, selection->data[i]); + if (!model) { + psWarning ("extended source model selection string includes an invalid name %s\n", (char *) selection->data[i]); + continue; + } + psMetadataAddMetadata (models, PS_LIST_TAIL, selection->data[i], PS_META_REPLACE, "extended model", model); + } + psFree (selection); + + if (models->list->n == 0) { + psWarning ("extended source model fits requested but no valid models in selection string (EXTENDED_SOURCE_MODELS_SELECTION = %s)\n", modelSelection); + psFree (fitOptions); + psFree (models); + psphotSersicModelClassCleanup(); + return true; + } + +got_models: + + psphotInitRadiusEXT (recipe, readout); + + // validate the model entries + psMetadataIterator *iter = psMetadataIteratorAlloc (models, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + + if (item->type != PS_DATA_METADATA) { + // XXX we could cull the bad entries or build a validated model folder + psAbort ("Invalid type for EXTENDED_SOURCE_MODEL entry %s, not a metadata folder", item->name); + } + + psMetadata *model = (psMetadata *) item->data.md; + + // check on the model type + char *modelName = psMetadataLookupStr (&status, model, "MODEL"); + int modelType = pmModelClassGetType (modelName); + if (modelType < 0) { + psAbort ("Unknown model class for EXTENDED_SOURCE_MODEL entry %s: %s", item->name, modelName); + } + psMetadataAddS32 (model, PS_LIST_TAIL, "MODEL_TYPE", PS_META_REPLACE, "", modelType); + + // check on the SNLIM, set a float value + char *SNword = psMetadataLookupStr (&status, model, "SNLIM"); + if (!status) { + psAbort("SNLIM not defined for extended source model %s\n", item->name); + } + float SNlim = atof (SNword); + psMetadataAddF32 (model, PS_LIST_TAIL, "SNLIM_VALUE", PS_META_REPLACE, "", SNlim); + + // check on the PSF-Convolution status + char *convolvedWord = psMetadataLookupStr (&status, model, "PSF_CONVOLVED"); + if (!status || (strcasecmp (convolvedWord, "true") && strcasecmp (convolvedWord, "false"))) { + psAbort ("PSF_CONVOLVED entry invalid or missing for EXTENDED_SOURCE_MODEL entry %s", item->name); + } + bool convolved = !strcasecmp (convolvedWord, "true"); + psMetadataAddBool (model, PS_LIST_TAIL, "PSF_CONVOLVED_VALUE", PS_META_REPLACE, "", convolved); + + if (convolved) { + psLogMsg ("psphot", PS_LOG_INFO, "using convolved model class %s (%s)", modelName, item->name); + } else { + psLogMsg ("psphot", PS_LOG_INFO, "using simple model class %s (%s)", modelName, item->name); + } + } + psFree (iter); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion *AnalysisRegion = psRegionAlloc(0,0,0,0); + *AnalysisRegion = psRegionForImage(readout->image, psRegionFromString (region)); + if (psRegionIsNaN (*AnalysisRegion)) psAbort("analysis region mis-defined"); + + // what fraction of the PSF is used? (radius in pixels : 2 -> 5x5 box) + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_EXTENDED_FIT"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, models); + + // Allocate a metadata iterator here because psMetadataIteratorAlloc/Free are not thread safe + psMetadataIterator *iter = psMetadataIteratorAlloc (models, PS_LIST_HEAD, NULL); + psArrayAdd(job->args, 1, iter); + psArrayAdd(job->args, 1, AnalysisRegion); + psArrayAdd(job->args, 1, fitOptions); + + PS_ARRAY_ADD_SCALAR(job->args, psfSize, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Next + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nconvolve + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for NconvolvePass + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nplain + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for NplainPass + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfaint + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail + +// XXX TEST + if (!isInteractive) { + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + psFree (fitOptions); + psFree (models); + psphotSersicModelClassCleanup(); + return false; + } + } else { + // run without threading + if (!psphotExtendedSourceFits_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + psFree (fitOptions); + psFree (models); + psphotSersicModelClassCleanup(); + return false; + } + psScalar *scalar = NULL; + scalar = job->args->data[9]; + Next += scalar->data.S32; + scalar = job->args->data[10]; + Nconvolve += scalar->data.S32; + scalar = job->args->data[11]; + NconvolvePass += scalar->data.S32; + scalar = job->args->data[12]; + Nplain += scalar->data.S32; + scalar = job->args->data[13]; + NplainPass += scalar->data.S32; + scalar = job->args->data[14]; + Nfaint += scalar->data.S32; + scalar = job->args->data[15]; + Nfail += scalar->data.S32; + psFree(job->args->data[3]); // iterator allocated above + psFree(job); + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + psFree (fitOptions); + psFree (models); + psphotSersicModelClassCleanup(); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[9]; + Next += scalar->data.S32; + scalar = job->args->data[10]; + Nconvolve += scalar->data.S32; + scalar = job->args->data[11]; + NconvolvePass += scalar->data.S32; + scalar = job->args->data[12]; + Nplain += scalar->data.S32; + scalar = job->args->data[13]; + NplainPass += scalar->data.S32; + scalar = job->args->data[14]; + Nfaint += scalar->data.S32; + scalar = job->args->data[15]; + Nfail += scalar->data.S32; + psFree(job->args->data[3]); // metadata iterator allocated above + } + psFree(job); + } + } + psFree (cellGroups); + psFree(AnalysisRegion); + psFree (fitOptions); + psFree (models); + + psphotSersicModelClassCleanup(); + + psphotVisualShowResidualImage (readout, false); + + psLogMsg ("psphot", PS_LOG_WARN, "extended source model fits: %f sec for %d objects\n", psTimerMark ("psphot.extended"), Next); + psLogMsg ("psphot", PS_LOG_INFO, " %d convolved models (%d passed)\n", Nconvolve, NconvolvePass); + psLogMsg ("psphot", PS_LOG_INFO, " %d plain models (%d passed)\n", Nplain, NplainPass); + psLogMsg ("psphot", PS_LOG_INFO, " %d too faint to fit, %d failed\n", Nfaint, Nfail); + + psphotFitSummaryExtended(); + + return true; +} + +// non-linear model fitting for extended sources +bool psphotExtendedSourceFits_Threaded (psThreadJob *job) { + + bool status; + int Next = 0; + int Nfaint = 0; + int Nfail = 0; + int Nconvolve = 0; + int NconvolvePass = 0; + int Nplain = 0; + int NplainPass = 0; + float fitRadius, windowRadius; + psScalar *scalar = NULL; + + // arguments: readout, sources, models, region, psfSize, maskVal, markVal + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psMetadata *models = job->args->data[2]; + psMetadataIterator *iter = job->args->data[3]; + psRegion *region = job->args->data[4]; + pmSourceFitOptions *fitOptions = job->args->data[5]; + + int psfSize = PS_SCALAR_VALUE(job->args->data[6],S32); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[7],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[8],PS_TYPE_IMAGE_MASK_DATA); + + // psTraceSetLevel ("psLib.math.psMinimizeLMChi2_Alt", 5); + + pmModelStatus badModel = PM_MODEL_STATUS_NONE; + badModel |= PM_MODEL_STATUS_BADARGS; + badModel |= PM_MODEL_STATUS_OFFIMAGE; + badModel |= PM_MODEL_STATUS_NAN_CHISQ; + badModel |= PM_MODEL_SERSIC_PCM_FAIL_GUESS; + badModel |= PM_MODEL_SERSIC_PCM_FAIL_GRID; + badModel |= PM_MODEL_PCM_FAIL_GUESS; + + // choose the sources of interest + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // rules for measuring petrosian parameters for specific objects are set in + // psphotChooseAnalysisOptions.c + if (!(source->tmpFlags & PM_SOURCE_TMPF_EXT_FIT)) continue; + + // limit selection by analysis region (XXX move this into psphotChooseAnalysisOption?) + if (source->peak->x < region->x0) continue; + if (source->peak->y < region->y0) continue; + if (source->peak->x > region->x1) continue; + if (source->peak->y > region->y1) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + Next ++; + + // set the fit radius based on the first radial moment (also sets the mask pixels) + psphotSetRadiusMomentsExact(&fitRadius, &windowRadius, readout, source, markVal); // NOTE : 6 allocs + + // UPDATE : we have changed the moments calculation. There is now an iteration within + // psphotKronMasked to determine moments appropriate for a larger object. The values + // Mrf, KronFlux, and KronFluxErr are calculated for the iterated radius. The other + // values are left at the psf-based values. + + // allocate the array to store the model fits + if (source->modelFits == NULL) { + source->modelFits = psArrayAllocEmpty (models->list->n); + } + +# ifdef TEST_OBJECT + bool testObject = false; + testObject |= ((fabs(source->peak->xf - 179) < 5) && (fabs(source->peak->yf - 1138) < 5)); + if (testObject) { + fprintf (stderr, "test object @ %f, %f\n", source->peak->xf, source->peak->yf); + psTraceSetLevel ("psModules.objects.pmPCM_MinimizeChisq", 5); + psTraceSetLevel ("psphot.psphotExtendedSourceFits_Threaded", 5); + } +# endif + + // loop here over the models chosen for each source (exclude by S/N) + // Reset the iterator + psMetadataIteratorSet(iter, PS_LIST_HEAD); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + + // XXX this should have been forced above + assert (item->type == PS_DATA_METADATA); + psMetadata *model = (psMetadata *) item->data.md; + + // check the SNlim and skip model if source is too faint + float FIT_SN_LIM = psMetadataLookupF32 (&status, model, "SNLIM_VALUE"); + assert (status); + + // limit selection to some SN limit for specific models (this value only applies if > EXTENDED_SOURCE_SN_LIM) + if (isfinite(FIT_SN_LIM)) { + if (source->moments->KronFlux < FIT_SN_LIM * source->moments->KronFluxErr) { + Nfaint ++; + continue; + } + } + + source->mode2 |= PM_SOURCE_MODE2_EXT_FITS_RUN; + + // check on the model type + pmModelType modelType = psMetadataLookupS32 (&status, model, "MODEL_TYPE"); + assert (status); + + // check on the PSF-Convolution status + bool convolved = psMetadataLookupBool (&status, model, "PSF_CONVOLVED_VALUE"); + assert (status); + + // fit the model as convolved or not + pmModel *modelFit = NULL; + if (convolved) { + // NOTE : 4 more allocs to here + modelFit = psphotFitPCM (readout, source, fitOptions, modelType, maskVal, markVal, psfSize); // NOTE : 2313 allocs in here + if (!modelFit) { + psTrace ("psphot", 5, "failed to fit psf-conv model for object at %f, %f", source->moments->Mx, source->moments->My); + Nfail ++; + source->mode2 |= PM_SOURCE_MODE2_EXT_FITS_FAIL; + continue; + } + psTrace ("psphot", 4, "fit psf-conv model for %f, %f : %s chisq = %f (npix: %d, niter: %d)\n", + source->moments->Mx, source->moments->My, pmModelClassGetName (modelFit->type), modelFit->chisq, modelFit->nPix, modelFit->nIter); + Nconvolve ++; + if (!(modelFit->flags & badModel)) { + float *PAR = modelFit->params->data.F32; + psEllipseAxes axes = pmPSF_ModelToAxes (PAR, modelFit->class->useReff); + if (axes.major >= 100) { + Nfail ++; + source->mode2 |= PM_SOURCE_MODE2_EXT_FITS_FAIL; + psFree(modelFit); + continue; + } + NconvolvePass ++; + source->mode |= PM_SOURCE_MODE_EXTENDED_FIT; + } + } else { + bool doneFits = false; + while (!doneFits) { + psFree (source->modelFlux); + source->modelFlux = NULL; + modelFit = psphotFitEXT (modelFit, readout, source, fitOptions, modelType, maskVal, markVal); + if (!modelFit) { + psTrace ("psphot", 5, "failed to fit plain model for object at %f, %f", source->moments->Mx, source->moments->My); + Nfail ++; + doneFits = true; + source->mode2 |= PM_SOURCE_MODE2_EXT_FITS_FAIL; + continue; + } + psTrace ("psphot", 4, "fit plain model for %f, %f : %s chisq = %f (npix: %d, niter: %d)\n", source->moments->Mx, source->moments->My, pmModelClassGetName (modelFit->type), modelFit->chisq, modelFit->nPix, modelFit->nIter); + Nplain ++; + if (!(modelFit->flags & badModel)) { + NplainPass ++; + source->mode |= PM_SOURCE_MODE_EXTENDED_FIT; + } + doneFits = true; + + if (modelType == pmModelClassGetType("PS_MODEL_TRAIL")) { + // calculate the object end points: + float *PAR = modelFit->params->data.F32; + float Xmin = PAR[PM_PAR_XPOS] - 0.5*PAR[PM_PAR_LENGTH]*sin(PAR[PM_PAR_THETA]); + float Ymin = PAR[PM_PAR_YPOS] - 0.5*PAR[PM_PAR_LENGTH]*cos(PAR[PM_PAR_THETA]); + float Xmax = PAR[PM_PAR_XPOS] + 0.5*PAR[PM_PAR_LENGTH]*sin(PAR[PM_PAR_THETA]); + float Ymax = PAR[PM_PAR_YPOS] + 0.5*PAR[PM_PAR_LENGTH]*cos(PAR[PM_PAR_THETA]); + + if ((fabs(source->peak->xf - 2572) < 20) && (fabs(source->peak->yf - 5874) < 20)) { + fprintf (stderr, "src vs fit : %d %d - %d %d | %f %f - %f %f\n", + source->pixels->col0, source->pixels->row0, + source->pixels->col0 + source->pixels->numCols, source->pixels->row0 + source->pixels->numRows, + Xmin, Ymin, Xmax, Ymax); + } + if (PAR[PM_PAR_LENGTH] > 0.9*fitRadius) { + doneFits = false; + fprintf (stderr, "update window : %f %f : %f -> %f\n", source->peak->xf, source->peak->yf, fitRadius, 2*fitRadius); + psphotSetWindowTrail (&fitRadius, &windowRadius, readout, source, markVal, fitRadius*2.0); + source->mode2 |= PM_SOURCE_MODE2_EXT_FITS_RETRY; + } + } + } + } + psAssert (modelFit, "modelFit not set?"); + + // test for fit quality / result + modelFit->fitRadius = fitRadius; + psArrayAdd (source->modelFits, 4, modelFit); + + psFree (modelFit); + } + + // we are allowed to fit both stars and non-stars here -- if we have fitted + // something which we think is a star, we should use that model to subtract the + // object from the image. + if (source->type == PM_SOURCE_TYPE_STAR) { + // ensure the modelPSF is cached + pmSourceCacheModel (source, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + continue; + } + + // evaluate the relative quality of the models, choose one + // the PSF model might be the best fit : allow it to succeed + float minChisq = NAN; + int minModel = -1; + for (int i = 0; i < source->modelFits->n; i++) { + pmModel *model = source->modelFits->data[i]; + + // skip the really bad fits + if (!(model->flags & PM_MODEL_STATUS_FITTED)) continue; + if (model->flags & badModel) continue; + + // if (model->flags & (PM_MODEL_STATUS_NONCONVERGE)) continue; + + if ((minModel < 0) || (model->chisq < minChisq)) { + minChisq = model->chisq; + minModel = i; + } + } + + // clear the circular mask + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + if (minModel == -1) { + // no valid extended fit; re-subtract the object, leave local sky + psTrace ("psphot", 5, "failed to fit extended source model to object at %f, %f", source->moments->Mx, source->moments->My); + + // ensure the modelEXT is cached + pmSourceCacheModel (source, maskVal); + +# if (0 && PS_TRACE_ON) + pmModel *model = source->modelFits->data[0]; + int flags = 0xffffffff; + if (model) { + flags = model->flags; + } + fprintf (stderr, "failed to fit extended source model to object %d @ %f, %f (%x)\n", source->id, source->moments->Mx, source->moments->My, flags); +#endif + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + source->mode2 |= PM_SOURCE_MODE2_EXT_FITS_NONE; + continue; + } + + // save the best extended model in modelEXT + psFree (source->modelEXT); + source->modelEXT = psMemIncrRefCounter (source->modelFits->data[minModel]); + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXTMODEL; + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + + // adjust the window so the subtraction covers the faint wings + psphotSetRadiusMoments(&fitRadius, &windowRadius, readout, source, markVal); + + // cache the model flux + if (source->modelEXT->isPCM) { + // fprintf (stderr, "subtract PCM extended source model for object %d @ %f, %f\n", source->id, source->moments->Mx, source->moments->My); + pmPCMCacheModel (source, maskVal, psfSize, fitOptions->nsigma); + } else { + // fprintf (stderr, "subtract non-PCM extended source model for object %d @ %f, %f\n", source->id, source->moments->Mx, source->moments->My); + pmSourceCacheModel (source, maskVal); + } + source->modelEXT->flags |= PM_MODEL_BEST_FIT; + + // subtract the best fit from the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + psTrace ("psphot", 4, "best ext model for %f, %f : %s chisq = %f\n", source->moments->Mx, source->moments->My, pmModelClassGetName (source->modelEXT->type), source->modelEXT->chisq); + psTrace ("psphot", 5, "extended source model for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + +# ifdef TEST_OBJECT + if (testObject) { + psTraceSetLevel ("psModules.objects.pmPCM_MinimizeChisq", 0); + psTraceSetLevel ("psphot.psphotExtendedSourceFits_Threaded", 0); + } +# endif + } + + // change the value of a scalar on the array (wrap this and put it in psArray.h) + scalar = job->args->data[9]; + scalar->data.S32 = Next; + + scalar = job->args->data[10]; + scalar->data.S32 = Nconvolve; + + scalar = job->args->data[11]; + scalar->data.S32 = NconvolvePass; + + scalar = job->args->data[12]; + scalar->data.S32 = Nplain; + + scalar = job->args->data[13]; + scalar->data.S32 = NplainPass; + + scalar = job->args->data[14]; + scalar->data.S32 = Nfaint; + + scalar = job->args->data[15]; + scalar->data.S32 = Nfail; + + return true; +} + +# define PAD_WINDOW 3.0 +bool psphotSetWindowTrail (float *fitRadius, float *windowRadius, pmReadout *readout, pmSource *source, psImageMaskType markVal, float newRadius) { + + psRegion newRegion; + + psAssert (source, "source not defined??"); + psAssert (source->moments, "moments not defined??"); + + *fitRadius = newRadius; + *windowRadius = *fitRadius + PAD_WINDOW; + + // check to see if new region is completely contained within old region + newRegion = psRegionForSquare (source->peak->xf, source->peak->yf, *windowRadius); + newRegion = psRegionForImage (readout->image, newRegion); + + // redefine the pixels to match + pmSourceRedefinePixelsByRegion (source, readout, newRegion); + + // clear the circular mask + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, source->peak->xf, source->peak->yf, *fitRadius, "OR", markVal); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFake.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFake.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFake.c (revision 41083) @@ -0,0 +1,220 @@ +# include "psphotInternal.h" + +#define TESTING + + +#ifdef TESTING +#define MIN_FLUX 0.1 // Minimum flux for faint sources +#endif + + +// Calculate the limiting magnitude for an image +// +// We limit ourselves to calculating the peak flux in the smoothed image, which should be close (modulo +// non-Gaussian PSF) to the limiting flux in the un-smoothed original image. +static bool fakeLimit(float *magLim, // Limiting magntiude, to return + float *minFlux, // Minimum flux, to return + const pmReadout *ro, // Readout of interest + float thresh, // Threshold for source identification + float xFWHM, float yFWHM, // Size of PSF + float smoothNsigma, // Smoothing limit + psImageMaskType maskVal // Value to mask + ); +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PM_ASSERT_READOUT_VARIANCE(ro, false); + PS_ASSERT_METADATA_NON_NULL(recipe, false); + + float smoothSigma = 0.5*(FWHM_X + FWHM_Y) / (2.0*sqrtf(2.0*log(2.0))); + psKernel *kernel = psImageSmoothKernel(smoothSigma, smoothNsigma); // Kernel used for smoothing + float factor = psImageCovarianceCalculateFactor(kernel, readout->covariance); // Covariance matrix + psFree(kernel); + + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics for variance + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(stats, NULL, ro->variance, ro->mask, maskVal, rng) || + !isfinite(stats->robustMedian)) { + psError(PSPHOT_ERR_DATA, false, "Unable to determine mean variance"); + psFree(stats); + psFree(rng); + return false; + } + psFree(rng); + float meanVar = stats->robustMedian; // Mean variance + psFree(stats); + + if (magLim) { + *magLim = -2.5 * log10(thresh * sqrtf(meanVar * factor)); + } + if (minFlux) { + int fudge = psMetadataLookupF32(NULL, recipe, "FAKE.MINFLUX"); // Fudge factor for minimum flux + *minFlux = sqrtf(meanVar) * fudge; + } + + return true; +} + +/// Generate a fake image and add it in to the existing readout +static bool fakeGenerate(psImage **xSrc, psImage **ySrc, // Positions of sources + const pmReadout *ro, // Readout of interest + const psVector *magOffsets, // Magnitude offsets for fake sources + int numSources, // Number of fake sources for each bin + float refMag, // Reference magnitude + float minFlux // Minimum flux level for fake image + ) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PM_ASSERT_READOUT_IMAGE(ro, false); + PS_ASSERT_METADATA_NON_NULL(recipe, false); + PS_ASSERT_VECTOR_NON_NULL(magOffsets, false); + PS_ASSERT_VECTOR_TYPE(magOffsets, PS_TYPE_F32, false); + + int numBins = magOffset->n; // Number of bins + int numCols = ro->image->numCols, numRows = ro->image->numRows; // Size of image + + *xSrc = psImageRecycle(*xSrc, numSources, numBins, PS_TYPE_F32); + *ySrc = psImageRecycle(*ySrc, numSources, numBins, PS_TYPE_F32); + + // Master list, for image creation + psVector *xAll = psVectorAlloc(numBins * numSources, PS_TYPE_F32); + psVector *yAll = psVectorAlloc(numBins * numSources, PS_TYPE_F32); + psVector *magAll = psVectorAlloc(numBins * numSources, PS_TYPE_F32); + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + for (int i = 0; i <= numBins; i++) { + psArray *data = psArrayAlloc(3); // Sources for bin + + psVector *x = psVectorAlloc(numSources, PS_TYPE_F32); + psVector *y = psVectorAlloc(numSources, PS_TYPE_F32); + + float mag = refMag + magOffset->data.F32[i]; // Instrumental magnitude of sources + + for (int j = 0; j <= numSources; j++) { + x->data.F32[j] = psRandomUniform(rng) * numCols; + y->data.F32[j] = psRandomUniform(rng) * numRows; + magAll->data.F32[i * numSources + j] = mag; + } + memcpy(xSrc->data.F32[i], x->data.F32, numSources * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + memcpy(ySrc->data.F32[i], y->data.F32, numSources * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + memcpy(&xAll->data.F32[i * numSources], x->data.F32, numSources * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + memcpy(&yAll->data.F32[i * numSources], y->data.F32, numSources * PSELEMTYPE_SIZEOF(PS_TYPE_F32)); + } + psFree(rng); + + pmReadout *fakeRO = pmReadoutAlloc(); // Fake readout + if (!pmReadoutFakeFromVectors(fakeRO, xAll, yAll, mag, NULL, NULL, psf, minFlux, NAN, false, false)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate fake image"); + psFree(fakeRO); + psFree(xAll); + psFree(yAll); + psFree(magAll); + return false; + } + psFree(xAll); + psFree(yAll); + psFree(magAll); + + psBinaryOp(ro->image, ro->image, "+", fakeRO->image); + psFree(fakeRO); + + return true; +} + + +// *** in this section, perform the photometry for fake sources *** +bool psphotFake(pmConfig *config, pmReadout *readout, const pmPSF *psf, + const psMetadata *recipe, const psArray *realSources) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PM_ASSERT_READOUT_NON_NULL(readout, false); + PM_ASSERT_READOUT_IMAGE(readout, false); + PS_ASSERT_PTR_NON_NULL(psf, false); + PS_ASSERT_METADATA_NON_NULL(recipe, false); + PS_ASSERT_ARRAY_NON_NULL(realSources, false); + + psTimerStart("psphot.fake"); + + // Collect recipe information + float xFWHM = psMetadataLookupF32(NULL, recipe, "FWHM_X"); // PSF size in x + float yFWHM = psMetadataLookupF32(NULL, recipe, "FWHM_Y"); // PSF size in y + if (!isfinite(xFWHM) || !isfinite(yFWHM)) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find FWHM_X and FWHM_Y in recipe"); + return false; + } + float smoothNsigma = psMetadataLookupF32(NULL, recipe, "PEAKS_SMOOTH_NSIGMA"); // Smoothing limit + if (!isfinite(smoothNsigma)) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find PEAKS_SMOOTH_NSIGMA in recipe"); + return false; + } + float thresh = psMetadataLookupF32(NULL, recipe, "PEAKS_NSIGMA_LIMIT_2"); + if (!isfinite(thresh)) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find PEAKS_NSIGMA_LIMIT_2 in recipe"); + return false; + } + psVector *magOffsets = psMetadataLookupVector(NULL, recipe, "FAKE.MAG"); // Magnitude offsets + if (!magOffset || magOffset->type.type != PS_TYPE_F32) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find FAKE.MAG F32 vector in recipe"); + return NULL; + } + int numSources = psMetadataLookupS32(NULL, recipe, "FAKE.NUM"); // Number of sources for each bin + if (numSources == 0) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find FAKE.NUM in recipe"); + return NULL; + } + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Value to mask + + + // remove all sources, adding noise for subtracted sources + psphotRemoveAllSourcesByArray(realSources, recipe); + psphotAddNoise(readout, realSources, recipe); + + float magLim; // Guess at limiting magnitude + float minFlux; // Minimum flux for fake image + if (!fakeLimit(&magLim, &minFlux, readout, thresh, xFWHM, yFWHM, smoothNsigma, maskVal)) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine limits for image"); + return false; + } + + psImage *xFake = NULL, *yFake = NULL; // Coordinates of sources, each bin in a row + if (!fakeGenerate(&xFake, &yFake, readout, magOffsets, numSources, magLim, minFlux)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate fake sources"); + psFree(xFake); + psFree(yFake); + return false; + } + + psImage *significance = psphotSignificanceImage(readout, recipe, 2, maskVal); + if (!significance) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate significance image"); + psFree(xFake); + psFree(yFake); + return false; + } + + int numBins = magOffsets->n; // Number of bins + psVector *frac = psVectorAlloc(numBins, PS_TYPE_S32); // Fraction of sources in each bin + for (int i = 0; i < numBins; i++) { + int numFound = 0; // Number found + for (int j = 0; j < numSources; j++) { + float x = xFake->data.F32[i][j], y = yFake->data.F32[i][j]; // Coordinates of interest + int xPix = x, yPix = y; // Pixel coordinates + if (significance->data.F32[yPix][xPix] > thresh) { + numFound++; + } + } + frac->data.S32[i] = (float)numFound / (float)numSources; + } + + psFree(xFake); + psFree(yFake); + psFree(significance); + + // Putting results on recipe because that appears to be the psphot standard, but it's not a good idea + psMetadataAddVector(readout->analysis, PS_LIST_TAIL, "FAKE.EFF", PS_META_REPLACE, "Efficiency fractions", frac); + psMetadataAddVector(readout->analysis, PS_LIST_TAIL, "FAKE.MAG", PS_META_REPLACE, "Efficiency magnitudes", magOffsets); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "FAKE.REF", PS_META_REPLACE, "Efficiency reference magnitude", magLim); + + psFree(frac); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFakeSources.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFakeSources.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFakeSources.c (revision 41083) @@ -0,0 +1,30 @@ +# include "psphotInternal.h" + +psArray *psphotFakeSources () { + + // psphotUpdateHeader (header, config); + + psArray *sources = psArrayAlloc (50); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = pmSourceAlloc (); + source->moments = pmMomentsAlloc (); + source->moments->Mx = 10; + source->moments->My = 10; + source->moments->Mxx = 1; + source->moments->Myy = 1; + source->moments->Mxy = 0; + source->moments->Sum = 1000; + source->moments->Peak = 100; + source->moments->Sky = 10; + source->moments->nPixels = 10; + + source->peak = pmPeakAlloc (10, 10, 0, 0); + source->type = PM_SOURCE_TYPE_STAR; + + pmModelType modelType = pmModelClassGetType ("PS_MODEL_QGAUSS"); + source->modelPSF = pmSourceModelGuess (source, modelType, 0, 0); + sources->data[i] = source; + } + return sources; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindDetections.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindDetections.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindDetections.c (revision 41083) @@ -0,0 +1,175 @@ +# include "psphotInternal.h" + +// we store the detections on the readout->analysis for each readout this function finds new +// peaks and new footprints. any old peaks are saved on oldPeaks. the resulting footprint set +// contains all footprints (old and new) +bool psphotFindDetections (pmConfig *config, const pmFPAview *view, const char *filerule, bool firstPass) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Find Detections ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotFindDetectionsReadout (config, view, filerule, i, recipe, firstPass)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to find initial detections for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// smooth the image, search for peaks, optionally define footprints based on the peaks +bool psphotFindDetectionsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool firstPass) { + + bool status; + int pass; + float NSIGMA_PEAK = 25.0; + int NMAX = 0; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + psAssert (markVal, "missing mark value?"); + + maskVal |= markVal; + + // Use the new pmFootprints approach? + const bool useFootprints = psMetadataLookupBool(NULL, recipe, "USE_FOOTPRINTS"); + const bool footprintUseUnsubtracted = psMetadataLookupBool(NULL, recipe, "FOOTPRINT_USE_UNSUBTRACTED"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + // on the initial pass, detections have not yet been allocated or saved on readout->analysis + if (!detections) { + // create the container + detections = pmDetectionsAlloc(); + + // save detections on the readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + } else { + psMemIncrRefCounter(detections); // so we can free the detections below + } + + // first pass vs other: if this is the first pass, the code will use PEAKS_NSIGMA_LIMIT and + // only attempt to detect PEAKS_NMAX entries. If 'firstPass' is false, the code will + // attempt to replace the subtracted sources in order to measure the footprints. After + // replacement, it is necessary to regenerate the significance image. If no sources are + // available, the code will skip the significance image regeneration step. + + bool replaceSourcesForFootprints = false; + if (firstPass) { + pass = 1; + NSIGMA_PEAK = psMetadataLookupF32 (&status, recipe, "PEAKS_NSIGMA_LIMIT"); PS_ASSERT (status, NULL); + NMAX = psMetadataLookupS32 (&status, recipe, "PEAKS_NMAX"); PS_ASSERT (status, NULL); + } else { + pass = 2; + replaceSourcesForFootprints = footprintUseUnsubtracted; + NSIGMA_PEAK = psMetadataLookupF32 (&status, recipe, "PEAKS_NSIGMA_LIMIT_2"); PS_ASSERT (status, NULL); + NMAX = 0; // unlimited number of peaks in final pass: allow a limit (PEAKS_NMAX_2) ? + } + + float threshold = PS_SQR(NSIGMA_PEAK); + + // move the old peaks array (if it exists) to oldPeaks XXX generically, we should be able + // to call this function an arbitrary number of times the old peaks are saved so they can + // be freed later -- the have to be freed after psphotFindFootprints is called below, since + // they are also owned by the oldFootprints, which are in turn merged into the new + // footprints. (what about the source->peak entry?) + + assert (detections->oldPeaks == NULL); + detections->oldPeaks = detections->peaks; + detections->peaks = NULL; + + // generate the smoothed significance image + pmReadout *significance = psphotSignificanceImage (readout, recipe, maskVal); + + // display the log significance image + psphotVisualShowLogSignificance (significance->variance, 0.0, 4.5); + + // display the significance image + psphotVisualShowSignificance (significance->variance, 0.98*threshold, 1.02*threshold); + + // detect the peaks in the significance image + int totalPeaks = 0; + detections->peaks = psphotFindPeaks (significance, readout, recipe, threshold, NMAX, &totalPeaks); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "PEAK_THRESHOLD", PS_META_REPLACE, "Peak Detection Threshold", threshold); + if (!detections->peaks) { + // we only get a NULL peaks array due to a programming or config error. + // this will result in a failure. + psFree (detections); + psError (PSPHOT_ERR_CONFIG, false, "failed on peak search"); + return false; + } + // hard limit on number of peaks we will accept. (To avoid memory overload in psphotStack) + int maxPeaks = psMetadataLookupS32 (&status, recipe, "PEAKS_NMAX_TOTAL"); PS_ASSERT (status, NULL); + if (maxPeaks && (totalPeaks > maxPeaks)) { + psFree (detections); + psError (PSPHOT_ERR_DATA, true, "Too many peaks %d found PEAKS_NMAX_TOTAL: %d", totalPeaks, maxPeaks); + return false; + } + + // optionally merge peaks into footprints + if (useFootprints) { + if (replaceSourcesForFootprints) { + bool modified = false; + // subtract the noise for all sources including satstars + modified |= psphotAddOrSubNoiseReadout(config, view, filerule, index, recipe, false); + modified |= psphotReplaceAllSourcesReadout (config, view, filerule, index, recipe, false); + + // add in the satstars + modified |= psphotAddOrSubSatstarsReadout (config, view, filerule, index, recipe, true); + + if (modified) { + psFree (significance); + significance = psphotSignificanceImage (readout, recipe, maskVal); + } + + // display the significance image + psphotVisualShowSignificance (significance->variance, 0.98*threshold, 1.02*threshold); + } + + psphotFindFootprints (detections, significance, readout, recipe, threshold, pass, maskVal); + + if (replaceSourcesForFootprints) { + psphotRemoveAllSourcesReadout (config, view, filerule, index, recipe, false); + + // subtract the satstars + psphotAddOrSubSatstarsReadout (config, view, filerule, index, recipe, false); + } + } + + // XXX do a second (or third?) pass with rebinning (to detected more extended sources) + + psFree (significance); + + // display the peaks and footprints + psphotVisualShowPeaks (detections); + psphotVisualShowFootprints (detections); + + psFree (detections); + + return true; +} + +// if we use the footprints, the output peaks list contains both old and new peaks, +// otherwise it only contains the new peaks. Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindFootprints.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindFootprints.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindFootprints.c (revision 41083) @@ -0,0 +1,94 @@ +# include "psphotInternal.h" + +bool psphotFindFootprints (pmDetections *detections, pmReadout *significance, pmReadout *readout, psMetadata *recipe, const float threshold, const int pass, psImageMaskType maskVal) { + + bool status; + + psTimerStart ("psphot.footprints"); + + int npixMin = psMetadataLookupS32(&status, recipe, "FOOTPRINT_NPIXMIN"); + PS_ASSERT (status, false); + + int growRadius = 0; + if (pass == 1) { + growRadius = psMetadataLookupS32(&status, recipe, "FOOTPRINT_GROW_RADIUS"); + } else { + growRadius = psMetadataLookupS32(&status, recipe, "FOOTPRINT_GROW_RADIUS_2"); + } + PS_ASSERT (status, false); + + // find the raw footprints in the smoothed significance image & assign the peaks to those footprints + psArray *footprints = pmFootprintsFind (significance->variance, threshold, npixMin); + + if (pmFootprintsAssignPeaks(footprints, detections->peaks) != PS_ERR_NONE) { + psAbort ("inconsistent peaks and footprints"); + } + + // footprints now owns the peaks; after culling (below), we will rebuild the peaks array + psFree (detections->peaks); + + psLogMsg ("psphot", PS_LOG_MINUTIA, "found %ld footprints: %f sec\n", footprints->n, psTimerMark ("psphot.footprints")); + + // optionally grow footprints isotropically by growRadius pixels + if (growRadius > 0) { + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolve + psArray *tmp = pmFootprintArrayGrow(footprints, growRadius); + psImageConvolveSetThreads(oldThreads); + psLogMsg ("psphot", PS_LOG_MINUTIA, "grow footprint coverage by %d pixels, %ld footprints become %ld footprints: %f sec\n", growRadius, footprints->n, tmp ? tmp->n : 0, psTimerMark ("psphot.footprints")); + if (tmp) { + psFree(footprints); + footprints = tmp; + } else { + psLogMsg ("psphot", PS_LOG_WARN, "pmFootprintArray grow returned NULL\n"); + } + + } + + if (pass == 2) { + // merge in old peaks; + const int includePeaks = 0x1 | 0x2; // i.e. both new and old footprints + + psArray *mergedFootprints = pmFootprintArraysMerge(detections->footprints, footprints, includePeaks); + psLogMsg ("psphot", PS_LOG_MINUTIA, "merged %ld new footprints with %ld existing ones: %f sec\n", footprints->n, (detections->footprints ? detections->footprints->n : 0), psTimerMark ("psphot.footprints")); + + psFree(footprints); + psFree(detections->footprints); + + // replace the merged footprints on the detection structure + detections->footprints = mergedFootprints; + } else { + detections->footprints = footprints; + } + + psphotCullPeaks(readout, significance, recipe, detections->footprints); + detections->peaks = pmFootprintArrayToPeaks(detections->footprints); + + // psphotFootprintSaddles (readout, detections->footprints); + // + // int nSaddle = 0; + // for (int i = 0; i < detections->peaks->n; i++) { + // pmPeak *peak = detections->peaks->data[i]; + // + // if (peak->saddlePoints) nSaddle += peak->saddlePoints->n; + // } + // fprintf (stderr, "%d saddle points for %ld peaks\n", nSaddle, detections->peaks->n); + + psLogMsg ("psphot", PS_LOG_WARN, "%ld peaks, %ld total footprints: %f sec\n", detections->peaks->n, detections->footprints->n, psTimerMark ("psphot.footprints")); + + return detections; +} + +bool psphotCheckFootprints (pmDetections *detections) { + + // check for messed up footprints in the old peaks + for (int i = 0; i < detections->oldPeaks->n; i++) { + pmPeak *peak = detections->oldPeaks->data[i]; + pmFootprint *footprint = peak->footprint; + if (!footprint) continue; + for (int j = 0; j < footprint->spans->n; j++) { + pmSpan *sp = footprint->spans->data[j]; + psAssert (sp, "missing span"); + } + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindPeaks.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindPeaks.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFindPeaks.c (revision 41083) @@ -0,0 +1,78 @@ +# include "psphotInternal.h" + +// Find peaks in the significance image above a threshold significance level. The significance +// image must be constructed to represent (S/N)^2. If nMax is non-zero, only return a maximum +// of nMax peaks +psArray *psphotFindPeaks (pmReadout *significance, pmReadout *readout, psMetadata *recipe, const float threshold, const int nMax, int *totalPeaks) { + + bool status = false; + + psTimerStart ("psphot.peaks"); + + // find the peaks in the smoothed image + // NOTE : significance->variance actually carries the detection S/N image + psArray *peaks = pmPeaksInImage (significance->variance, threshold); + if (peaks == NULL) { + // we only get a NULL peaks array due to a programming or config error. + // this will result in a failure. + psError(PSPHOT_ERR_DATA, false, "no peaks found in this image"); + return NULL; + } + // return the total number of peaks found before the nMax limit is applied + if (totalPeaks) { + *totalPeaks = peaks->n; + } + + if (peaks->n == 0) { + // XXX do we need to set something in the readout->analysis to indicate that + // we tried and failed to find peaks (something in the header data) + psLogMsg ("psphot", PS_LOG_INFO, "no peaks found in this image"); + return peaks; + } + + // Convert the peak values to S/N = sqrt(significance). + // Get the peak flux from the unsmoothed image. + // Rescale the peak position errors using the peak variance + // The peak pixel coords are guaranteed to be on the image + int row0 = readout->image->row0; + int col0 = readout->image->col0; + for (int i = 0; i < peaks->n; i++) { + pmPeak *peak = peaks->data[i]; + peak->rawFlux = readout->image->data.F32[peak->y-row0][peak->x-col0]; + peak->rawFluxStdev = sqrt(readout->variance->data.F32[peak->y-row0][peak->x-col0]); + peak->smoothFlux = significance->image->data.F32[peak->y-row0][peak->x-col0]; + peak->smoothFluxStdev = peak->smoothFlux / sqrt(significance->variance->data.F32[peak->y-row0][peak->x-col0]); + // NOTE smoothFluxStdev is actually (sqrt(variance) / covar_factor) + + // do we need this or not? + // peak->SN = sqrt(peak->detValue); + + if (readout->variance && isfinite (peak->dx)) { + peak->dx *= sqrt(readout->variance->data.F32[peak->y-row0][peak->x-col0]); + } + if (readout->variance && isfinite (peak->dy)) { + peak->dy *= sqrt(readout->variance->data.F32[peak->y-row0][peak->x-col0]); + } + } + + // limit the total number of returned peaks as specified + psArraySort (peaks, pmPeaksSortByRawFluxDescend); + if (nMax && (peaks->n > nMax)) { + psArray *tmpPeaks = psArrayAllocEmpty (nMax); + for (int i = 0; i < nMax; i++) { + psArrayAdd (tmpPeaks, 100, peaks->data[i]); + } + psFree (peaks); + peaks = tmpPeaks; + } + + // optional dump of all peak data + char *output = psMetadataLookupStr (&status, recipe, "PEAKS_OUTPUT_FILE"); + if (output && strcasecmp (output, "NONE")) { + pmPeaksWriteText (peaks, output); + } + psLogMsg ("psphot", PS_LOG_WARN, "%ld peaks: %f sec\n", peaks->n, psTimerMark ("psphot.peaks")); + + return peaks; + +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSet.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSet.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSet.c (revision 41083) @@ -0,0 +1,58 @@ +# include "psphotInternal.h" + +// This is only used by psphotModelTest.c +bool psphotFitSet (pmSource *source, pmModel *oneModel, char *fitset, pmSourceFitMode mode, psImageMaskType maskVal) { + + double x, y, Io; + + FILE *f = fopen (fitset, "r"); + if (f == NULL) return false; + + psArray *modelSet = psArrayAllocEmpty (16); + + while (fscanf (f, "%lf %lf %lf", &x, &y, &Io) == 3) { + pmModel *model = pmModelAlloc (oneModel->type); + + for (psS32 i = 0; i < model->params->n; i++) { + model->params->data.F32[i] = oneModel->params->data.F32[i]; + model->dparams->data.F32[i] = oneModel->dparams->data.F32[i]; + } + model->params->data.F32[1] = Io; + model->params->data.F32[2] = x; + model->params->data.F32[3] = y; + psArrayAdd (modelSet, 16, model); + } + + // Define source fitting parameters for extended source fits + pmSourceFitOptions *fitOptions = pmSourceFitOptionsAlloc(); + fitOptions->mode = PM_SOURCE_FIT_EXT; + fitOptions->covarFactor = 1.0; + // XXX for now, use the defaults for the rest: + // fitOptions->nIter = fitIter; + // fitOptions->tol = fitTol; + // fitOptions->poissonErrors = poisson; + // fitOptions->weight = PS_SQR(skySig); + + // XXX pmSourceFitSet must cache the modelFlux? + pmSourceFitSet (source, modelSet, fitOptions, maskVal); + + // write out positive object + psphotSaveImage (NULL, source->pixels, "object.fits"); + + // subtract object, leave local sky + for (int i = 0; i < modelSet->n; i++) { + pmModel *model = modelSet->data[i]; + pmModelSub (source->pixels, source->maskObj, model, PM_MODEL_OP_FULL, maskVal); + + fprintf (stderr, "output parameters (obj %d):\n", i); + for (int n = 0; n < model->params->n; n++) { + fprintf (stderr, "%d : %f\n", n, model->params->data.F32[n]); + } + } + + // write out + psphotSaveImage (NULL, source->pixels, "resid.fits"); + psphotSaveImage (NULL, source->maskObj, "mask.fits"); + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSourcesLinear.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSourcesLinear.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSourcesLinear.c (revision 41083) @@ -0,0 +1,718 @@ +# include "psphotInternal.h" + +// fit flux (and optionally sky model) to all reasonable sources +// with the linear fitting process. sources must have an associated +// model with selected pixels, and the fit radius must be defined + +// given the set of sources, each of which points to the pixels in the +// science image, we construct a set of simulated sources with their own pixels. +// these are used to determine the simultaneous linear fit of fluxes. +// the analysis is performed wrt the simulated pixel values + +static bool SetBorderMatrixElements (psSparseBorder *border, pmReadout *readout, psArray *sources, bool constant_weights, int SKY_FIT_ORDER, psImageMaskType markVal); + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotFitSourcesLinear (pmConfig *config, const pmFPAview *view, const char *filerule, bool final, bool skipNegativeFluxSources) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Fit Source (Linear) ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + pmSourceFitVarMode fitVarMode = psphotGetFitVarMode (recipe); + if (!fitVarMode) { + psError (PSPHOT_ERR_CONFIG, true, "failed to get LINEAR_FIT_VARIANCE_MODE"); + return false; + } + // MODEL_VAR requires 2 passes -- in the first, we get the rough fluxes; in the second, we + // use the flux to define the model variance before fitting the objects. Other modes only + // do a single pass. + pmSourceFitVarMode fitVarModePass1 = (fitVarMode == PM_SOURCE_PHOTFIT_MODEL_VAR) ? PM_SOURCE_PHOTFIT_CONST : fitVarMode; + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psLogMsg ("psphot", PS_LOG_DETAIL, "skipping fit sources for input file %d", i); + continue; + } + + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + if (!psphotFitSourcesLinearReadout (recipe, readout, sources, psf, final, fitVarModePass1, skipNegativeFluxSources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i); + return false; + } + + // the MODEL_VAR weighting scheme requires knowledge of the model fluxes to generate the variance + // after we have determined the initial set of fits, then we can generate the variance image and + // re-run the fit against that variance. + if (fitVarMode == PM_SOURCE_PHOTFIT_MODEL_VAR) { + // generate the model variance image & source pointers + if (!psphotGenerateModelVariance (config, view, file, i, recipe, readout, sources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i); + return false; + } + + // replace all sources (use TMPF_SUBTRACTED as test) + psphotReplaceAllSourcesReadout (config, view, filerule, i, recipe, false); + + // rerun fit with correct fitVarMode + // XXX: does skipNegativeFlux work here? + if (!psphotFitSourcesLinearReadout (recipe, readout, sources, psf, final, fitVarMode, skipNegativeFluxSources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i); + return false; + } + + // free the model variance image & source pointers + if (!psphotFreeModelVariance (readout, sources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to fit sources (linear) for %s entry %d", filerule, i); + return false; + } + } + + psphotVisualShowResidualImage (readout, (num > 0)); + psphotVisualShowPeaks (detections); + psphotVisualShowObjectRegions (readout, recipe, sources); + } + return true; +} + +// look up the fit variance mode from the recipe; older recipes do not have the value +// 'LINEAR_FIT_VARIANCE_MODE'; in those cases, look for 'CONSTANT_PHOTOMETRIC_WEIGHTS' as a boolean and +// set the value to either CONST or IMAGE_VAR +pmSourceFitVarMode psphotGetFitVarMode (psMetadata *recipe) { + + bool status = false; + + char *fitVarModeString = psMetadataLookupStr(&status, recipe, "LINEAR_FIT_VARIANCE_MODE"); + if (!status) { + bool CONSTANT_PHOTOMETRIC_WEIGHTS = psMetadataLookupBool(&status, recipe, "CONSTANT_PHOTOMETRIC_WEIGHTS"); + if (!status) { + psAbort("You must provide a value for LINEAR_FIT_VARIANCE_MODE or CONSTANT_PHOTOMETRIC_WEIGHTS"); + } + pmSourceFitVarMode fitVarMode = CONSTANT_PHOTOMETRIC_WEIGHTS ? PM_SOURCE_PHOTFIT_CONST : PM_SOURCE_PHOTFIT_IMAGE_VAR; + return fitVarMode; + } + if (!strcasecmp(fitVarModeString, "CONSTANT") || !strcasecmp(fitVarModeString, "CONST")) { + return PM_SOURCE_PHOTFIT_CONST; + } + if (!strcasecmp(fitVarModeString, "IMAGE") || !strcasecmp(fitVarModeString, "IMAGE_VAR")) { + return PM_SOURCE_PHOTFIT_IMAGE_VAR; + } + if (!strcasecmp(fitVarModeString, "SKY") || !strcasecmp(fitVarModeString, "MODEL_SKY")) { + return PM_SOURCE_PHOTFIT_MODEL_SKY; + } + if (!strcasecmp(fitVarModeString, "MODEL") || !strcasecmp(fitVarModeString, "MODEL_VAR")) { + return PM_SOURCE_PHOTFIT_MODEL_VAR; + } + psError (PSPHOT_ERR_CONFIG, false, "Invalid value for LINEAR_FIT_VARIANCE_MODE (%s)", fitVarModeString); + return PM_SOURCE_PHOTFIT_NONE; +} + +bool psphotFitSourcesLinearReadout (psMetadata *recipe, pmReadout *readout, psArray *sources, pmPSF *psf, bool final, pmSourceFitVarMode fitVarMode, bool skipNegativeFluxSources) { + bool status; + float x; + float y; + float f; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping linear fit"); + return true; + } + + psTimerStart ("psphot.linear"); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // source analysis is done in spatial order + sources = psArraySort (sources, pmSourceSortByY); + + // storage array for fitSources + psArray *fitSources = psArrayAllocEmpty (sources->n); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion AnalysisRegion = psRegionFromString (region); + AnalysisRegion = psRegionForImage (readout->image, AnalysisRegion); + if (psRegionIsNaN (AnalysisRegion)) psAbort("analysis region mis-defined"); + + int SKY_FIT_ORDER = psMetadataLookupS32(&status, recipe, "SKY_FIT_ORDER"); + if (!status) { + SKY_FIT_ORDER = 0; + } + bool SKY_FIT_LINEAR = psMetadataLookupBool(&status, recipe, "SKY_FIT_LINEAR"); + if (!status) { + SKY_FIT_LINEAR = false; + } + + float MIN_VALID_FLUX = psMetadataLookupF32(&status, recipe, "PSF_FIT_MIN_VALID_FLUX"); + if (!status) { + MIN_VALID_FLUX = 0.0; + } + if (skipNegativeFluxSources && MIN_VALID_FLUX < 0) { + MIN_VALID_FLUX = 0.0; + } + float MAX_VALID_FLUX = psMetadataLookupF32(&status, recipe, "PSF_FIT_MAX_VALID_FLUX"); + if (!status) { + MAX_VALID_FLUX = 1e+8; + } + + float cutModelSum = psMetadataLookupF32(&status, recipe, "PSF_FIT_MODEL_SUM_FRAC_CUT"); + float cutMaskedSum = psMetadataLookupF32(&status, recipe, "PSF_FIT_MASKED_SUM_FRAC_CUT"); + + // XXX test: choose a larger-than expected radius: + float covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "covariance factor: %f\n", covarFactor); + + // XXX do not apply covarFactor for the moment... + // covarFactor = 1.0; + + int Nsat = 0; + + // select the sources which will be used for the fitting analysis + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // psAssert (source->peak, "source without peak??"); + // psAssert (source->peak->footprint, "peak without footprint??"); + + if (final) { + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) continue; + } else { + // if (source->mode & PM_SOURCE_MODE_BLEND) continue; + } + + // turn this bit off and turn it on again if we pass this test + source->mode &= ~PM_SOURCE_MODE_LINEAR_FIT; + + // skip non-astronomical objects (very likely defects) + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // do not include CRs in the full ensemble fit + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue; + + // do not include MOMENTS_FAILURES in the fit + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + + // XXX count saturated stars + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + Nsat ++; + } + + // generate model for sources without, or skip if we can't + if (!source->modelFlux) { + if (!pmSourceCacheModel (source, maskVal)) continue; + } + + // save the original coords + x = source->peak->xf; + y = source->peak->yf; + + // is the source in the region of interest? + if (x < AnalysisRegion.x0) continue; + if (y < AnalysisRegion.y0) continue; + if (x > AnalysisRegion.x1) continue; + if (y > AnalysisRegion.y1) continue; + + // check the integral of the model : is it large enough? + // apply mask? + float modelSum = 0.0; + float maskedSum = 0.0; + for (int iy = 0; iy < source->modelFlux->numRows; iy++) { + for (int ix = 0; ix < source->modelFlux->numCols; ix++) { + modelSum += source->modelFlux->data.F32[iy][ix]; + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal) continue; + maskedSum += source->modelFlux->data.F32[iy][ix]; + } + } + if (modelSum == 0 || maskedSum == 0) { + // no pixels at all drop the modelPSF (probably only an issue for external sources + // which are created from a fake peak (full force)) + psFree(source->modelPSF); + continue; + } + bool isPSF = false; + pmModel *model = pmSourceGetModel (&isPSF, source); + + float Io = model->params->data.F32[PM_PAR_I0]; + model->params->data.F32[PM_PAR_I0] = 1.0; + float normFlux = model->class->modelFlux(model->params); + model->params->data.F32[PM_PAR_I0] = Io; + + // printf("%5d %4.3f %4.3f %4.3f\n", source->seq, normFlux, modelSum, maskedSum); + //float cut = .85; + //if (modelSum < cut * normFlux) continue; + //if (maskedSum < cut * normFlux) continue; + + // MEH add the log comments back in as trace instead, psphot 5? + if (modelSum < 0.8*normFlux) { + psTrace ("psphot", 5, "low-sig model @ %f, %f (%f masked sum, %f sum, %f normFlux, %f peak)\n", + source->peak->xf, source->peak->yf, maskedSum, modelSum, normFlux, source->peak->rawFlux); + } + if (maskedSum < 0.5*normFlux) { + psTrace ("psphot", 5, "worrying model @ %f, %f (%f masked sum, %f sum, %f normFlux, %f peak)\n", + source->peak->xf, source->peak->yf, maskedSum, modelSum, normFlux, source->peak->rawFlux); + } + if (modelSum < cutModelSum * normFlux) continue; + if (maskedSum < cutMaskedSum * normFlux) continue; + + // clear the 'mark' pixels and remask on the fit aperture + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, model->fitRadius, "OR", markVal); + + // we call this function multiple times. for the first time, we have only PSF models for all objects + // the second time has extended sources. If we ever fit the PSF model, we should raise this bit + source->mode |= PM_SOURCE_MODE_LINEAR_FIT; + if (isPSF) { + source->mode |= PM_SOURCE_MODE_PSFMODEL; + } + + psArrayAdd (fitSources, 100, source); + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built fitSources: %f sec (%ld objects)\n", psTimerMark ("psphot.linear"), sources->n); + + if (fitSources->n == 0) { + psFree(fitSources); + return true; + } + + // vectors to store stats for each object + // psVector *variance = psVectorAlloc (fitSources->n, PS_TYPE_F32); + psVector *errors = psVectorAlloc (fitSources->n, PS_TYPE_F32); + + // create the border matrix (includes the sparse matrix) + // for just sky: 1 row; for x,y terms: 3 rows + psSparse *sparse = psSparseAlloc (fitSources->n, 100); + int nBorder = (SKY_FIT_ORDER == 0) ? 1 : 3; + psSparseBorder *border = psSparseBorderAlloc (sparse, nBorder); + + // if fitVarMode is MODEL_VAR, then we need to generate the model image variance + // XXX we have two possibilities here: + + // 1) do 2 passes, where in the first case we use the CONST weighting, and in the second + // use the fitted model values to define the model + + // 2) do a single pass, and use the model guess to define the model variance (but do I trust the Model Guess?) + + // fill out the sparse matrix elements and border elements (B) + // SRCi is the current source of interest + // SRCj is a possibly overlapping source + for (int i = 0; i < fitSources->n; i++) { + pmSource *SRCi = fitSources->data[i]; + + // diagonal elements of the sparse matrix (auto-cross-product) + float MM = pmSourceModelDotModel (SRCi, SRCi, fitVarMode, covarFactor, maskVal); + psSparseMatrixElement (sparse, i, i, MM); + + // if we have used CONSTANT errors, then we need to re-calculate the value of the parameter error + if (fitVarMode != PM_SOURCE_PHOTFIT_IMAGE_VAR) { + float var = pmSourceModelDotModel (SRCi, SRCi, PM_SOURCE_PHOTFIT_IMAGE_VAR, covarFactor, maskVal); + errors->data.F32[i] = 1.0 / sqrt(var); + } else { + errors->data.F32[i] = 1.0 / sqrt(MM); + } + + // find the image x model value + float FM = pmSourceDataDotModel (SRCi, SRCi, fitVarMode, covarFactor, maskVal); + psSparseVectorElement (sparse, i, FM); + + // add the per-source variances (border region) + switch (SKY_FIT_ORDER) { + case 1: + f = pmSourceModelWeight (SRCi, 1, fitVarMode, covarFactor, maskVal); + psSparseBorderElementB (border, i, 1, f); + f = pmSourceModelWeight (SRCi, 2, fitVarMode, covarFactor, maskVal); + psSparseBorderElementB (border, i, 2, f); + + case 0: + f = pmSourceModelWeight (SRCi, 0, fitVarMode, covarFactor, maskVal); + psSparseBorderElementB (border, i, 0, f); + break; + + default: + psAbort("Invalid SKY_FIT_ORDER %d\n", SKY_FIT_ORDER); + break; + } + + // loop over all other stars following this one + for (int j = i + 1; j < fitSources->n; j++) { + pmSource *SRCj = fitSources->data[j]; + + // skip over disjoint source images, break after last possible overlap + if (SRCi->pixels->row0 + SRCi->pixels->numRows < SRCj->pixels->row0) break; + if (SRCj->pixels->row0 + SRCj->pixels->numRows < SRCi->pixels->row0) continue; + if (SRCi->pixels->col0 + SRCi->pixels->numCols < SRCj->pixels->col0) continue; + if (SRCj->pixels->col0 + SRCj->pixels->numCols < SRCi->pixels->col0) continue; + + // got an overlap; calculate cross-product and add to output array + f = pmSourceModelDotModel (SRCi, SRCj, fitVarMode, covarFactor, maskVal); + psSparseMatrixElement (sparse, j, i, f); + } + } + +# if (0) + static int npass = 0; + char name[128]; + FILE *f1 = NULL; + int fd = -1; + + snprintf (name, 128, "sparse.Aij.%02d.dat", npass); + f1 = fopen (name, "w"); + psAssert (f1, "failed to open file\n"); + fd = fileno (f1); + p_psVectorPrint (fd, sparse->Aij, "Aij"); + fclose (f1); + + snprintf (name, 128, "sparse.Bfj.%02d.dat", npass); + f1 = fopen (name, "w"); + psAssert (f1, "failed to open file\n"); + fd = fileno (f1); + p_psVectorPrint (fd, sparse->Bfj, "Bfj"); + fclose (f1); + + snprintf (name, 128, "sparse.Qii.%02d.dat", npass); + f1 = fopen (name, "w"); + psAssert (f1, "failed to open file\n"); + fd = fileno (f1); + p_psVectorPrint (fd, sparse->Qii, "Qii"); + fclose (f1); + npass ++; +# endif + + psSparseResort (sparse); + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built matrix: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + // set the sky, sky_x, sky_y components of border matrix + SetBorderMatrixElements (border, readout, fitSources, (fitVarMode == PM_SOURCE_PHOTFIT_CONST), SKY_FIT_ORDER, markVal); + + psSparseConstraint constraint; + constraint.paramMin = MIN_VALID_FLUX; + constraint.paramMax = MAX_VALID_FLUX; + constraint.paramDelta = 1e7; + + // solve for normalization terms (need include local sky?) + psVector *norm = NULL; + psVector *skyfit = NULL; + if (SKY_FIT_LINEAR) { + psSparseBorderSolve (&norm, &skyfit, constraint, border, 5); + psLogMsg ("psphot", PS_LOG_MINUTIA, "skyfit: %f\n", skyfit->data.F32[0]); + } else { + norm = psSparseSolve (NULL, constraint, sparse, 5); + skyfit = NULL; + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "solve matrix: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + // Philosophical question: we measure bright objects in three passes: 1) linear fit; 2) + // non-linear fit; 3) linear fit: should we retain the chisq and errors from the + // intermediate non-linear fit? the non-linear fit provides better values for the position + // errors, and for extended sources, the shape errors + + // adjust I0 for fitSources and subtract + for (int i = 0; i < fitSources->n; i++) { + pmSource *source = fitSources->data[i]; + pmModel *model = pmSourceGetModel (NULL, source); + + // assign linearly-fitted normalization + if (isnan(norm->data.F32[i])) { + psAbort("linear fitted source is nan"); + } + + model->params->data.F32[PM_PAR_I0] = norm->data.F32[i]; + model->dparams->data.F32[PM_PAR_I0] = errors->data.F32[i]; + + if (norm->data.F32[i] < MIN_VALID_FLUX) { + fprintf (stderr, "fit out of bounds for %f,%f : %f\n", source->peak->xf, source->peak->yf, norm->data.F32[i]); + model->params->data.F32[PM_PAR_I0] = MIN_VALID_FLUX; + } + + // clear the 'mark' pixels so the subtraction covers the full window + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // subtract object & add noise + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "sub models: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + // mean stats on fit windows + float sumRadius = 0.0; + float sumPixels = 0.0; + float sumSource = 0.0; + + // measure chisq for each source + // for (int i = 0; final && (i < fitSources->n); i++) { + for (int i = 0; i < fitSources->n; i++) { + pmSource *source = fitSources->data[i]; + pmModel *model = pmSourceGetModel (NULL, source); + + // accumulate fit windows statistics + sumRadius += model->fitRadius; + sumPixels += M_PI*PS_SQR(model->fitRadius); + sumSource += 1.0; + + if (!(source->mode & PM_SOURCE_MODE_NONLINEAR_FIT)) { + model->nPar = 1; // LINEAR-only sources have 1 parameter; NONLINEAR sources have their original value + } + pmSourceChisq (model, source->pixels, source->maskObj, source->variance, maskVal); + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "get chisqs: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + float meanRadius = sumRadius / sumSource; + float meanPixels = sumPixels / sumSource; + + // psFree (index); + psFree (sparse); + psFree (fitSources); + psFree (norm); + psFree (skyfit); + psFree (errors); + psFree (border); + + psLogMsg ("psphot.ensemble", PS_LOG_WARN, "measure ensemble of PSFs (mean radius = %f pixels, mean area = %f pixels: %f sec\n", meanRadius, meanPixels, psTimerMark ("psphot.linear")); + + psphotVisualPlotChisq (sources); + // psphotVisualShowFlags (sources); + + // We have to place this visualization here because the models are not realized until + // psphotGuessModels or fitted until psphotFitSourcesLinear. + // psphotVisualShowPSFStars (recipe, psf, sources); + + return true; +} + +// Calculate the weight terms for the sky fit component of the matrix. This function operates +// on the pixels which correspond to all of the sources of interest. These elements fill in +// the border matrix components in the sparse matrix equation. +static bool SetBorderMatrixElements (psSparseBorder *border, pmReadout *readout, psArray *sources, bool constant_weights, int SKY_FIT_ORDER, psImageMaskType markVal) { + + // generate the image-wide weight terms + // turn on MARK for all image pixels + psRegion fullArray = psRegionSet (0, 0, 0, 0); + fullArray = psRegionForImage (readout->mask, fullArray); + psImageMaskRegion (readout->mask, fullArray, "OR", markVal); + + // turn off MARK for all object pixels + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + pmModel *model = pmSourceGetModel (NULL, source); + if (model == NULL) continue; + float x = model->params->data.F32[PM_PAR_XPOS]; + float y = model->params->data.F32[PM_PAR_YPOS]; + psImageMaskCircle (source->maskView, x, y, model->fitRadius, "AND", PS_NOT_IMAGE_MASK(markVal)); + } + + // accumulate the image statistics from the masked regions + psF32 **image = readout->image->data.F32; + psF32 **variance = readout->variance->data.F32; + psImageMaskType **mask = readout->mask->data.PS_TYPE_IMAGE_MASK_DATA; + + double w, x, y, x2, xy, y2, xc, yc, wt, f, fo, fx, fy; + w = x = y = x2 = xy = y2 = fo = fx = fy = 0; + + int col0 = readout->image->col0; + int row0 = readout->image->row0; + + for (int j = 0; j < readout->image->numRows; j++) { + for (int i = 0; i < readout->image->numCols; i++) { + if (mask[j][i]) continue; + if (constant_weights) { + wt = 1.0; + } else { + wt = variance[j][i]; + } + f = image[j][i]; + w += 1/wt; + fo += f/wt; + if (SKY_FIT_ORDER == 0) continue; + + xc = i + col0; + yc = j + row0; + x += xc/wt; + y += yc/wt; + x2 += xc*xc/wt; + xy += xc*yc/wt; + y2 += yc*yc/wt; + fx += f*xc/wt; + fy += f*yc/wt; + } + } + + // turn off MARK for all image pixels + psImageMaskRegion (readout->mask, fullArray, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // set the Border T elements + psSparseBorderElementG (border, 0, fo); + psSparseBorderElementT (border, 0, 0, w); + if (SKY_FIT_ORDER > 0) { + psSparseBorderElementG (border, 0, fx); + psSparseBorderElementG (border, 0, fy); + psSparseBorderElementT (border, 1, 0, x); + psSparseBorderElementT (border, 2, 0, y); + psSparseBorderElementT (border, 0, 1, x); + psSparseBorderElementT (border, 1, 1, x2); + psSparseBorderElementT (border, 2, 1, xy); + psSparseBorderElementT (border, 0, 2, y); + psSparseBorderElementT (border, 1, 2, xy); + psSparseBorderElementT (border, 2, 2, y2); + } + + return true; +} + +bool psphotModelBackgroundReadout(psImage *model, // Model image + psImage *modelStdev, // Model stdev image + psMetadata *analysis, // Analysis metadata for outputs + pmReadout *readout, // Readout for which to generate a background model + psImageBinning *binning, // Binning parameters + const pmConfig *config,// Configuration + bool useVarianceImage + ); + + bool psphotGenerateModelVariance (pmConfig *config, const pmFPAview *view, pmFPAfile *file, int index, psMetadata *recipe, pmReadout *readout, psArray *sources) { + + bool status = false; + psRegion fullRegion = psRegionSet (0, 0, 0, 0); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // create a model variance image + psImage *modelVar = psImageCopy (NULL, readout->variance, PS_TYPE_F32); + + // find the binning information + psImageBinning *backBinning = psphotBackgroundBinning (modelVar, config); + assert (backBinning); + + psImage *varModel = psImageAlloc(backBinning->nXruff, backBinning->nYruff, PS_TYPE_F32); // Background model + psImage *varModelStdev = psImageAlloc(backBinning->nXruff, backBinning->nYruff, PS_TYPE_F32); // Background model + + // generate an image of the mean variance image in DN + if (!psphotModelBackgroundReadout(varModel, varModelStdev, NULL, readout, backBinning, config, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate background model"); + psFree (varModel); + psFree (varModelStdev); + return false; + } + + // linear interpolation to full-scale + if (!psImageUnbin (modelVar, varModel, backBinning)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning"); + psFree (varModel); + psFree (varModelStdev); + return false; + } + psFree (varModel); + psFree (varModelStdev); + + float gain = 1.0; // accept 1.0 as a default since it is not critical to the analysis + pmCell *cell = readout->parent; // The parent cell + if (cell) { + gain = psMetadataLookupF32(&status, cell->concepts, "CELL.GAIN"); // Cell gain + if (!status) { + gain = 1.0; // set note above + } + } + if (gain > 2.0) { /* warn? */ } + // XXX we are not actually using the gain, but need to test it to avoid gcc pedantic warnings + + // insert all of the source models + for (int i = 0; i < sources->n; i++) { + + // source of interest + pmSource *source = sources->data[i]; + + // skip sources which were not fitted already + if (!(source->mode & PM_SOURCE_MODE_LINEAR_FIT)) continue; + + // pixel region appropriate for the source + psRegion region = psRegionForImage (source->pixels, fullRegion); + + // define the source->modelVar pixels (view on modelVar image) + psAssert (!source->modelVar, "programming error : modelVar should be NULL here"); + psAssert (source->modelFlux, "programming error : modelFlux should not be NULL here"); + psAssert (source->modelFlux->data.F32, "programming error : modelFlux should not be NULL here"); + source->modelVar = psImageSubset(modelVar, region); + + // add the source model to the model variance image + // XXX note that this should be added with gain applied + // var_DN = flux_DN / gain [e/DN] + // to do this requires an API upgrade... + pmSourceAdd (source, PM_MODEL_OP_MODELVAR, maskVal); + } + + // we save the model variance for future reference + psMetadataAddImage(readout->analysis, PS_LIST_TAIL, "PSPHOT.MODEL.VAR", PS_META_REPLACE, "model variance", modelVar); + psFree (modelVar); + + return true; +} + +bool psphotFreeModelVariance (pmReadout *readout, psArray *sources) { + + bool status = false; + + // find the binning information + psImage *modelVar = psMetadataLookupPtr(&status, readout->analysis, "PSPHOT.MODEL.VAR"); + assert (modelVar); + + if (modelVar) { + psMetadataRemoveKey (readout->analysis, "PSPHOT.MODEL.VAR"); + } + + // clear modelVar pointers for all of the source models + for (int i = 0; i < sources->n; i++) { + + // source of interest + pmSource *source = sources->data[i]; + psFree (source->modelVar); + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSourcesLinearStack.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSourcesLinearStack.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFitSourcesLinearStack.c (revision 41083) @@ -0,0 +1,225 @@ +# include "psphotInternal.h" +// XXX need array of covar factors for each image +// XXX define the 'good' / 'bad' flags? + +# define COVAR_FACTOR 1.0 + +bool psphotFitSourcesLinearStack (pmConfig *config, psArray *objects, bool final) { + + bool status; + float f; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + psTimerStart ("psphot.linear"); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // analysis is done in spatial order (to speed up overlap search) + objects = psArraySort (objects, pmPhotObjSortByX); + + // storage array for fitSources + psArray *fitSources = psArrayAllocEmpty (objects->n); + + bool CONSTANT_PHOTOMETRIC_WEIGHTS = psMetadataLookupBool(&status, recipe, "CONSTANT_PHOTOMETRIC_WEIGHTS"); + psAssert (status, "You must provide a value for the BOOL recipe CONSTANT_PHOTOMETRIC_WEIGHTS"); + + float MIN_VALID_FLUX = psMetadataLookupF32(&status, recipe, "PSF_FIT_MIN_VALID_FLUX"); + if (!status) { + MIN_VALID_FLUX = 0.0; + } + float MAX_VALID_FLUX = psMetadataLookupF32(&status, recipe, "PSF_FIT_MAX_VALID_FLUX"); + if (!status) { + MAX_VALID_FLUX = 1e+8; + } + + // XXX store a local static array of covar factors for each of the images (by image ID) + // float covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + // psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "covariance factor: %f\n", covarFactor); + + // select the sources which will be used for the fitting analysis + for (int i = 0; i < objects->n; i++) { + pmPhotObj *object = objects->data[i]; + if (!object) continue; + if (!object->sources) continue; + + // XXX check an element of the group to see if we should use it + // if (!object->flags & PM_PHOT_OBJ_BAD) continue; + + for (int j = 0; j < object->sources->n; j++) { + pmSource *source = object->sources->data[j]; + if (!source) continue; + + // turn this bit off and turn it on again if we keep this source + source->mode &= ~PM_SOURCE_MODE_LINEAR_FIT; + + // skip non-astronomical objects (very likely defects) + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + + // do not include CRs in the full ensemble fit + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue; + + // do not include MOMENTS_FAILURES in the fit + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + + if (final) { + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) continue; + } else { + // if (source->mode & PM_SOURCE_MODE_BLEND) continue; + } + + // generate model for sources without, or skip if we can't + if (!source->modelFlux) { + if (!pmSourceCacheModel (source, maskVal)) continue; + } + + // check the integral of the model : is it large enough? + float modelSum = 0.0; + for (int iy = 0; iy < source->modelFlux->numRows; iy++) { + for (int ix = 0; ix < source->modelFlux->numCols; ix++) { + modelSum += source->modelFlux->data.F32[iy][ix]; + } + } + if (modelSum < 0.5) continue; // skip sources with no model constraint (somewhat arbitrary limit) + if (modelSum < 0.8) { + fprintf (stderr, "low-sig model @ %f, %f (%f sum, %f peak)\n", + source->peak->xf, source->peak->yf, modelSum, source->peak->rawFlux); + } + + bool isPSF = false; + pmModel *model = pmSourceGetModel (&isPSF, source); + + // clear the 'mark' pixels and remask on the fit aperture + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, model->fitRadius, "OR", markVal); + + // we call this function multiple times. for the first time, we have only PSF models for all objects + // the second time has extended sources. If we ever fit the PSF model, we should raise this bit + source->mode |= PM_SOURCE_MODE_LINEAR_FIT; + if (isPSF) { + source->mode |= PM_SOURCE_MODE_PSFMODEL; + } + psArrayAdd (fitSources, 100, source); + } + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built fitSources: %f sec (%ld objects)\n", psTimerMark ("psphot.linear"), objects->n); + + if (fitSources->n == 0) { + return true; + } + + // vectors to store stats for each object + // psVector *variance = psVectorAlloc (fitSources->n, PS_TYPE_F32); + psVector *errors = psVectorAlloc (fitSources->n, PS_TYPE_F32); + + // create the sparse matrix + psSparse *sparse = psSparseAlloc (fitSources->n, 100); + + // fill out the sparse matrix elements and border elements (B) + // SRCi is the current source of interest + // SRCj is a possibly overlapping source + for (int i = 0; i < fitSources->n; i++) { + pmSource *SRCi = fitSources->data[i]; + + // diagonal elements of the sparse matrix (auto-cross-product) + f = pmSourceModelDotModel (SRCi, SRCi, CONSTANT_PHOTOMETRIC_WEIGHTS, COVAR_FACTOR, maskVal); + psSparseMatrixElement (sparse, i, i, f); + + // the formal error depends on the weighting scheme + if (CONSTANT_PHOTOMETRIC_WEIGHTS) { + float var = pmSourceModelDotModel (SRCi, SRCi, false, COVAR_FACTOR, maskVal); + errors->data.F32[i] = 1.0 / sqrt(var); + } else { + errors->data.F32[i] = 1.0 / sqrt(f); + } + + // find the image x model value + f = pmSourceDataDotModel (SRCi, SRCi, CONSTANT_PHOTOMETRIC_WEIGHTS, COVAR_FACTOR, maskVal); + psSparseVectorElement (sparse, i, f); + + // loop over all other stars following this one + for (int j = i + 1; j < fitSources->n; j++) { + pmSource *SRCj = fitSources->data[j]; + + // we only need to generate dot terms for source on the same image + if (SRCj->imageID != SRCi->imageID) { continue; } + + // skip over disjoint source images, break after last possible overlap + if (SRCj->pixels->row0 + SRCj->pixels->numRows < SRCi->pixels->row0) continue; // source(i) is above source(j) + if (SRCi->pixels->row0 + SRCi->pixels->numRows < SRCj->pixels->row0) continue; // source(i) is below source(j) + if (SRCj->pixels->col0 + SRCj->pixels->numCols < SRCi->pixels->col0) continue; // source(i) is right of source(j) + if (SRCi->pixels->col0 + SRCi->pixels->numCols < SRCj->pixels->col0) break; // source(i) is left of source(j) [no other source(j) can overlap source(i)] + + // got an overlap; calculate cross-product and add to output array + f = pmSourceModelDotModel (SRCi, SRCj, CONSTANT_PHOTOMETRIC_WEIGHTS, COVAR_FACTOR, maskVal); + psSparseMatrixElement (sparse, j, i, f); + } + } + + psSparseResort (sparse); + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built matrix: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + psSparseConstraint constraint; + constraint.paramMin = MIN_VALID_FLUX; + constraint.paramMax = MAX_VALID_FLUX; + constraint.paramDelta = 1e7; + + // solve for normalization terms (need include local sky?) + psVector *norm = NULL; + norm = psSparseSolve (NULL, constraint, sparse, 5); + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "solve matrix: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + // adjust I0 for fitSources and subtract + for (int i = 0; i < fitSources->n; i++) { + pmSource *source = fitSources->data[i]; + pmModel *model = pmSourceGetModel (NULL, source); + + // assign linearly-fitted normalization + if (isnan(norm->data.F32[i])) { + psAbort("linear fitted source is nan"); + } + + model->params->data.F32[PM_PAR_I0] = norm->data.F32[i]; + model->dparams->data.F32[PM_PAR_I0] = errors->data.F32[i]; + + // clear the 'mark' pixels so the subtraction covers the full window + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // subtract object + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "sub models: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + // measure chisq for each source + // for (int i = 0; final && (i < fitSources->n); i++) { + for (int i = 0; i < fitSources->n; i++) { + pmSource *source = fitSources->data[i]; + pmModel *model = pmSourceGetModel (NULL, source); + if (!(source->mode & PM_SOURCE_MODE_NONLINEAR_FIT)) { + model->nPar = 1; // LINEAR-only sources have 1 parameter; NONLINEAR sources have their original value + } + pmSourceChisq (model, source->pixels, source->maskObj, source->variance, maskVal); + } + psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "get chisqs: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem); + + // psFree (index); + psFree (sparse); + psFree (fitSources); + psFree (norm); + psFree (errors); + + psLogMsg ("psphot.ensemble", PS_LOG_WARN, "measure ensemble of PSFs: %f sec\n", psTimerMark ("psphot.linear")); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFootprintSaddles.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFootprintSaddles.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFootprintSaddles.c (revision 41083) @@ -0,0 +1,129 @@ +# include "psphotInternal.h" +# ifndef ROUND +# define ROUND(X) ((int) ((X) + 0.5*SIGN(X))) +# endif + +psVector *getSaddlePoint (pmReadout *readout, int X1, int Y1, int X2, int Y2, bool swapcoords); + +// for each footprint with multiple detections, find the complete set of saddles, that is, the +// local minima which lie on the lines between the peaks (same as a 'Coll' but less homophonic +// with 'cull') +bool psphotFootprintSaddles(pmReadout *readout, psArray *footprints) { // array of pmFootprints + +# if (PM_PEAKS_CULL_WITH_SMOOTHED_IMAGE) + psLogMsg ("psphot", PS_LOG_INFO, "Finding saddle points from footprints using the smoothed image"); +# else + psLogMsg ("psphot", PS_LOG_INFO, "Finding saddle points from footprints using the raw (unsmoothed) image"); +# endif + + int xMax = readout->image->numCols - 1; + int yMax = readout->image->numRows - 1; + + // FILE *f = fopen ("saddlepoints.dat", "w"); + + for (int i = 0; i < footprints->n; i++) { + pmFootprint *fp = footprints->data[i]; + if (fp->peaks == NULL) continue; + if (fp->peaks->n < 2) continue; + + // loop over all peaks in the footprint: + for (int is = 0; is < fp->peaks->n; is++) { + + pmPeak *startPeak = fp->peaks->data[is]; + if (!startPeak->saddlePoints) startPeak->saddlePoints = psArrayAllocEmpty (4); + + // loop over all peaks in the footprint, skipping this one + for (int ie = is + 1; ie < fp->peaks->n; ie++) { + if (ie == is) continue; + + pmPeak *endPeak = fp->peaks->data[ie]; + if (!endPeak->saddlePoints) endPeak->saddlePoints = psArrayAllocEmpty (4); + + // step across all pixels between the startPeak and the endPeak, using a + // bresenham-like line-drawing operation + + int X1 = PS_MIN(xMax, PS_MAX(0, ROUND(startPeak->xf))); + int Y1 = PS_MIN(yMax, PS_MAX(0, ROUND(startPeak->yf))); + int X2 = PS_MIN(xMax, PS_MAX(0, ROUND(endPeak->xf))); + int Y2 = PS_MIN(yMax, PS_MAX(0, ROUND(endPeak->yf))); + + int dX = X2 - X1; + int dY = Y2 - Y1; + + bool FlipCoords = (abs(dX) < abs(dY)); + bool FlipDirect = FlipCoords ? (Y1 > Y2) : (X1 > X2); + + psVector *saddlePoint = NULL; + if (!FlipDirect && !FlipCoords) saddlePoint = getSaddlePoint (readout, X1, Y1, X2, Y2, false); + if ( FlipDirect && !FlipCoords) saddlePoint = getSaddlePoint (readout, X2, Y2, X1, Y1, false); + if (!FlipDirect && FlipCoords) saddlePoint = getSaddlePoint (readout, Y1, X1, Y2, X2, true); + if ( FlipDirect && FlipCoords) saddlePoint = getSaddlePoint (readout, Y2, X2, Y1, X1, true); + psAssert (saddlePoint, "failed to get saddlePoint"); + + // fprintf (f, "%f %f | %d %d | %f %f\n", startPeak->xf, startPeak->yf, saddlePoint->data.S32[0], saddlePoint->data.S32[1], endPeak->xf, endPeak->yf); + + psArrayAdd (startPeak->saddlePoints, 4, saddlePoint); + psArrayAdd (endPeak->saddlePoints, 4, saddlePoint); + psFree (saddlePoint); + // NOTE : all saddlePoints for the startPeak are also saddlePoints for the endPeak + } + } + } + // fclose (f); + return true; +} + +// psLogMsg ("psphot", PS_LOG_INFO, "%ld peaks, %ld total footprints: %f sec\n", detections->peaks->n, detections->footprints->n, + +psVector *getSaddlePoint (pmReadout *readout, int X1, int Y1, int X2, int Y2, bool swapcoords) { + + psAssert (X1 >= 0, "invalid"); + psAssert (Y1 >= 0, "invalid"); + psAssert (X1 < readout->image->numCols, "invalid"); + psAssert (Y1 < readout->image->numRows, "invalid"); + + psAssert (X2 >= 0, "invalid"); + psAssert (Y2 >= 0, "invalid"); + psAssert (X2 < readout->image->numCols, "invalid"); + psAssert (Y2 < readout->image->numRows, "invalid"); + + psVector *saddlePoint = psVectorAlloc(2, PS_TYPE_F32); + + int dX = X2 - X1; + int dY = Y2 - Y1; + + float minValue = swapcoords ? readout->image->data.F32[X1][Y1] : readout->image->data.F32[Y1][X1]; + saddlePoint->data.S32[0] = swapcoords ? Y1 : X1; + saddlePoint->data.S32[1] = swapcoords ? X1 : Y1; + + int Y = Y1; + int e = 0; + for (int X = X1; X <= X2; X++) { + if (swapcoords) { + float newValue = readout->image->data.F32[X][Y]; + if (newValue < minValue) { + minValue = newValue; + saddlePoint->data.S32[0] = Y; + saddlePoint->data.S32[1] = X; + } + } else { + float newValue = readout->image->data.F32[Y][X]; + if (newValue < minValue) { + minValue = newValue; + saddlePoint->data.S32[0] = X; + saddlePoint->data.S32[1] = Y; + } + } + e += dY; + int e2 = 2 * e; + if (e2 > dX) { + Y++; + e -= dX; + } + if (e2 < -dX) { + Y--; + e += dX; + } + } + return (saddlePoint); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForced.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForced.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForced.c (revision 41083) @@ -0,0 +1,35 @@ +# include "psphotStandAlone.h" + +int main (int argc, char **argv) { + + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotForcedArguments (argc, argv); + assert(config); + + psphotVersionPrint(); + + // load input data (config and images (signal, noise, mask) + if (!psphotParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotImageLoop (config, PSPHOT_FORCED)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", 3, "complete psphot run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedArguments.c (revision 41083) @@ -0,0 +1,199 @@ +# include "psphotStandAlone.h" + +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile) +{ + fprintf(ofile, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "%s -help\n" + "\n" + "%s -version\n" + "\n" + "where:\n" + " FileNameList is a text file containing filenames, one per line\n" + " MaskFileNameList is a text file of mask filenames, one per line\n" + " VarFileNameList is a text file of variance filenames, one per line\n" + " OutFileBaseName is the 'root name' for output files\n" + "\n" + "additional options:\n" + " -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n" + " specify PSF rather than letting %s estimate it\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify additional sources for PSF generation\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -photcode PhotoCodeName\n" + " specify photocode\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -nthreads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n", + program,program,program,program,program); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "Try '%s -help' for more options and explanation\n", + program,program,program); + if (exitCode != PS_EXIT_SUCCESS) + psErrorStackPrint(stderr, "Error reading arguments\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + +pmConfig *psphotForcedArguments(int argc, char **argv) { + + int N; + bool status, status1, status2; + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + // Number of threads is handled + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + if (psArgumentGet(argc, argv, "-help") || + psArgumentGet(argc, argv, "-h")) + writeHelpInfo(argv[0], config, stdout); + + // visual : interactive display mode + if ((N = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (N, &argc, argv); + pmVisualSetVisual(true); + } + + // break : used from recipe throughout psphotReadout + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF + if ((N = psArgumentGet (argc, argv, "-region"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + // + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); + + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "PSPHOT.PSF", "-psf", "-psflist"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, true, "No psf model is supplied (use -psf)"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + status1 = pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC", "-src", "-srclist"); + status2 = pmConfigFileSetsMD (config->arguments, &argc, argv, "SRCTEXT", "-srctext", "-srctextlist"); + + if (!status1 && !status2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "No source list is supplied (use one of -src, -srctext, -srclist, or -srctextlist)"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc == 1) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotForcedArguments...\n"); + return (config); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedImageLoop.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedImageLoop.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedImageLoop.c (revision 41083) @@ -0,0 +1,153 @@ +# include "psphotStandAlone.h" + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +bool psphotForcedImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *load = psMetadataLookupPtr (&status, config->files, "PSPHOT.LOAD"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + pmHDU *lastHDU = NULL; // Last HDU updated + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + psImageMaskType maskTest = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, load->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + // load just the input image data (image, mask, weight) + pmFPAfileActivate (config->files, false, NULL); + pmFPAfileActivate (config->files, true, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, true, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, true, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // mosaic the cells of a chip into a single contiguous (trimmed) chip + if (!psphotMosaicChip(config, view, "PSPHOT.INPUT", "PSPHOT.LOAD")) ESCAPE ("Unable to mosaic chip."); + + // Read WCS if easy. + // XXX Since we're mosaicking cells, we ignore the case where the WCS is defined for a cell. + { + pmChip *inChip = pmFPAviewThisChip(view, input->fpa); // Mosaicked chip + pmHDU *hduLow = pmHDUGetLowest(input->fpa, inChip, NULL); + if (hduLow && !pmAstromReadWCS(input->fpa, inChip, hduLow->header, 1.0)) { + psWarning("Unable to read WCS astrometry from header."); + psErrorClear(); + pmHDU *hduHigh = pmHDUGetHighest(input->fpa, inChip, NULL); + if (hduHigh && hduHigh != hduLow && + !pmAstromReadWCS(input->fpa, chip, hduHigh->header, 1.0)) { + psWarning("Unable to read WCS astrometry from primary header."); + psErrorClear(); + } + } + } + + // try to load other supporting data (PSF, SRC, etc). + // do not re-load the following three files + pmFPAfileActivate (config->files, true, NULL); + pmFPAfileActivate (config->files, false, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, false, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, false, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // re-activate files so they will be closed and freed below + pmFPAfileActivate (config->files, true, NULL); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + // Update the header + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + psphotVersionHeaderFull(hdu->header); + lastHDU = hdu; + } + + // if an external mask is supplied, ensure that NAN pixels are also masked + if (readout->mask) { + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); // Mask value for saturated pixels + if (!pmReadoutMaskInvalid(readout, maskTest, maskSat)) { + psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels."); + psFree(view); + return false; + } + } + + // run the actual photometry analysis on this chip/cell/readout + if (!psphotForcedReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + } + + // drop all versions of the internal files + status = true; + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files"); + psFree (view); + return false; + } + } + + // save output which is saved at the chip level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed output for Chip in psphot."); + } + // save output which is saved at the fpa level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed ouput for FPA in psphot."); + + // fail if we failed to handle an error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + return true; +} + +// I/O files related to psphot: +// PSPHOT.INPUT : input image file(s) +// PSPHOT.RESID : residual image +// PSPHOT.OUTPUT : output object tables (object) + +// PSPHOT.BACKSUB : background subtracted image +// PSPHOT.BACKGND : background model (full-scale image?) +// PSPHOT.BACKMDL : background model (binned image?) +// PSPHOT.PSF : sample PSF images + +// PSPHOT.MASK +// PSPHOT.VARIANCE +// Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotForcedReadout.c (revision 41083) @@ -0,0 +1,90 @@ +# include "psphotInternal.h" + +bool psphotForcedReadout(pmConfig *config, const pmFPAview *view, const char *filerule) { + + // measure the total elapsed time in psphotReadout. XXX the current threading plan + // for psphot envisions threading within psphotReadout, not multiple threads calling + // the same psphotReadout. In the current plan, this dtime is the elapsed time used + // jointly by the multiple threads, not the total time used by all threads. + psTimerStart ("psphotReadout"); + + // allow objects to be fit with ugly models (central holes, extreme asymmetry, etc) + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + // set the photcode for this image + if (!psphotAddPhotcode (config, view, filerule)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // optional break-point for processing + char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); + PS_ASSERT_PTR_NON_NULL (breakPt, false); + + // Generate the mask and weight images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + if (!strcasecmp (breakPt, "NOTHING")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!psphotSubtractBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "BACKMDL")) { + return psphotReadoutCleanup (config, view, filerule); + } + + if (!psphotLoadPSF (config, view, filerule)) { + // this only happens if we had a programming error in psphotLoadPSF + psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // include externally-supplied sources + psphotLoadExtSources (config, view, filerule); + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, filerule); + + // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) + psphotFitSourcesLinear (config, view, filerule, false, false); + + // identify CRs and extended sources + // XXX do I want to do this step? + // XXX do I want to / need to calculate the moments? + // psphotSourceSize (config, readout, sources, recipe, psf, 0); + + // calculate source magnitudes + psphotMagnitudes(config, view, filerule); + + // XXX do I want to do this? + // if (!psphotEfficiency(config, readout, view, psf, recipe, sources)) { + // psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources"); + // psErrorClear(); + // } + + // replace background in residual image + psphotSkyReplace (config, view, filerule); + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, filerule); + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForce.SourceStats.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForce.SourceStats.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForce.SourceStats.c (revision 41083) @@ -0,0 +1,205 @@ +# include "psphotInternal.h" + +// construct pixels and measure moments for sources. +// NOTE: this function differs from psphotSourceStats in the following ways: +// 1) detections->allSources are used instead of newSources +// 2) the sources are used directly (peaks are not assigned to sources here) + +// NOTE: this function is meant to be used by psphotFullForce, and the mode of the loaded +// sources should have (MODE_EXTERNAL) set. If so, the Mx,My values will not be recalculated + +bool psphotFullForceSourceStats (pmConfig *config, const pmFPAview *view, const char *filerule, bool setWindow) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Source Stats ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + // int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + // if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + // if (i == chisqNum) continue; // skip chisq image + if (!psphotFullForceSourceStatsReadout (config, view, filerule, i, recipe, setWindow)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to find initial detections for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotFullForceSourceStatsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool setWindow) { + + bool status = false; + + psTimerStart ("psphot.stats"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psTraceGetLevel("psphot") > 5) { + static int pass = 0; + char name[64]; + sprintf (name, "srstats.v%d.fits", pass); + psphotSaveImage(NULL, readout->image, name); + pass ++; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // determine properties (sky, moments) of initial sources + float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + psAssert (status, "missing SKY_OUTER_RADIUS in recipe?"); + + // XXX this seems like an arbitrary number... + OUTER = PS_MAX(OUTER, 20.0); // XXX Guarantee that we can encompass the max moments radius + + float INNER = psMetadataLookupF32 (&status, recipe, "SKY_INNER_RADIUS"); psAssert (status, "missing SKY_INNER_RADIUS"); + float MIN_SN = psMetadataLookupF32 (&status, recipe, "MOMENTS_SN_MIN"); psAssert (status, "missing MOMENTS_SN_MIN"); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + psAssert (maskVal, "missing MASK.PSPHOT"); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + psAssert (markVal, "missing MARK.PSPHOT"); + + // psArray *sources = detections->allSources; + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + + // generate the array of sources, define the associated pixel + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + psAssert (source->peak, "how can we have a peak-less source?"); + psAssert (source->moments, "how can we have a moments-less source?"); + + // allocate image, weight, mask arrays for each peak (square of radius OUTER) + pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); + } + + if (setWindow) { + // calculate the best gaussian window (keep centroid positions) + // NOTE: if sources->mode,mode2 has MODE_EXTERNAL,MODE2_MATCHED, Mx,My are not changed + if (!psphotSetMomentsWindow(recipe, readout->analysis, sources, maskVal)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Failed to determine Moments Window!"); + psFree(detections->newSources); + return false; + } + + int error = psErrorCodeLast(); + if (error) { + psLogMsg ("psphot", PS_LOG_WARN, "Clearing error %d that psphotSetMomentsWindow left on the error stack.", error); + psErrorClear(); + } + } + + // if we have measured the window, we will be saving the modified version of these recipe values on readout->analysis + float SIGMA = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + SIGMA = psMetadataLookupF32 (&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float RADIUS = psMetadataLookupF32 (&status, readout->analysis, "PSF_MOMENTS_RADIUS"); + if (!status) { + RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_MOMENTS_RADIUS"); + } + float MIN_KRON_RADIUS = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_MIN_KRON"); + if (!status) { + MIN_KRON_RADIUS = 0.75*SIGMA; + } + + // threaded measurement of the source magnitudes + int Nfail = 0; + int Nmoments = 0; + int Nfaint = 0; + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_SOURCE_STATS"); + + psArrayAdd(job->args, 1, cells->data[j]); // sources + + PS_ARRAY_ADD_SCALAR(job->args, INNER, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, MIN_SN, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, RADIUS, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, SIGMA, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, MIN_KRON_RADIUS, PS_TYPE_F32); + + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nmoments + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfaint + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to launch thread job PSPHOT_SOURCE_STATS"); + psFree(detections->newSources); + return false; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS"); + psFree(detections->newSources); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[8]; + Nmoments += scalar->data.S32; + scalar = job->args->data[9]; + Nfail += scalar->data.S32; + scalar = job->args->data[10]; + Nfaint += scalar->data.S32; + } + psFree(job); + } + } + + psFree (cellGroups); + + psLogMsg ("psphot", PS_LOG_WARN, "%ld sources, %d moments, %d faint, %d failed: %f sec\n", sources->n, Nmoments, Nfaint, Nfail, psTimerMark ("psphot.stats")); + + psphotVisualShowMoments (sources); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForce.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForce.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForce.c (revision 41083) @@ -0,0 +1,40 @@ +# include "psphotStandAlone.h" + +int main (int argc, char **argv) { + + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotFullForceArguments (argc, argv); + assert(config); + + if (!psphotStatsFileOpen(config)) { + psErrorStackPrint(stderr, "Error creating statsFile\n"); + exit (psphotGetExitStatus()); + } + + psphotVersionPrint(); + + // load input data (config and images (signal, noise, mask) + if (!psphotParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotImageLoop (config, PSPHOT_FULL_FORCE)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", 3, "complete psphotFullForce run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceArguments.c (revision 41083) @@ -0,0 +1,209 @@ +# include "psphotStandAlone.h" + +static void usage(const char *program, psMetadata *arg, pmConfig *config, int exitCode); +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile); + +pmConfig *psphotFullForceArguments(int argc, char **argv) { + + int N; + bool status, status1, status2, status3; + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + PS_ARGUMENTS_GENERIC (psphot, config, argc, argv); + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + // Number of threads is handled + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ); + + if (psArgumentGet(argc, argv, "-help")) writeHelpInfo(argv[0], config, stdout); + if (psArgumentGet(argc, argv, "-h")) writeHelpInfo(argv[0], config, stdout); + + // visual : interactive display mode + if ((N = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (N, &argc, argv); + pmVisualSetVisual(true); + } + + // break : used from recipe throughout psphotReadout + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF + if ((N = psArgumentGet (argc, argv, "-region"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + if ((N = psArgumentGet(argc, argv, "-stats"))) { + psArgumentRemove(N, &argc, argv); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "STATS", PS_DATA_STRING, + "Filename for statistics of output file", argv[N]); + psArgumentRemove(N, &argc, argv); + } + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + // + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); + + pmConfigFileSetsMD (config->arguments, &argc, argv, "PSPHOT.PSF", "-psf", "-psflist"); + + status1 = pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC", "-src", "-srclist"); + status2 = pmConfigFileSetsMD (config->arguments, &argc, argv, "SRCTEXT", "-srctext", "-srctextlist"); + status3 = pmConfigFileSetsMD (config->arguments, &argc, argv, "FORCE", "-force", "-forcelist"); + + if (!(status1 || status2 || status3)) { + // XXX require -force version? + psError(PSPHOT_ERR_ARGUMENTS, true, "No source list is supplied (use one of -force, -forcelist, -src, -srctext, -srclist, or -srctextlist)"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc == 1) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotFullForceArguments...\n"); + return (config); +} + +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile) +{ + fprintf(ofile, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "%s -help\n" + "\n" + "%s -version\n" + "\n" + "where:\n" + " FileNameList is a text file containing filenames, one per line\n" + " MaskFileNameList is a text file of mask filenames, one per line\n" + " VarFileNameList is a text file of variance filenames, one per line\n" + " OutFileBaseName is the 'root name' for output files\n" + "also required:\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify sources to be measured (required)\n" + "\n" + "additional options:\n" + " -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n" + " specify PSF rather than letting %s estimate it\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -threads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n", + program,program,program,program,program); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "also required:\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify sources to be measured (required)\n" + "\n" + "Try '%s -help' for more options and explanation\n", + program,program,program); + if (exitCode != PS_EXIT_SUCCESS) + psErrorStackPrint(stderr, "Error reading arguments\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceImageLoop.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceImageLoop.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceImageLoop.c (revision 41083) @@ -0,0 +1,153 @@ +# include "psphotStandAlone.h" + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +bool psphotFullForceImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *load = psMetadataLookupPtr (&status, config->files, "PSPHOT.LOAD"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + pmHDU *lastHDU = NULL; // Last HDU updated + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + psImageMaskType maskTest = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, load->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + // load just the input image data (image, mask, weight) + pmFPAfileActivate (config->files, false, NULL); + pmFPAfileActivate (config->files, true, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, true, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, true, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // mosaic the cells of a chip into a single contiguous (trimmed) chip + if (!psphotMosaicChip(config, view, "PSPHOT.INPUT", "PSPHOT.LOAD")) ESCAPE ("Unable to mosaic chip."); + + // Read WCS if easy. + // XXX Since we're mosaicking cells, we ignore the case where the WCS is defined for a cell. + { + pmChip *inChip = pmFPAviewThisChip(view, input->fpa); // Mosaicked chip + pmHDU *hduLow = pmHDUGetLowest(input->fpa, inChip, NULL); + if (hduLow && !pmAstromReadWCS(input->fpa, inChip, hduLow->header, 1.0)) { + psWarning("Unable to read WCS astrometry from header."); + psErrorClear(); + pmHDU *hduHigh = pmHDUGetHighest(input->fpa, inChip, NULL); + if (hduHigh && hduHigh != hduLow && + !pmAstromReadWCS(input->fpa, chip, hduHigh->header, 1.0)) { + psWarning("Unable to read WCS astrometry from primary header."); + psErrorClear(); + } + } + } + + // try to load other supporting data (PSF, SRC, etc). + // do not re-load the following three files + pmFPAfileActivate (config->files, true, NULL); + pmFPAfileActivate (config->files, false, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, false, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, false, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // re-activate files so they will be closed and freed below + pmFPAfileActivate (config->files, true, NULL); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + // Update the header + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + psphotVersionHeaderFull(hdu->header); + lastHDU = hdu; + } + + // if an external mask is supplied, ensure that NAN pixels are also masked + if (readout->mask) { + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); // Mask value for saturated pixels + if (!pmReadoutMaskInvalid(readout, maskTest, maskSat)) { + psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels."); + psFree(view); + return false; + } + } + + // run the actual photometry analysis on this chip/cell/readout + if (!psphotFullForceReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + } + + // drop all versions of the internal files + status = true; + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files"); + psFree (view); + return false; + } + } + + // save output which is saved at the chip level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed output for Chip in psphot."); + } + // save output which is saved at the fpa level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed ouput for FPA in psphot."); + + // fail if we failed to handle an error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + return true; +} + +// I/O files related to psphot: +// PSPHOT.INPUT : input image file(s) +// PSPHOT.RESID : residual image +// PSPHOT.OUTPUT : output object tables (object) + +// PSPHOT.BACKSUB : background subtracted image +// PSPHOT.BACKGND : background model (full-scale image?) +// PSPHOT.BACKMDL : background model (binned image?) +// PSPHOT.PSF : sample PSF images + +// PSPHOT.MASK +// PSPHOT.VARIANCE +// Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceReadout.c (revision 41083) @@ -0,0 +1,142 @@ +# include "psphotInternal.h" + +bool psphotFullForceReadout(pmConfig *config, const pmFPAview *view, const char *filerule) { + + // measure the total elapsed time in psphotReadout. + psTimerStart ("psphotReadout"); + + // allow objects to be fit with ugly models (central holes, extreme asymmetry, etc) + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + // set the photcode for this image + if (!psphotAddPhotcode (config, view, filerule)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // optional break-point for processing + char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); + PS_ASSERT_PTR_NON_NULL (breakPt, false); + + // Generate the mask and weight images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + if (!strcasecmp (breakPt, "NOTHING")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!psphotSubtractBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "BACKMDL")) { + return psphotReadoutCleanup (config, view, filerule); + } + + if (!psphotLoadPSF (config, view, filerule)) { + // this only happens if we had a programming error in psphotLoadPSF + psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // include externally-supplied sources (loaded onto detections->newSources) + psphotLoadExtSources (config, view, filerule); + +#ifdef notmoved + // XXX: moved down below + // merge the newly selected sources into the existing list (detections->allSources) + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); +#endif + + // construct sources and measure moments and other basic stats (saved on detections->allSources) + // all sources use the auto-scaled window appropriate to a PSF, except for the saturated + // stars : these use a larger window (3x the basic window) + if (!psphotFullForceSourceStats (config, view, filerule, true)) { // pass 1 + psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); + return psphotReadoutCleanup (config, view, filerule); + } + + // classify sources based on moments, brightness. if a PSF model has been loaded, the PSF + // clump defined for it is used not measured (detections->newSources) + if (!psphotRoughClass (config, view, filerule)) { // pass 1 + psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough classifications"); + return psphotReadoutCleanup (config, view, filerule); + } + + // merge the newly selected sources into the existing list (detections->allSources) + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + + // generate a psf model for any readouts which need one + // psphotFullForcePSF (config, view, filerule); + // XXX I'm not sure we need a different algorithm here or not : we are supplying the + // sources; we should mark with a flag bit the ones we actually want to use as PSF + // stars (this means we need to supply this info in the load). + if (!psphotChoosePSF (config, view, filerule, false)) { + // PSPHOT_ERR_DATA causes this program to exit gracefully + psError (PSPHOT_ERR_DATA, false, "failed to construct psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + // XXX this currently only generates a PSF model + psphotGuessModels (config, view, filerule); + + // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) + // XXX this will fit non-PSF sources if they are supplied + psphotFitSourcesLinear (config, view, filerule, false, false); + + // measure moments + // XXX does this set the correct gaussian window (how to do multiple windows?) + // we are calling this above... + // psphotSourceStats (config, view, filerule, false); + + // measure kron fluxes + psphotKronFlux (config, view, filerule); + + psphotChipParams (config, view, filerule); + + // measure petro fluxes + psphotPetroFlux (config, view, filerule); + + // measure radial apertures + psphotRadialApertures (config, view, filerule, 0); + + // measure galaxy shapes + psphotGalaxyShape (config, view, filerule); + + // measure aperture photometry corrections + if (!psphotApResid (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed on psphotApResid"); + return psphotReadoutCleanup (config, view, filerule); + } + + // calculate source magnitudes (psf mag and ap mag) + psphotMagnitudes(config, view, filerule); + + // calculate lensing parameters + if (!psphotLensing(config, view, filerule)) { + psErrorStackPrint(stderr, "Unable to do lensing parameters."); + psErrorClear(); + } + + // replace background in residual image + psphotSkyReplace (config, view, filerule); + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, filerule); + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceSummary.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceSummary.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceSummary.c (revision 41083) @@ -0,0 +1,337 @@ +# ifdef HAVE_CONFIG_H +# include +# endif + +#include +#include +#include +#include "psphot.h" + +// For simplicilty, this program's (simple) functions are all contained in this file. +static pmConfig* psphotFullForceSummaryArguments(int, char**); +static bool psphotFullForceSummaryParseCamera(pmConfig *); +static bool psphotFullForceSummaryImageLoop(pmConfig*); + +int main (int argc, char **argv) { + + psMemInit(); + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotFullForceSummaryArguments (argc, argv); + assert(config); + +// psphotVersionPrint(); + + if (!psphotFullForceSummaryParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (1); + } + + if (!psphotFullForceSummaryImageLoop (config)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit(1); + } + + // XXX:check for memory leaks + exit (0); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status + +void usage() { + fprintf(stderr, "usage: psphotFullForceSummary -inputs \n"); + exit (1); +} + +static pmConfig* psphotFullForceSummaryArguments(int argc, char **argv) { + + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + int N; + + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + if ((N = psArgumentGet(argc, argv, "-input"))) { + if (argc <= N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + usage(); + } + psArgumentRemove(N, &argc, argv); + + unsigned int numBad = 0; // Number of bad lines + psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[N], false); // Input file info + if (!inputs || numBad > 0) { + psErrorStackPrint(stderr, "Unable to cleanly read MDC file with inputs."); + exit(PS_EXIT_CONFIG_ERROR); + } + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "INPUTS", 0, "Metadata with input details", inputs); + psFree(inputs); + + psArgumentRemove(N, &argc, argv); + } else { + psErrorStackPrint(stderr, "-input must be supplied."); + usage(); + } + if ((N = psArgumentGet(argc, argv, "-cff"))) { + if (argc <= N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + usage(); + } + psArgumentRemove(N, &argc, argv); + + // We read CFF file as table not as sources. All we want are the nominal parameters input to psphotFullForce + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "CFF_FILE", 0, "CFF file with nominal parameters", argv[N]); + psArgumentRemove(N, &argc, argv); + } else { + psErrorStackPrint(stderr, "-cff must be supplied."); + usage(); + } + + if (argc < 2) { + psErrorStackPrint(stderr, "Output is required."); + usage(); + } + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", PS_DATA_STRING, "", argv[1]); + + return config; +} + +static bool psphotFullForceSummaryParseCamera(pmConfig *config) { + bool status = false; + + psMetadata *inputs = psMetadataLookupMetadata(&status, config->arguments, "INPUTS"); // The inputs info + if (!inputs) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find inputs."); + return false; + } + + int nInputs = inputs->list->n; + + for (int i = 0; i < nInputs; i++) { + psMetadataItem *item = psMetadataGet(inputs, i); + + if (item->type != PS_DATA_STRING) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Component %s of the input metadata is not of type STRING", item->name); + return false; + + } + psString sourcesFilename = item->data.str; + + psArray *dummy = psArrayAlloc(1); // dummy array of filenames + dummy->data[0] = psStringCopy(sourcesFilename); + + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "FILENAMES", PS_META_REPLACE, + "Filenames for file rule definition", dummy); + psFree(dummy); + + bool found = false; + pmFPAfile *file = pmFPAfileDefineFromArgs(&found, config, "PSPHOT.INPUT.CMF", "FILENAMES"); + if (!file || !found) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file %s from %s", "PSPHOT.INPUT.CMF", sourcesFilename); + return false; + } + if (file->type != PM_FPA_FILE_CMF) { + psError(PS_ERR_IO, true, "%s is not of type %s", sourcesFilename, pmFPAfileStringFromType(PM_FPA_FILE_CMF)); + return false; + } + } + psMetadataRemoveKey(config->arguments, "FILENAMES"); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.CMF.NUM", PS_META_REPLACE, "number of inputs", + nInputs); + + + pmFPA *outputFPA = pmFPAConstruct(config->camera, config->cameraName); + if (!outputFPA) { + psError(psErrorCodeLast(), false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *output = pmFPAfileDefineOutput(config, outputFPA, "PSPHOT.FULLFORCE.OUTPUT"); + psFree(outputFPA); // Drop reference + if (!output) { + psError(psErrorCodeLast(), false, _("Unable to generate output file from PSPHOT.FULLFORCE.OUTPUT")); + return false; + } + if (output->type != PM_FPA_FILE_CMF) { + psError(PSPHOT_ERR_CONFIG, true, "PSPHOT.FULLFORCE.OUTPUT is not of type CMF"); + return false; + } + output->save = true; + + // We read cff file as a table not as sources + psString cffName = psMetadataLookupStr(NULL, config->arguments, "CFF_FILE"); + if (!cffName) { + psError(PSPHOT_ERR_CONFIG, true, "CFF_FILE is missing from arguments"); + return false; + } + psString resolvedName = pmConfigConvertFilename(cffName, config, false, false); + if (!resolvedName) { + psError(PSPHOT_ERR_CONFIG, false, "failed to resolve CFF_FILE %s", cffName); + return false; + } + psFits *fits = psFitsOpen(resolvedName, "r"); + if (!fits) { + psError(PSPHOT_ERR_CONFIG, false, "failed to open fits CFF_FILE %s", resolvedName); + return false; + } + psArray *inTable = psFitsReadTable(fits); + if (!inTable) { + psError(PSPHOT_ERR_CONFIG, false, "failed to read cff fits table from CFF_FILE %s", resolvedName); + return false; + } + psFitsClose(fits); + + // Convert to a set of arrays indexed by model type + 1 + // which contain pointers to arrays indexed by ID +#define MAX_MODEL_TYPE 10 + psArray *sortedTables = psArrayAlloc(MAX_MODEL_TYPE+1); + for (int i=0; in; i++) { + psMetadata *row = inTable->data[i]; + psS32 ID = psMetadataLookupS32(&status, row, "ID"); + psS32 modelType = psMetadataLookupS32(&status, row, "MODEL_TYPE"); + // XXX: need to use the lookup table functions to be ready for changes in the model type numbers + if (modelType+1 >= MAX_MODEL_TYPE) { + psError(PSPHOT_ERR_CONFIG, false, "found modelType %d max allowed is %d", modelType, MAX_MODEL_TYPE); + return false; + } + psArray *sortedTable = sortedTables->data[modelType+1]; + if (!sortedTable) { + sortedTable = psArrayAlloc(4*inTable->n); + sortedTables->data[modelType+1] = sortedTable; + // dont' free sortedTable the array of tables gets our reference + } + if (ID >= sortedTable->n) { + sortedTable = psArrayRealloc(sortedTable, 2*ID); + // Why doesn't psArrayRealloc do this????? + sortedTable->n = sortedTable->nalloc; + } + if (sortedTable->data[ID]) { + psError(PSPHOT_ERR_CONFIG, true, "Duplicate row with ID %d", ID); + return false; + } + sortedTable->data[ID] = psMemIncrRefCounter(row); + } + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "CFF_TABLES", PS_META_REPLACE, "cff tables", sortedTables); + psFree(inTable); + psFree(sortedTables); + + return true; +} + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +bool psphotFullForceSummaryImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT.CMF"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAfile *output = psMetadataLookupPtr (&status, config->files, "PSPHOT.FULLFORCE.OUTPUT"); + if (!output) { + psError(PSPHOT_ERR_PROG, false, "Can't find output data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + if (!pmFPAAddSourceFromView(output->fpa, view, output->format)) { + psError(PS_ERR_UNKNOWN, false, "Unable to insert HDU into FPA for writing.\n"); + psFree(view); + return NULL; + } + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { +// psLogMsg ("psphotFullForceSummary", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphotFullForceSummary."); + + // We read the WCS from the first input + { + pmHDU *hduLow = pmHDUGetLowest(input->fpa, chip, NULL); + if (hduLow && !pmAstromReadWCS(input->fpa, chip, hduLow->header, 1.0)) { + psWarning("Unable to read WCS astrometry from header."); + psErrorClear(); + pmHDU *hduHigh = pmHDUGetHighest(input->fpa, chip, NULL); + if (hduHigh && hduHigh != hduLow && + !pmAstromReadWCS(input->fpa, chip, hduHigh->header, 1.0)) { + psWarning("Unable to read WCS astrometry from primary header."); + psErrorClear(); + } + } + } + + // Copy the transformations from the input + output->fpa->fromTPA = psMemIncrRefCounter(input->fpa->fromTPA); + output->fpa->toTPA = psMemIncrRefCounter(input->fpa->toTPA); + output->fpa->toSky = psMemIncrRefCounter(input->fpa->toSky); + pmChip *outputChip = pmFPAviewThisChip(view, output->fpa); + outputChip->toFPA = psMemIncrRefCounter(chip->toFPA); + outputChip->fromFPA = psMemIncrRefCounter(chip->fromFPA); + if (output->fpa->hdu->header == NULL) { + output->fpa->hdu->header = psMetadataAlloc(); + } + // XXX: how come psphot and psphotStack don't have to do this? + if (!pmAstromWriteWCS(output->fpa->hdu->header, output->fpa, outputChip, 0.001)) { + ESCAPE("failure to copy WCS to header"); + } + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + // psLogMsg ("psphotFullForceSummary", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + // psLogMsg ("psphotFullForceSummary", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + if (!psphotFullForceSummaryReadout(config, view)) { + ESCAPE ("failure in psphotFullForceSummaryReadout"); + } + } + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psphotFullForceSummary."); + } + + + // If these keywords are not set we get a warning message on output. Since we are not copying the input header + // and do not have an image pmFPA stuff doesn't have values for these + // XXX: What other keywords and concepts should be set (or copied)? + int numCols = psMetadataLookupS32(&status, input->fpa->hdu->header, "IMNAXIS1"); + int numRows = psMetadataLookupS32(&status, input->fpa->hdu->header, "IMNAXIS2"); + psMetadataAddS32(output->fpa->hdu->header, PS_LIST_TAIL, "IMNAXIS1", PS_META_REPLACE, "", numCols); + psMetadataAddS32(output->fpa->hdu->header, PS_LIST_TAIL, "IMNAXIS2", PS_META_REPLACE, "", numRows); + + // XXX: Also add psphot version information + + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for FPA in psphot."); + + // fail if we encountered an unhandled error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceSummaryReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceSummaryReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotFullForceSummaryReadout.c (revision 41083) @@ -0,0 +1,745 @@ +#include "psphotInternal.h" + +typedef struct { + psF32 Q; + psF32 NSigma; + psF32 clampSN; +#ifdef notdef + // these are per object now + int numTrials; + psF64 fRmajorMin; + psF64 fRmajorMax; + psF64 fRmajorDel; + psF64 fRminorMin; + psF64 fRminorMax; + psF64 fRminorDel; + psVector *fRmajor; + psVector *fRminor; +#endif + psArray *zeroPt; // zero points for each input used with exptime to scale flux + psArray *exptime; // exposure times for each input + psArray *cffTables; // one for each model type index is (model_type + 1) entry 0 is no extended model (star) +} galaxyShapeSummaryOptions; + + + +static pmSource *psphotFullForceSummarizeObject(pmConfig *config, pmPhotObj *obj, psVector *fluxScaleFactor, galaxyShapeSummaryOptions *options); +static pmPhotObj *findObjectForSource(psArray **pObjects, pmSource *source); + +static bool setOptions(galaxyShapeSummaryOptions *options, pmReadout *readout, psMetadata *recipe, bool saveVectors); +static bool checkOptions(galaxyShapeSummaryOptions *options, pmReadout *readout, psMetadata *recipe); + + +bool psphotFullForceSummaryReadout (pmConfig *config, const pmFPAview *view) { + + bool status; + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, "PSPHOT.INPUT.CMF"); + + pmFPAfile *output = psMetadataLookupPtr (&status, config->files, "PSPHOT.FULLFORCE.OUTPUT"); + pmCell *outputCell = pmFPAviewThisCell(view, output->fpa); + pmReadout *outputReadout = pmFPAviewThisReadout(view, output->fpa); + if (!outputReadout) { + outputReadout = pmReadoutAlloc(outputCell); + } + + // Get the exposure parameters for the output from recipe and set them on the output + psF32 outputZeroPoint = psMetadataLookupF32(&status, recipe, "PSPHOT.FULLFORCE.ZERO_PT"); + psF32 outputExptime = psMetadataLookupF32(&status, recipe, "PSPHOT.FULLFORCE.EXPTIME"); + + psMetadataAddF32(output->fpa->concepts, PS_LIST_TAIL, "FPA.ZP", PS_META_REPLACE, "Magnitude zero point", + outputZeroPoint); + psMetadataAddF32(outputCell->concepts, PS_LIST_TAIL, "CELL.EXPOSURE", PS_META_REPLACE, "Exposure time (sec)", + outputExptime); + + // Create objects from the various input's sources + // loop over the available readouts + + psVector *fluxScaleFactor = psVectorAlloc(num, PS_TYPE_F32); + galaxyShapeSummaryOptions options; + psArray *objects = NULL; + for (int index = 0; index < num; index++) { + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PSPHOT.INPUT.CMF", index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (index == 0) { + // Get the galaxy shape recipe values, from the analysis if present + // or from the recipe if not + if (!setOptions(&options, readout, recipe, true)) { + psError (PS_ERR_UNKNOWN, false, "problem determining galaxy shape options."); + return false; + } + options.cffTables = psMetadataLookupPtr(NULL, config->arguments, "CFF_TABLES"); + if (!options.cffTables) { + psError (PS_ERR_UNKNOWN, true, "Cannot find cff table in arguments."); + return false; + } + } else { + // Make sure that this input was created with the same galaxy shapes recipe + if (!checkOptions(&options, readout, recipe)) { + psError (PS_ERR_UNKNOWN, false, "galaxy shape options do not match for input %d", index); + return false; + } + } + + // look up zero point + psF32 zero_point = psMetadataLookupF32(&status, readout->parent->parent->parent->concepts, "FPA.ZP"); + psF32 exptime = psMetadataLookupF32(&status, readout->parent->parent->parent->concepts, "FPA.EXPOSURE"); + + psF32 scaleFactor = pow(10, 0.4 * (outputZeroPoint - zero_point)) * outputExptime / exptime; + fluxScaleFactor->data.F32[index] = scaleFactor; + + // find detections + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + sources = psArraySort (sources, pmSourceSortBySeq); + + if (objects == NULL) { + pmSource *lastSource = sources->data[sources->n - 1]; + psAssert(lastSource, "last source is null!"); + objects = psArrayAlloc(lastSource->seq + 1); + } + + for (int i=0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + source->imageID = index; + findObjectForSource(&objects, source); + } + } + + pmDetections *outputDetections = pmDetectionsAlloc(); + if (!psMetadataAddPtr (outputReadout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", + PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", outputDetections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + + psArray *outputSources = psArrayAllocEmpty (objects->n); + + // Loop over objects and compute the summaries + long nObjects = 0; + long nSources = 0; + for (int i=0 ; in; i++) { + pmPhotObj *obj = objects->data[i]; + if (!obj) continue; + + ++nObjects; + pmSource *source = psphotFullForceSummarizeObject(config, obj, fluxScaleFactor, &options); + if (source) { + psArrayAdd (outputSources, 100, source); + ++nSources; + } + } + + psLogMsg("psphot", PS_LOG_INFO, "constructed %ld output sources, from %ld objects.", nSources, nObjects); + + psFree(fluxScaleFactor); + + if (nSources) { + // We have data + outputDetections->allSources = outputSources; + outputReadout->data_exists = true; + outputReadout->parent->data_exists = true; + outputReadout->parent->parent->data_exists = true; + } else { + // XXX: tooo set a quality or fault code + return false; + } + + return true; +} + +static pmPhotObj *findObjectForSource(psArray **pObjects, pmSource *source) { + int seq = source->seq; + + psArray *objects = *pObjects; + if (seq >= objects->n) { + // We need to expand the object array. Kind of suprising. + objects = *pObjects = psArrayRealloc(objects, seq+1); + } + + // Look up object for this seq + pmPhotObj *obj = objects->data[seq]; + if (!obj) { + // not found allocate one + obj = pmPhotObjAlloc(); + objects->data[seq] = obj; + } + pmPhotObjAddSource(obj, source); + + return obj; +} + +static pmSource *psphotFullForceSummarizeObject(pmConfig *config, pmPhotObj *obj, psVector *fluxScaleFactor, galaxyShapeSummaryOptions *options) { + + pmSource *outSrc = NULL; + + + pmSource *source0 = obj->sources->data[0]; + int ID = source0->seq; + + // Find the parameters from the cff tables. nModels is the number of extended model types that + // this source has an entry for + int nModels = 0; + // skip table 0 which is modelType -1 not extended + for (int i=1; icffTables->n; i++) { + psArray *table = options->cffTables->data[i]; + if (!table) continue; + if (ID >= table->n) continue; + if (table->data[ID]) { + nModels++; + } + } + + // Loop over sources for this object. Start output source for first input that has + // good pixel frac + for (int i=0; i < obj->sources->n; i++) { + pmSource *source = obj->sources->data[i]; + + // XXX: get cut from recipe + if (source->pixWeightNotPoor < .9) continue; + + // For now just start the output source as a copy of the first input source that makes cuts + if (!outSrc) { + outSrc = pmSourceCopy(source); + // This copies + // the peak + // the moments which are mostly nan except for Mrf Mx, My, and some of the kron parameters + // + // type, mode, flags + // magnitudes + outSrc->imageID = 0; + outSrc->seq = source->seq; + + if (source->modelPSF) { + outSrc->modelPSF = psMemIncrRefCounter(source->modelPSF); + } + if (source->extpars) { + outSrc->extpars = psMemIncrRefCounter(source->extpars); + } + } + if (nModels && source->modelFits && source->modelFits->n) { + if (!outSrc->modelFits) { + outSrc->modelFits = psArrayAllocEmpty(nModels); + } + for (int iModel = 0; iModel < source->modelFits->n; iModel++) { + pmModel *newModel = source->modelFits->data[iModel]; + pmModel *outModel = NULL; + for (int jModel = 0; jModel < outSrc->modelFits->n; jModel++) { + outModel = outSrc->modelFits->data[jModel]; + if (newModel->type == outModel->type) { + // already have an entry for this type + break; + } + outModel = NULL; + } + if (!outModel) { + // no model of this type in output source yet. Add this one. + // Note: the parameters that we need (position, angle, index, if applicable) + // are the same for all sources so copying any of them is fine + psArrayAdd(outSrc->modelFits, 1, newModel); + } + } + } + } + + if (!outSrc) { + // no good measurements for this source (how?) + return NULL; + } + + // now loop over the model fits and summarize the galaxy shape measurements for each + for (int iModel=0; outSrc->modelFits && iModel < outSrc->modelFits->n; iModel++) { + pmModel *model = outSrc->modelFits->data[iModel]; + int modelType = model->type; + psAssert(modelType >= 0 && modelType+1 < options->cffTables->n, "model type out of range"); + + psArray *cffTable = options->cffTables->data[modelType+1]; + psAssert(cffTable, "missing cff table?"); + + psMetadata *cffRow = cffTable->data[outSrc->seq]; + if (!cffRow) { + psError (PS_ERR_PROGRAMMING, true, "NO cff data for object %d model %d", outSrc->seq, modelType); + psFree(outSrc); + return NULL; + } + bool mdok; + psF32 R_MAJOR = psMetadataLookupF32(&mdok, cffRow, "R_MAJOR"); + if (!mdok) { + psError (PS_ERR_PROGRAMMING, true, "can't find R_MAJOR for object %d type %d", outSrc->seq, modelType); + psFree(outSrc); + return NULL; + } + psF32 R_MINOR = psMetadataLookupF32(&mdok, cffRow, "R_MINOR"); + if (!mdok) { + psError (PS_ERR_PROGRAMMING, true, "can't find R_MINOR for object %d type %d", outSrc->seq, modelType); + psFree(outSrc); + return NULL; + } + + psVector *sumWeightedFlux = NULL; + psVector *sumInvSig2 = NULL; + psVector *numerator = NULL; + psVector *fRmajorVec = NULL; + psVector *fRminorVec = NULL; + psF32 totalNPix = 0; + long vectorLength = 0; + psF32 fRmajorMin = 0; + psF32 fRmajorMax = 0; + psF32 fRmajorDel = 0; + psF32 fRminorMin = 0; + psF32 fRminorMax = 0; + psF32 fRminorDel = 0; + for (int i=0; i < obj->sources->n; i++) { + pmSource *source = obj->sources->data[i]; + + if (source->pixWeightNotPoor < .9) continue; + + // The only parameters that we are summarizing currently are galaxy shapes. + if (!source->modelFits) continue; + if (!source->galaxyFits) continue; + + // XXX: put this into a function + pmSourceGalaxyFits *galaxyFits = NULL; + for (int j=0; jgalaxyFits->n; j++) { + galaxyFits = source->galaxyFits->data[j]; + if (galaxyFits && galaxyFits->modelType == modelType) break; + galaxyFits = NULL; + } + + if (galaxyFits && galaxyFits->nPix && galaxyFits->chisq->n) { + if (numerator == NULL) { + // first source with galaxyFits allocate accumulators + vectorLength = galaxyFits->chisq->n; + sumWeightedFlux = psVectorAlloc(vectorLength, PS_TYPE_F32); + psVectorInit(sumWeightedFlux, 0.0); + sumInvSig2 = psVectorAlloc(vectorLength, PS_TYPE_F32); + psVectorInit(sumInvSig2, 0.0); + numerator = psVectorAlloc(vectorLength, PS_TYPE_F32); + psVectorInit(numerator, 0.0); + + // Initialize the fractional radius vectors + // save these to initialize the output galaxy fits + fRmajorMin = galaxyFits->fRmajorMin; + fRmajorMax = galaxyFits->fRmajorMax; + fRmajorDel = galaxyFits->fRmajorDel; + fRminorMin = galaxyFits->fRminorMin; + fRminorMax = galaxyFits->fRminorMax; + fRminorDel = galaxyFits->fRminorDel; + fRmajorVec = psVectorAlloc(vectorLength, PS_TYPE_F32); + fRminorVec = psVectorAlloc(vectorLength, PS_TYPE_F32); + int k = 0; + for (float fRmajor = fRmajorMin; fRmajor < fRmajorMax + 0.5*fRmajorDel; fRmajor += fRmajorDel) { + for (float fRminor = fRminorMin; fRminor < fRminorMax + 0.5*fRminorDel; fRminor += fRminorDel) { + fRminorVec->data.F32[k] = fRminor; + fRmajorVec->data.F32[k] = fRmajor; + k++; + } + } + psAssert(k == vectorLength, "mismatched vectors"); + } + + // Die if the lengths of the vectors is not the same in all sources +#ifdef notdef + psAssert(vectorLength == options->numTrials, "length of chisq vector %ld does not match options %d", + vectorLength, options->numTrials); +#endif + psAssert(galaxyFits->chisq->n == vectorLength, "length of chisq vectors do not match %ld %ld", + galaxyFits->chisq->n, vectorLength); + + psF32 scaleFactor = fluxScaleFactor->data.F32[source->imageID]; + + totalNPix += galaxyFits->nPix; + + for (int k = 0; k < vectorLength; k++) { + psF32 chisq = galaxyFits->chisq->data.F32[k]; + psF32 flux = galaxyFits->Flux->data.F32[k] * scaleFactor; + psF32 dFlux = galaxyFits->dFlux->data.F32[k] * scaleFactor; + + numerator->data.F32[k] += chisq * galaxyFits->nPix; + + psF32 invSig2 = 1.0 / (dFlux * dFlux); + sumInvSig2->data.F32[k] += invSig2; + + sumWeightedFlux->data.F32[k] += flux * invSig2; + } + } + } + + if (vectorLength) { + // allocate galaxyFits for the output source + if (!outSrc->galaxyFits) { + outSrc->galaxyFits = psArrayAllocEmpty(1); + } + pmSourceGalaxyFits *galaxyFits = pmSourceGalaxyFitsAlloc(); + psArrayAdd(outSrc->galaxyFits, 1, galaxyFits); + psFree(galaxyFits); + + galaxyFits->nPix = totalNPix; + galaxyFits->modelType = modelType; + psVector *fluxVec = + galaxyFits->Flux = psVectorRecycle(galaxyFits->Flux, vectorLength, PS_TYPE_F32); + psVector *dFluxVec = + galaxyFits->dFlux = psVectorRecycle(galaxyFits->dFlux, vectorLength, PS_TYPE_F32); + psVector *chisqVec = + galaxyFits->chisq = psVectorRecycle(galaxyFits->chisq, vectorLength, PS_TYPE_F32); + + galaxyFits->fRmajorMin = fRmajorMin; + galaxyFits->fRmajorMax = fRmajorMax; + galaxyFits->fRmajorDel = fRmajorDel; + galaxyFits->fRminorMin = fRminorMin; + galaxyFits->fRminorMax = fRminorMax; + galaxyFits->fRminorDel = fRminorDel; + + // fill the summary galaxyFits vectors and find the trial with the minimum value for chisq + + int min_k = -1; + psF32 minChisq = NAN; + + for (int k = 0; k < vectorLength; k++) { + fluxVec->data.F32[k] = sumWeightedFlux->data.F32[k] / sumInvSig2->data.F32[k]; + dFluxVec->data.F32[k] = 1.0 / sumInvSig2->data.F32[k]; + + psF32 thischisq = chisqVec->data.F32[k] = numerator->data.F32[k] / totalNPix; + + if (isfinite(thischisq) && (!isfinite(minChisq) || thischisq < minChisq)) { + min_k = k; + minChisq = thischisq; + } + } + + psFree(numerator); + psFree(sumInvSig2); + psFree(sumWeightedFlux); + + if (min_k >= 0 && isfinite(minChisq)) { + // copy the best fit params to the model + // fractional radii with the lowest chisq + psEllipseAxes axes = pmPSF_ModelToAxes(model->params->data.F32, model->class->useReff); + + // examine the params for the trial with minimum chisq. + bool fitMajor = true; + bool fitMinor = true; + psF64 fRmajorBest = fRmajorVec->data.F32[min_k]; + if ((fabs(fRmajorBest - galaxyFits->fRmajorMin) < galaxyFits->fRmajorDel) || + (fabs(fRmajorBest - galaxyFits->fRmajorMax) < galaxyFits->fRmajorDel)) { + fitMajor = false; + } + psF64 fRminorBest = fRminorVec->data.F32[min_k]; + if ((fabs(fRminorBest - galaxyFits->fRminorMin) < galaxyFits->fRminorDel) || + (fabs(fRminorBest - galaxyFits->fRminorMax) < galaxyFits->fRminorDel)) { + fitMinor = false; + } + // If either major or minor is at one of the limits do not fit report the minimum value + bool useFit = fitMajor && fitMinor; + + // save the flux and dFlux values from entry with lowest chisq + psF64 fluxBest = fluxVec->data.F32[min_k]; + psF64 dFluxBest = dFluxVec->data.F32[min_k]; + psF64 dFlux0 = NAN; + psF64 flux0 = NAN; + + if (useFit) { + #define NUM_TRIALS_INIT 9 + psVector *major = psVectorAllocEmpty(NUM_TRIALS_INIT, PS_TYPE_F64); + psVector *minor = psVectorAllocEmpty(NUM_TRIALS_INIT, PS_TYPE_F64); + psVector *chisq = psVectorAllocEmpty(NUM_TRIALS_INIT, PS_TYPE_F64); + psVector *flux = psVectorAllocEmpty(NUM_TRIALS_INIT, PS_TYPE_F64); + psVector *dFlux = psVectorAllocEmpty(NUM_TRIALS_INIT, PS_TYPE_F64); + + // XXX: use a recipe parameter instead of 2.2 + psF64 maxDeltaMaj = 2.2 * galaxyFits->fRmajorDel; + psF64 maxDeltaMin = 2.2 * galaxyFits->fRminorDel; + + psF64 majorMin = NAN; + psF64 majorMax = NAN; + psF64 minorMin = NAN; + psF64 minorMax = NAN; + psF64 chisqMin = NAN; + psF64 chisqMax = NAN; + for (int k = 0; k < vectorLength; k++) { + + if (fabs(fRmajorVec->data.F32[k] - fRmajorBest) < maxDeltaMaj && + fabs(fRminorVec->data.F32[k] - fRminorBest) < maxDeltaMin) { + + if (isfinite(chisqVec->data.F32[k]) && + isfinite(fluxVec->data.F32[k]) && + isfinite(dFluxVec->data.F32[k])) { + + // compute major and minor radius vectors from nominal and trial fractions + // also find the ranges in the vectors + psF64 thisMajor = R_MAJOR * fRmajorVec->data.F32[k]; + if (!isfinite(majorMin) || thisMajor < majorMin) { + majorMin = thisMajor; + } + if (!isfinite(majorMax) || thisMajor > majorMax) { + majorMax = thisMajor; + } + psVectorAppend(major, thisMajor); + + psF64 thisMinor = R_MINOR * fRminorVec->data.F32[k]; + if (!isfinite(minorMin) || thisMinor < minorMin) { + minorMin = thisMinor; + } + if (!isfinite(minorMax) || thisMinor > minorMax) { + minorMax = thisMinor; + } + psVectorAppend(minor, thisMinor); + + psF64 thisChisq = chisqVec->data.F32[k]; + if (!isfinite(chisqMin) || thisChisq < chisqMin) { + chisqMin = thisChisq; + } + if (!isfinite(chisqMax) || thisChisq > chisqMax) { + chisqMax = thisChisq; + } + psVectorAppend(chisq, thisChisq); + + psVectorAppend(flux, fluxVec->data.F32[k]); + psVectorAppend(dFlux, dFluxVec->data.F32[k]); + } + } + } + + + // see if we ever get too few good values (haven't seen this happen) + if (major->n < NUM_TRIALS_INIT) { + fprintf(stderr, "only found %ld good points wanted %d for seq: %d\n", + major->n, NUM_TRIALS_INIT, outSrc->seq); + } + + // rescale data - this helps avoid precision errors. + psF64 majorRange = majorMax - majorMin; + if (majorRange == 0) { + majorRange = 1; + } + psF64 minorRange = minorMax - minorMin; + if (minorRange == 0) { + minorRange = 1; + } + psF64 chisqRange = chisqMax - chisqMin; + if (chisqRange == 0) { + chisqRange = 1; + } + for (int k = 0; k < major->n; k++) { + major->data.F64[k] = (major->data.F64[k] - majorMin) / majorRange; + minor->data.F64[k] = (minor->data.F64[k] - minorMin) / minorRange; + chisq->data.F64[k] = (chisq->data.F64[k] - chisqMin) / chisqRange; + } + + // Fit chisq versus rMajor and rMinor + psPolynomial2D *chisqFit = psPolynomial2DAlloc(PS_POLYNOMIAL_ORD, 2, 2); + psPolynomial2D *fluxFit = psPolynomial2DAlloc(PS_POLYNOMIAL_ORD, 2, 2); + psPolynomial2D *fluxErrorFit = psPolynomial2DAlloc(PS_POLYNOMIAL_ORD, 2, 2); + chisqFit->coeffMask[2][2] = PS_POLY_MASK_SET; + chisqFit->coeffMask[2][1] = PS_POLY_MASK_SET; + chisqFit->coeffMask[1][2] = PS_POLY_MASK_SET; + fluxFit->coeffMask[2][2] = PS_POLY_MASK_SET; + fluxFit->coeffMask[2][1] = PS_POLY_MASK_SET; + fluxFit->coeffMask[1][2] = PS_POLY_MASK_SET; + fluxErrorFit->coeffMask[2][2] = PS_POLY_MASK_SET; + fluxErrorFit->coeffMask[2][1] = PS_POLY_MASK_SET; + fluxErrorFit->coeffMask[1][2] = PS_POLY_MASK_SET; + + bool goodFit = psVectorFitPolynomial2D (chisqFit, NULL, 0xff, chisq, NULL, major, minor); + + if (goodFit) { + // find the axes for the minimum chisq from the formula + psF64 **coeff = chisqFit->coeff; + psF64 D = 4*coeff[2][0]*coeff[0][2] - coeff[1][1]*coeff[1][1]; + psF64 major0 = ((coeff[1][1]*coeff[0][1] - 2*coeff[0][2]*coeff[1][0]) / D) * majorRange + majorMin; + psF64 minor0 = ((coeff[1][1]*coeff[1][0] - 2*coeff[2][0]*coeff[0][1]) / D) * minorRange + minorMin; + + // estimated chisq at the minimum + psF64 chisq0 = psPolynomial2DEval(chisqFit, major0, minor0) * chisqRange + chisqMin; + + // now fit the flux ... + bool goodFluxFit = psVectorFitPolynomial2D(fluxFit, NULL, 0xFF, flux, NULL, major, minor); + + // .. and compute flux at the minimum chisq position + if (goodFluxFit) { + flux0 = psPolynomial2DEval(fluxFit, major0, minor0); + } else { + flux0 = fluxBest; + } + + // .. and compute dFlux at the minimum chisq position + bool goodFluxErrorFit = psVectorFitPolynomial2D(fluxErrorFit, NULL, 0xFF, dFlux, NULL, major, minor); + if (goodFluxErrorFit) { + dFlux0 = psPolynomial2DEval(fluxErrorFit, major0, minor0); + } else { + dFlux0 = dFluxBest; + } + +#ifdef PRINTVALS + fprintf (stderr, "%4d %3d %3ld | %6.3f %6.3f %6.3f %4.2f | %6.3f %6.3f %6.3f %4.2f | %7.4f %7.4f %7.1f %7.1f\n", + outSrc->seq, min_k, major->n, R_MAJOR, fRmajorBest*R_MAJOR, major0, major0/R_MAJOR, R_MINOR, fRminorBest*R_MINOR, minor0, minor0/R_MINOR, minChisq, chisq0, fluxBest, flux0); +#endif + + + axes.major = major0; + axes.minor = minor0; + model->chisq = chisq0; +#ifdef DUMPVECTORS + char fn[80]; + + sprintf(fn, "vectors/%s.%05d.txt", goodFit ? "g" : "b", outSrc->seq); + + FILE *f = fopen(fn, "w"); + fprintf(f, "#major minor chisq i\n"); + for (int k = 0; k < chisq->n; k++) { + fprintf(f, "%10.6f %10.6f %10.6f %5d\n", + major->data.F64[k]/R_MAJOR, minor->data.F64[k]/R_MINOR, chisq->data.F64[k], k); + } + fclose(f); +#endif + } else { + model->flags |= PM_MODEL_STATUS_NONCONVERGE; + +#ifdef PRINTVALS + fprintf(stderr, "%4d %3d %3ld | %6.3f %6.3f %6.3f %6.3f bad fit\n", + outSrc->seq, min_k, major->n, R_MAJOR, fRmajorBest, R_MINOR, fRminorBest); +#endif + psErrorClear(); + // psFree(outSrc->galaxyFits); + + } + psFree(chisqFit); + psFree(fluxFit); + psFree(fluxErrorFit); + psFree(major); + psFree(minor); + psFree(chisq); + psFree(flux); + psFree(dFlux); + } else { + // No fit + // Set the values based on the trial with the best chisq + axes.major = R_MAJOR * fRmajorBest; + axes.minor = R_MINOR * fRminorBest; + model->chisq = minChisq; + flux0 = fluxBest; + dFlux0 = dFluxBest; +#ifdef PRINTVALS + fprintf(stderr, "%4d %3d %3ld | %6.3f %6.3f %6.3f %6.3f skip fit\n", + outSrc->seq, min_k, 0L, R_MAJOR, fRmajorBest, R_MINOR, fRminorBest); +#endif + } + // now save the model parameters in the model structure + + // a crazy model can raise an assert in psEllise + if ((fabs(axes.major) > 10000 * fabs(axes.minor)) || (fabs(axes.minor) > 10000 * fabs(axes.major))) { + model->params->data.F32[PM_PAR_SXX] = NAN; + model->params->data.F32[PM_PAR_SYY] = NAN; + model->params->data.F32[PM_PAR_SXY] = NAN; + } else { + pmPSF_AxesToModel (model->params->data.F32, axes, model->class->useReff); + } + + model->mag = -2.5 * log10(flux0); + model->magErr = dFlux0 / flux0; // 1 / SN + // XXX: should there be a different flag if we didn't do a fit of the chisq + model->flags |= PM_MODEL_STATUS_FITTED; + } + } + psFree(fRminorVec); + psFree(fRmajorVec); + } + + return outSrc; +} + +#define GETVAL(member, key) \ + opt->member = psMetadataLookupF32(&status, md, key); \ + if (!status) { \ + psError (PSPHOT_ERR_CONFIG, true, "failed to looup value for %s in %s", key, \ + useAnalysis ? "readout->analysis" : "recipe"); \ + } + +static bool setOptions(galaxyShapeSummaryOptions *opt, pmReadout *readout, psMetadata *recipe, bool makeVectors) { + bool status; + bool useAnalysis; // fall back to recipe if we dont' find values in analysis. Probably should no longer do this + +#ifdef notdef + psMetadataLookupF32(&useAnalysis, readout->analysis, "GALAXY_SHAPES_FR_MAJOR_MIN"); +#endif + psMetadataLookupF32(&useAnalysis, readout->analysis, "GALAXY_SHAPES_Q"); + psMetadata *md = useAnalysis ? readout->analysis : recipe; + + GETVAL(Q, "GALAXY_SHAPES_Q"); + GETVAL(NSigma, "GALAXY_SHAPES_NSIGMA"); + GETVAL(clampSN, "GALAXY_SHAPES_CLAMP_SN"); + +#ifdef notdef + // these are per object now + GETVAL(fRmajorMin, "GALAXY_SHAPES_FR_MAJOR_MIN"); + GETVAL(fRmajorMax, "GALAXY_SHAPES_FR_MAJOR_MAX"); + GETVAL(fRmajorDel, "GALAXY_SHAPES_FR_MAJOR_DEL"); + GETVAL(fRminorMin, "GALAXY_SHAPES_FR_MINOR_MIN"); + GETVAL(fRminorMax, "GALAXY_SHAPES_FR_MINOR_MAX"); + GETVAL(fRminorDel, "GALAXY_SHAPES_FR_MINOR_DEL"); + + opt->numTrials = ceil((opt->fRmajorMax - opt->fRmajorMin + 0.5*opt->fRmajorDel) / opt->fRmajorDel) * + ceil((opt->fRminorMax - opt->fRminorMin + 0.5*opt->fRminorDel) / opt->fRminorDel) ; + + if (makeVectors) { + opt->fRminor = psVectorAlloc(opt->numTrials, PS_TYPE_F32); + opt->fRmajor = psVectorAlloc(opt->numTrials, PS_TYPE_F32); + int i = 0; + for (float fRmajor = opt->fRmajorMin; fRmajor < opt->fRmajorMax + 0.5*opt->fRmajorDel; + fRmajor += opt->fRmajorDel) { + for (float fRminor = opt->fRminorMin; fRminor < opt->fRminorMax + 0.5*opt->fRminorDel; + fRminor += opt->fRminorDel) { + opt->fRminor->data.F32[i] = fRminor; + opt->fRmajor->data.F32[i] = fRmajor; + i++; + } + } + psAssert(i == opt->numTrials, "Something's wrong with my loop got %d entries expected %d", i, opt->numTrials); + } else { + opt->fRminor = NULL; + opt->fRmajor = NULL; + } +#endif + + return true; +} + +#define CHECKVAL(left, right, val, message) \ + if (left->val != right.val) { \ + psError (PSPHOT_ERR_CONFIG, true, message); \ + return false; \ + } + +static bool checkOptions(galaxyShapeSummaryOptions *options, pmReadout *readout, psMetadata *recipe) { + galaxyShapeSummaryOptions thisReadoutsOptions; + + if (!setOptions(&thisReadoutsOptions, readout, recipe, false)) { + psError (PS_ERR_UNKNOWN, false, "problem determining galaxy shape options for readout"); + return false; + } + CHECKVAL(options, thisReadoutsOptions, Q, "mismatched Q"); + CHECKVAL(options, thisReadoutsOptions, NSigma, "mismatched NSIGMA"); + CHECKVAL(options, thisReadoutsOptions, clampSN, "mismatched cleampSN"); + +#ifdef notdef + // these are per object now + CHECKVAL(options, thisReadoutsOptions, numTrials, "mismatched number of trials") + CHECKVAL(options, thisReadoutsOptions, fRmajorMin, "mismatched fRmajorMin") + CHECKVAL(options, thisReadoutsOptions, fRmajorMax, "mismatched fRmajorMax") + CHECKVAL(options, thisReadoutsOptions, fRmajorDel, "mismatched fRmajorDel") + CHECKVAL(options, thisReadoutsOptions, fRminorMin, "mismatched fRminorMin") + CHECKVAL(options, thisReadoutsOptions, fRminorMax, "mismatched fRminorMax") + CHECKVAL(options, thisReadoutsOptions, fRminorDel, "mismatched fRminorDel") +#endif + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGalaxyParams.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGalaxyParams.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGalaxyParams.c (revision 41083) @@ -0,0 +1,684 @@ +# include "psphotInternal.h" + +# define PRINT_STUFF 0 +# define USE_FOOTPRINTS 1 + +// measure some properties on the exended objects + +bool psphotGalaxyParams (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Galaxy Parameters ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // add this + bool doGalaxyParams = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_GALAXY_PARAMS"); + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + bool doAnnuli = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ANNULI"); + bool doExtModels = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS"); + + // All of these must be true to measure galaxy parameters + if (!doGalaxyParams || !doPetrosian || !doAnnuli || !doExtModels) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping Galaxy parameter source measurements\n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotGalaxyParamsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on galaxy parameter measurements for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// # define COUNT_SKIPS +# ifdef COUNT_SKIPS +/*** for the moment, this test code : it is not thread safe ***/ +static int Nskip1 = 0; +static int Nskip2 = 0; +static int Nskip3 = 0; +static int Nskip4 = 0; +static int Nskip5 = 0; +static int Nskip6 = 0; +static int Nskip7 = 0; +static int Nskip8 = 0; +# define SKIP(VALUE) { VALUE++; continue; } +# else +# define SKIP(VALUE) {continue;} +# endif //notdef + +// aperture-like measurements for extended sources +bool psphotGalaxyParamsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + int Next = 0; + int Npetro = 0; + int Nannuli = 0; + + psTimerStart ("psphot.galaxy"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // XXX EAM TEST + // psphotSaveImage(NULL, readout->image, "dump.v0.fits"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source size"); + return true; + } + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // get the sky noise from the background analysis; if this is missing, get the user-supplied value + float skynoise = psMetadataLookupF32 (&status, readout->analysis, "SKY_STDEV"); + if (!status) { + skynoise = psMetadataLookupF32 (&status, recipe, "SKY.NOISE"); + psWarning ("failed to get sky noise level from background analysis; defaulting to user supplied value of %f\n", skynoise); + } + + float fitNsigmaConv = psMetadataLookupF32 (&status, recipe, "EXT_FIT_NSIGMA_CONV"); // number of sigma for the convolutio + if (!status || !isfinite(fitNsigmaConv) || fitNsigmaConv <= 0) { + fitNsigmaConv = 5.0; + } + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + + psImage *footprintImage = NULL; +#if USE_FOOTPRINTS + footprintImage = psImageAlloc(readout->image->numCols, readout->image->numRows, PS_TYPE_S32); + psImageInit(footprintImage, 0); + pmSetFootprintArrayIDsForImage(footprintImage, detections->footprints, false); +#endif + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion *AnalysisRegion = psRegionAlloc(0,0,0,0); + *AnalysisRegion = psRegionForImage (readout->image, psRegionFromString (region)); + if (psRegionIsNaN (*AnalysisRegion)) psAbort("analysis region mis-defined"); + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_GALAXY_PARAMS"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, AnalysisRegion); + psArrayAdd(job->args, 1, recipe); + + PS_ARRAY_ADD_SCALAR(job->args, skynoise, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, fitNsigmaConv, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, psfSize, PS_TYPE_F32); + psArrayAdd(job->args, 1, footprintImage); + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Next + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Npetro + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nannuli + + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } +# else + if (!psphotGalaxyParams_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } + psScalar *scalar = NULL; + scalar = job->args->data[8]; + Next += scalar->data.S32; + scalar = job->args->data[9]; + Npetro += scalar->data.S32; + scalar = job->args->data[10]; + Nannuli += scalar->data.S32; + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[8]; + Next += scalar->data.S32; + scalar = job->args->data[9]; + Npetro += scalar->data.S32; + scalar = job->args->data[10]; + Nannuli += scalar->data.S32; + } + psFree(job); + } + } + psFree (cellGroups); + psFree(AnalysisRegion); + psFree(footprintImage); + + // XXX EAM TEST + // psphotSaveImage(NULL, readout->image, "dump.v1.fits"); + + psLogMsg ("psphot", PS_LOG_WARN, "galaxy parameter analysis: %f sec for %d objects\n", psTimerMark ("psphot.galaxy"), Next); + psLogMsg ("psphot", PS_LOG_INFO, " %d petrosian\n", Npetro); + psLogMsg ("psphot", PS_LOG_INFO, " %d annuli\n", Nannuli); + +// # if (PS_TRACE_ON) +# ifdef COUNT_SKIPS + fprintf (stderr, "ext analysis skipped @ 1 : %d\n", Nskip1); + fprintf (stderr, "ext analysis skipped @ 2 : %d\n", Nskip2); + fprintf (stderr, "ext analysis skipped @ 3 : %d\n", Nskip3); + fprintf (stderr, "ext analysis skipped @ 4 : %d\n", Nskip4); + fprintf (stderr, "ext analysis skipped @ 5 : %d\n", Nskip5); + fprintf (stderr, "ext analysis skipped @ 6 : %d\n", Nskip6); + fprintf (stderr, "ext analysis skipped @ 7 : %d\n", Nskip7); + fprintf (stderr, "ext analysis skipped @ 8 : %d\n", Nskip8); +#endif + + // psphotRadialProfileShowSkips (); + + // psphotVisualShowResidualImage (readout, false); + + // psphotVisualShowPetrosians (sources); + + // XXX TEST: SAVE IMAGE: + // psphotSaveImage (NULL, readout->image, "resid.fits"); + + return true; +} + +bool psphotGalaxyParams_Threaded (psThreadJob *job) { + + // bool status; + + int Next = 0; + int Npetro = 0; + int Nannuli = 0; + int Ngood = 0; + + // arguments: readout, sources, models, region, psfSize, maskVal, markVal + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psRegion *region = job->args->data[2]; + psMetadata *recipe = job->args->data[3]; + +#if (!USE_FOOTPRINTS) + float skynoise = PS_SCALAR_VALUE(job->args->data[4],F32); +#endif + float fitNsigmaConv = PS_SCALAR_VALUE(job->args->data[5],F32); + float psfSize = PS_SCALAR_VALUE(job->args->data[6],F32); + psImage *footprintImage = job->args->data[7]; + + bool status; +// bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // don't try and use bad models + pmModelStatus badModel = PM_MODEL_STATUS_NONE; +// badModel |= PM_MODEL_STATUS_NONCONVERGE; this causes most objects to get unmeasured + badModel |= PM_MODEL_STATUS_BADARGS; + badModel |= PM_MODEL_STATUS_OFFIMAGE; + badModel |= PM_MODEL_STATUS_NAN_CHISQ; + badModel |= PM_MODEL_SERSIC_PCM_FAIL_GUESS; + badModel |= PM_MODEL_SERSIC_PCM_FAIL_GRID; + badModel |= PM_MODEL_PCM_FAIL_GUESS; + + badModel |= PM_MODEL_STATUS_LIMITS; + + pmModelType sersicModelType = pmModelClassGetType("PS_MODEL_SERSIC"); + + psVector *bgPixels = NULL; + psVector *bgPixelsFlip = NULL; +#define dontMAKE_MARK_IMAGE 1 +#ifdef MAKE_MARK_IMAGE + psImage *markImage = NULL; +#endif + +#define MARK_SOURCE 0 +#define MARK_MASK 1 +#define MARK_BG 2 + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + +#define dontDUMP_IMAGES 1 +#ifdef DUMP_IMAGES + char *filebase = "/data/ippc18.0/home/bills/debug/psphot/images"; + char filename[80]; + psMetadata *dummyHeader = psMetadataAlloc(); +#endif + + // choose the sources of interest + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // Nall ++; + + bool testObject = FALSE; + // testObject = testObject || ((fabs(source->peak->xf - 2468) < 10) && (fabs(source->peak->yf - 3955) < 10)); + // testObject = testObject || ((fabs(source->peak->xf - 432) < 10) && (fabs(source->peak->yf - 781) < 10)); + // testObject = testObject || ((fabs(source->peak->xf - 300) < 10) && (fabs(source->peak->yf - 5988) < 10)); + if (testObject) { + fprintf (stderr, "test object: %f, %f\n", source->peak->xf, source->peak->yf); + testObject = TRUE; + } + + // rules for measuring petrosian parameters for specific objects are set in + // psphotChooseAnalysisOptions.c + if (!(source->tmpFlags & PM_SOURCE_TMPF_PETRO)) SKIP (Nskip1); + + // limit selection by analysis region (XXX move this into psphotChooseAnalysisOption?) + if (source->peak->x < region->x0) SKIP (Nskip2); + if (source->peak->y < region->y0) SKIP (Nskip3); + if (source->peak->x > region->x1) SKIP (Nskip4); + if (source->peak->y > region->y1) SKIP (Nskip5); + + // Should we skip objects that do not have an extended model? + if (!source->extpars) SKIP (Nskip6); + if (!source->modelFits) SKIP (Nskip7); + + // Flux cut to reduce sources to those that match SDSS + // if (source->moments->KronFlux < 2e5) continue; + + pmModel *sersicModel = NULL; + for (int j = 0; j < source->modelFits->n; j++) { + pmModel *model = source->modelFits->data[j]; + if (model->type == sersicModelType) { + sersicModel = model; + break; + } + } + if (!sersicModel) SKIP (Nskip8); + if (sersicModel->flags & badModel) SKIP (Nskip8); + + // check out the model params and skip those that are too large + psF32 *PAR = sersicModel->params->data.F32; + psF32 Xo = PAR[PM_PAR_XPOS]; + psF32 Yo = PAR[PM_PAR_YPOS]; + psF32 Io = PAR[PM_PAR_I0]; + // + // The half light radius to the major axis of the model + psEllipseAxes axes = pmPSF_ModelToAxes (PAR, sersicModel->class->useReff); + if (axes.major >= 100) SKIP (Nskip8); + + source->extpars->ghalfLightRadius = axes.major; + Next ++; + + // if sersicModel is not modelEXT make it so + pmSourceMode saveMode = source->mode; + pmSourceType saveType = source->type; + bool isExtended = saveType == PM_SOURCE_TYPE_EXTENDED; + pmModel *extModel = source->modelEXT; + + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + if (!isExtended || extModel != sersicModel) { + // XXX: I'm not doing this right. For now just look at objects for which + source->modelEXT = psMemIncrRefCounter (sersicModel); + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXTMODEL; + // do we need this? + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + } + + + // and we will integrate out to 2 * half light radius + psF32 R = 2 * axes.major; + psF32 R2 = PS_SQR(R); + + float radius = source->peak->xf - source->pixels->col0; + radius = PS_MAX (radius, source->peak->yf - source->pixels->row0); + radius = PS_MAX (radius, source->pixels->numRows - source->peak->yf + source->pixels->row0); + radius = PS_MAX (radius, source->pixels->numCols - source->peak->xf + source->pixels->col0); + // Make image at least 10 * half light radius + radius = PS_MAX (radius, 5*axes.major); + pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, 2*radius); + + // RedefinePixels frees modelFlux + if (source->modelEXT->isPCM) { + pmPCMCacheModel (source, maskVal, psfSize, fitNsigmaConv); + } else { + pmSourceCacheModel (source, maskVal); + } + +#ifdef DUMP_IMAGES + sprintf(filename, "%s/%04d-pix.fits", filebase, source->id); + psphotSaveImage(dummyHeader, source->pixels, filename); +#endif + + // Create the markImage and accumlate the Flux + + // center of mass in subimage. Note: the calculation below uses pixel index, so we correct + // xCM, yCM from pixel coords to pixel index here. + psF32 xCM = Xo - 0.5 - source->pixels->col0; // coord of peak in subimage + psF32 yCM = Yo - 0.5 - source->pixels->row0; // coord of peak in subimage + + psF32 **vPix = source->pixels->data.F32; + psImageMaskType **vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + bgPixels = psVectorRecycle(bgPixels, source->pixels->numCols*source->pixels->numRows, PS_TYPE_F32); + bgPixels->n = 0; + bgPixelsFlip = psVectorRecycle(bgPixelsFlip, source->pixels->numCols*source->pixels->numRows, PS_TYPE_F32); + bgPixelsFlip->n = 0; + +#ifdef MAKE_MARK_IMAGE + // The markImage may no longer be needed + markImage = psImageRecycle(markImage, source->pixels->numCols, source->pixels->numRows, PS_TYPE_U8); + + // assume all pixels are masked to start + psImageInit(markImage, MARK_MASK); + psU8 **vMark = markImage->data.U8; +#define SET_MARK(_val) vMark[row][col] = _val +#else +#define SET_MARK(_val) +#endif + +#if (USE_FOOTPRINTS) + psS32 **vFootprint = footprintImage->data.S32; +#else + // XXX use recipe value instead of hard coded 1.5 + psF32 threshold = skynoise * 3; +#endif + + psF32 sumI = 0; + psS32 row0 = source->pixels->row0; + psS32 col0 = source->pixels->col0; + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + psF32 yDiff = row - yCM; + // y coordinate of mirror pixel + int yFlip = yCM - yDiff; + if (yFlip < 0) continue; + if (yFlip >= source->pixels->numRows) continue; + psS32 yImage = row + row0; + for (psS32 col = 0; col < source->pixels->numCols ; col++) { + // check mask and value for this pixel + if (vMsk && (vMsk[row][col] & maskVal)) continue; + psF32 pixel = vPix[row][col]; + if (isnan(pixel)) continue; + // valid pixel + + psS32 xImage = col + col0; + + psF32 xDiff = col - xCM; + + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 <= R2) { + sumI += pixel; + } + +#if (USE_FOOTPRINTS) + // this is a pixel in the object + if (vFootprint[yImage][xImage] == source->peak->footprint->id) { + SET_MARK(MARK_SOURCE); + continue; + } + // this is a pixel in another object + if (vFootprint[yImage][xImage] > 0.0) continue; +#else + // this is a pixel in an object (this or the other) + if (fabs(pixel) >= threshold) { + continue; + } +#endif + // we have a background pixel. + // Check whether it's mirror is a background pixel as well. + + // x coordinate of mirror pixel + int xFlip = xCM - xDiff; + if (xFlip < 0) continue; + if (xFlip >= source->pixels->numCols) continue; + // check mask and value for mirror pixel + if (vMsk && (vMsk[yFlip][xFlip] & maskVal)) continue; + psF32 mirrorPixel = vPix[yFlip][xFlip]; + if (isnan(mirrorPixel)) continue; + + +#if (USE_FOOTPRINTS) + bool goodMirror = (vFootprint[yFlip + row0][xFlip + col0] == 0); +#else + bool goodMirror = (fabs(mirrorPixel) < threshold); +#endif + if (goodMirror) { + // This is a good background pixel with a good mirror Save the values + SET_MARK(MARK_BG); + psVectorAppend(bgPixels, pixel); + psVectorAppend(bgPixelsFlip, mirrorPixel); + } + } + } +#ifdef DUMP_IMAGES + sprintf(filename, "%s/%04d-mrk.fits", filebase, source->id); + psphotSaveImage(dummyHeader, markImage, filename); +#endif + + // subtract model to create the residual image + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { + // Hmm something has gone wrong + fprintf(stderr, "failed to subtract sersic model for source %d\n", source->id); + goto restoreModel; + } +#ifdef DUMP_IMAGES + sprintf(filename, "%s/%04d-res.fits", filebase, source->id); + psphotSaveImage(dummyHeader, source->pixels, filename); +#endif + + // don't know if these pointers could change in the background subtraction so grab them again + vPix = source->pixels->data.F32; + vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + psF32 **vVar = source->variance->data.F32; + psF32 **vModel = source->modelFlux->data.F32; + // Accumulate Rt and Ra + psF32 sumRt = 0; + psF32 sumRa = 0; + int numPixels = 0; + psF32 sumB = 0; + psF32 sumModel = 0; + int numB = 0; + for (psS32 row = 0; row < source->pixels->numRows; row++) { + psF32 yDiff = row - yCM; + if (fabs(yDiff) > R) continue; + // y coordinate of mirror pixel + int yFlip = yCM - yDiff; + if (yFlip < 0) continue; + if (yFlip >= source->pixels->numRows) continue; + for (psS32 col = 0; col < source->pixels->numCols ; col++) { + // check mask and value for this pixel + if (vMsk && (vMsk[row][col] & maskVal)) continue; + psF32 pixel = vPix[row][col]; + if (isnan(pixel)) continue; + + psF32 xDiff = col - xCM; + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > R2) continue; + + // Measurement of bumpiness parameter excludes the pixels + // within 2 pixels of the center + if (r2 > 4) { + psF32 variance = vVar[row][col]; + if (isfinite(variance)) { + psF32 modelPixel = vModel[row][col]; + if (!isfinite(modelPixel)) { + // XXX: Can this happen in a good model? + fprintf(stderr, "non-finite modelPixel %4d (%4d, %4d)\n", source->id, col, row); + } + numB++; + sumB += PS_SQR(pixel) - variance; + sumModel += Io * modelPixel; + } + } + + // x coordinate of mirror pixel + int xFlip = xCM - xDiff; + if (xFlip < 0) continue; + if (xFlip >= source->pixels->numCols) continue; + // check mask and value for mirror pixel + if (vMsk && (vMsk[yFlip][xFlip] & maskVal)) continue; + psF32 mirrorPixel = vPix[yFlip][xFlip]; + if (isnan(mirrorPixel)) continue; + + sumRt += fabs(pixel + mirrorPixel); + sumRa += fabs(pixel - mirrorPixel); + ++numPixels; + } + } + + if (numB > 0) { + source->extpars->gbumpy = sqrt(sumB/numB) / (sumModel / numB); + } + + if (bgPixels->n < numPixels) { + // fprintf(stderr, "could not find enough background pixels for source %d found %ld need %d\n", source->id, bgPixels->n, numPixels); + goto restoreModel; + } + Ngood++; + + // and calculate Bt and Ba + psF32 sumBt = 0; + psF32 sumBa = 0; + psF32 *vB = bgPixels->data.F32; + psF32 *vBFlip = bgPixelsFlip->data.F32; + long length = bgPixels->n; + // Randomly select numPixels background pixels + for (int n = 0; n < numPixels; n++) { + double rand = psRandomUniform(rng); + long index = floor(rand * length); + if (index >= length) { + fprintf(stderr, "random index %ld is larger than vector length %ld\n", + index, length); + continue; + } + sumBt += fabs(vB[index] + vBFlip[index]); + sumBa += fabs(vB[index] - vBFlip[index]); + } + + // finally save the results + source->extpars->gRT = 0.5 * (sumRt - sumBt)/sumI; + source->extpars->gRA = 0.5 * (sumRa - sumBa)/sumI; + source->extpars->gS2 = source->extpars->gRT + source->extpars->gRA; + +#if (PRINT_STUFF) + if (Ngood % 40 == 1) { + fprintf (stderr, " id sumI rHL rad sumI gRT gRA sumRt sumBt numPixels numBGPixels\n"); + } + fprintf (stderr, "%5d %5.1f %5.1f : %5.2e %4.1f %10.1f %6.3f %6.3f %5.2e %5.2e %10d %6ld\n", + source->id, source->peak->xf, source->peak->yf, + sumI, source->extpars->ghalfLightRadius, + radius, + source->extpars->gRT, + source->extpars->gRA, sumRt, sumBt, numPixels, bgPixels->n); +#endif + + // Now put things back the way that they were. + restoreModel: + + if (testObject) { + fprintf (stderr, "done with %d\n", source->id); + } + +# define USE_BEST_FIT TRUE + // XXX TEST: Keep the sersic model regardless + if (USE_BEST_FIT && ((extModel != sersicModel) || !isExtended)) { + // add the sersic model back in + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // replace the extended model + if (extModel != sersicModel) { + psMemDecrRefCounter (sersicModel); + source->modelEXT = extModel; + } + source->mode = saveMode; + source->type = saveType; + + // RedefinePixels frees modelFlux + if (isExtended && source->modelEXT->isPCM) { + pmPCMCacheModel (source, maskVal, psfSize, fitNsigmaConv); + } else { + pmSourceCacheModel (source, maskVal); + } + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + } + +#ifdef MAKE_MARK_IMAGE + psFree(markImage); +#endif +#ifdef DUMP_IMAGES + psFree(dummyHeader); +// psphotSaveImage raises an error + psErrorClear(); +#endif + psFree(bgPixels) + psFree(bgPixelsFlip) + psFree(rng); + + psScalar *scalar = NULL; + +// change the value of a scalar on the array (wrap this and put it in psArray.h) + scalar = job->args->data[8]; + scalar->data.S32 = Next; + + scalar = job->args->data[9]; + scalar->data.S32 = Npetro; + + scalar = job->args->data[10]; + scalar->data.S32 = Nannuli; + + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGalaxyShape.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGalaxyShape.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGalaxyShape.c (revision 41083) @@ -0,0 +1,453 @@ +# include "psphotInternal.h" + +bool psphotGalaxyShape (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Galaxy Shapes ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full non-linear fits / extended source analysis? + // XXX for this to be true for psphotFullForce?? + if (!psMetadataLookupBool (&status, recipe, "GALAXY_SHAPES")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping galaxy shape measurements\n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + // psAssert (psf, "missing psf?"); + + if (!psphotGalaxyShapeReadout (config, recipe, view, filerule, readout, sources, psf)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotGalaxyShapeReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char * filerule, pmReadout *readout, psArray *sources, pmPSF *psf) { + + bool status = false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping galaxy shapes"); + return true; + } + + psTimerStart ("psphot.galaxy"); + + psphotInitRadiusEXT (recipe, readout); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + maskVal |= markVal; + + psphotGalaxyShapeOptions *opt = psphotGalaxyShapeOptionsAlloc(); + opt->Q = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_Q"); + psAssert (status, "missing GALAXY_SHAPES_Q"); + opt->NSigma = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_NSIGMA"); + psAssert (status, "missing GALAXY_SHAPES_NSIGMA"); + opt->clampSN = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_CLAMP_SN"); + psAssert (status, "missing GALAXY_SHAPES_NSIGMA"); + psString modelTypeToSave = psMetadataLookupStr(&status, recipe, "EXT_MODEL_TYPE_FORCE"); + if (modelTypeToSave && strcmp(modelTypeToSave, "ALL") && strcmp(modelTypeToSave, "BEST")) { + opt->extModelType = pmModelClassGetType(modelTypeToSave); + } else { + opt->extModelType = pmModelClassGetType("PS_MODEL_SERSIC"); + } + +#ifdef notdef + opt->fRmajorMin = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_FR_MAJOR_MIN"); if (!status) opt->fRmajorMin = 0.5; + opt->fRmajorMax = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_FR_MAJOR_MAX"); if (!status) opt->fRmajorMax = 2.0; + opt->fRmajorDel = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_FR_MAJOR_DEL"); if (!status) opt->fRmajorDel = 0.1; + opt->fRminorMin = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_FR_MINOR_MIN"); if (!status) opt->fRminorMin = 0.5; + opt->fRminorMax = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_FR_MINOR_MAX"); if (!status) opt->fRminorMax = 2.0; + opt->fRminorDel = psMetadataLookupF32(&status, recipe, "GALAXY_SHAPES_FR_MINOR_DEL"); if (!status) opt->fRminorDel = 0.1; +#endif + + // what fraction of the PSF is used? (radius in pixels : 2 -> 5x5 box) + // NOTE: this is only used if we are NOT smoothing with a 1D Gaussian + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + float fitNsigmaConv = psMetadataLookupF32 (&status, recipe, "EXT_FIT_NSIGMA_CONV"); // number of sigma for the convolutio + if (!status || !isfinite(fitNsigmaConv) || fitNsigmaConv <= 0) { + fitNsigmaConv = 5.0; + } + + // Define source fitting parameters for extended source fits + // we are not doing LMM fitting, so most options are irrelevant + pmSourceFitOptions *fitOptions = pmSourceFitOptionsAlloc(); + fitOptions->mode = PM_SOURCE_FIT_EXT_AND_SKY; + fitOptions->covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + fitOptions->nsigma = fitNsigmaConv; + + // Poisson or Constant weight for chisq tests? + fitOptions->poissonErrors = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS_POISSON"); + if (!status) fitOptions->poissonErrors = true; + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend"); + return true; + } + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_GALAXY_SHAPES"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, fitOptions); + psArrayAdd(job->args, 1, opt); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, psfSize, PS_TYPE_S32); + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } +# else + if (!psphotGalaxyShape_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + // psFree(AnalysisRegion); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + psFree (cellGroups); + psFree (fitOptions); + psFree (opt); + + psLogMsg ("psphot.galaxy", PS_LOG_WARN, "measure galaxy shapes : %f sec for %ld objects\n", psTimerMark ("psphot.galaxy"), sources->n); + return true; +} + +bool psphotGalaxyShape_Threaded (psThreadJob *job) { + + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + pmSourceFitOptions *fitOptions = job->args->data[2]; + psphotGalaxyShapeOptions *opt = job->args->data[3]; + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[4],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[5],PS_TYPE_IMAGE_MASK_DATA); + int psfSize = PS_SCALAR_VALUE(job->args->data[6],S32); + + float fitRadius; + float windowRadius; + + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + + // check status of this source's moments + if (!source->moments) continue; + if (!(source->tmpFlags & PM_SOURCE_TMPF_MOMENTS_MEASURED)) continue; + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + if (!source->modelPSF) continue; + + // psphotSetRadiusMomentsExact sets the radius based on Mrf + if (!isfinite(source->moments->Mrf)) continue; + + // modelFits is allocated if a galaxy fit is requested + if (!source->modelFits) continue; + + psphotSetRadiusMomentsExact(&fitRadius, &windowRadius, readout, source, markVal); // NOTE : 6 allocs + + // skip saturated stars modeled with a radial profile + // XXX worry about this at some point.. + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // this function populates moments->Mrf,GalaxyShape,GalaxyShapeErr + // do the following for a set of shapes (Ex,Ey) + psphotGalaxyShapeGrid (source, fitOptions, opt, maskVal, psfSize); + + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + + return true; +} + +bool psphotGalaxyShapeGrid (pmSource *source, pmSourceFitOptions *fitOptions, psphotGalaxyShapeOptions *opt, psImageMaskType maskVal, int psfSize) { + + + for (int iModel = 0 ; iModel < source->modelFits->n; iModel++) { + pmModel *model = source->modelFits->data[iModel]; + if (!model) return false; + + pmModelType modelType = model->type; + model->flags = PM_MODEL_STATUS_NONE; + + // we are using fitOptions->mode : be sure this makes sense + pmPCMdata *pcm = pmPCMinit (source, fitOptions, model, maskVal, psfSize); + if (!pcm) return false; + + // we are fitting only PM_PAR_I0; the shape elements are generated from a grid + psF32 *PAR = pcm->modelConv->params->data.F32; + + pmSourceGalaxyFits *galaxyFits = pmSourceGalaxyFitsAlloc(); + + // set the options for this source + if (!psphotGalaxyShapeOptionsSet(source, galaxyFits, opt)) { + // this source won't work + psFree(pcm); + psFree(galaxyFits); + continue; + } + + if (!source->galaxyFits) { + source->galaxyFits = psArrayAllocEmpty(1); + } + psArrayAdd(source->galaxyFits, 1, galaxyFits); + psFree(galaxyFits); + + // I have some source guess (e0, e1, e2) + psEllipseAxes guessAxes = pmPSF_ModelToAxes (PAR, model->class->useReff); + + float fRmajorBest = NAN; + float fRminorBest = NAN; + float chisqBest = NAN; + for (float fRmajor = galaxyFits->fRmajorMin; fRmajor < galaxyFits->fRmajorMax + 0.5*galaxyFits->fRmajorDel; fRmajor += galaxyFits->fRmajorDel) { + for (float fRminor = galaxyFits->fRminorMin; fRminor < galaxyFits->fRminorMax + 0.5*galaxyFits->fRminorDel; fRminor += galaxyFits->fRminorDel) { + + psEllipseAxes testAxes = guessAxes; + testAxes.major = guessAxes.major * fRmajor; + testAxes.minor = guessAxes.minor * fRminor; + + pmPSF_AxesToModel (PAR, testAxes, model->class->useReff); + + psphotGalaxyShapeSource (pcm, source, galaxyFits, maskVal, psfSize, true); + + int i = galaxyFits->chisq->n - 1; + float flux = galaxyFits->Flux->data.F32[i]; + if (isfinite(flux)) { + float thisChisq = galaxyFits->chisq->data.F32[i]; + if (isfinite(thisChisq) && isfinite(flux) && (!isfinite(chisqBest) || thisChisq < chisqBest)) { + chisqBest = thisChisq; + fRmajorBest = fRmajor; + fRminorBest = fRminor; + } + } + // reset I0 to avoid potential problems on the next iteration + PAR[PM_PAR_I0] = 1.0; + } + } + + if (isfinite(chisqBest)) { + // now save the best fitting model as the source's extended model ... + psEllipseAxes testAxes = guessAxes; + + // ... unless this macro is defined +#ifndef SAVE_NOMINAL_MODEL + testAxes.major = guessAxes.major * fRmajorBest; + testAxes.minor = guessAxes.minor * fRminorBest; +#endif + + pmPSF_AxesToModel (PAR, testAxes, model->class->useReff); + + psphotGalaxyShapeSource (pcm, source, galaxyFits, maskVal, psfSize, false); + + // Replace modelEXT with the best model from the first of the model fits if one of them is good + if (isfinite(PAR[PM_PAR_I0]) && modelType == opt->extModelType) { + psFree (source->modelEXT); + + source->modelEXT = psMemIncrRefCounter (pcm->modelConv); + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXTMODEL; + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + + // cache the model flux + pmPCMCacheModel (source, maskVal, psfSize, fitOptions->nsigma); + } + } + + psFree (pcm); + } + return true; +} + +// fit the given model to the source and find chisq & normalization +// XXX is this a single-component model? sersic with a supplied index, Reff, axis ratio, and theta? +bool psphotGalaxyShapeSource (pmPCMdata *pcm, pmSource *source, pmSourceGalaxyFits *galaxyFits, psImageMaskType maskVal, int psfSize, bool saveResults) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + + // generate the modelFlux + // need to reset here each time since we assign below to calculate the flux + // XXX note that this does not add sky to model + pmPCMMakeModel (source, pcm->modelConv, pcm->nsigma, maskVal, psfSize); + pcm->modelConv->params->data.F32[PM_PAR_I0] = 1.0; + + int nPix = 0; + float YY = 0.0; + float YM = 0.0; + float MM = 0.0; + bool usePoisson = pcm->poissonErrors; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[iy][ix] == 0) { + continue; + } + + // skip nan value points + if (!isfinite(source->pixels->data.F32[iy][ix])) { + continue; + } + + float fy = source->pixels->data.F32[iy][ix]; + float fm = source->modelFlux->data.F32[iy][ix]; + float wt = (usePoisson) ? 1.0 / source->variance->data.F32[iy][ix] : 1.0; + + YY += PS_SQR(fy) * wt; + YM += fm * fy * wt; + MM += PS_SQR(fm) * wt; + nPix ++; + } + } + + float Io = YM / MM; + float dIo = sqrt (1.0 / MM); + float Chisq = (YY - 2 * Io * YM + Io * Io * MM) / (float) nPix; + // NOTE : if !poissonErrors, Chisq is not really the chisq, but is scaled by the flux. + + pcm->modelConv->params->data.F32[PM_PAR_I0] = Io; + float flux = pcm->modelConv->class->modelFlux (pcm->modelConv->params); + float dflux = flux * (dIo / Io); + + + if (saveResults) { + psVectorAppend (galaxyFits->Flux, flux); + psVectorAppend (galaxyFits->dFlux, dflux); + psVectorAppend (galaxyFits->chisq, Chisq); + galaxyFits->nPix = nPix; + galaxyFits->modelType = pcm->modelConv->type; + } + + return true; +} + +/**** support functions ****/ + +void psphotGalaxyShapeOptionsFree (psphotGalaxyShapeOptions *opt) { + return; +} + +psphotGalaxyShapeOptions *psphotGalaxyShapeOptionsAlloc() +{ + psphotGalaxyShapeOptions *opt = (psphotGalaxyShapeOptions *) psAlloc(sizeof(psphotGalaxyShapeOptions)); + psMemSetDeallocator(opt, (psFreeFunc) psphotGalaxyShapeOptionsFree); + + return opt; +} + +bool psphotGalaxyShapeOptionsSet(pmSource *source, pmSourceGalaxyFits *galaxyFits, psphotGalaxyShapeOptions *options) +{ + // XXX: put these in recipe + // doesn't make sense to use too small of a spacing + float clampedSN = source->extSN < options->clampSN ? source->extSN : options->clampSN; + + float f_del = options->Q / clampedSN; + float f_min = 1 - options->NSigma * f_del; + float f_max = 1 + options->NSigma * f_del; + + // if f_min goes negative skip this object + if (f_min < 0) return false; + + galaxyFits->fRmajorMin = f_min; + galaxyFits->fRmajorMax = f_max; + galaxyFits->fRmajorDel = f_del; + galaxyFits->fRminorMin = f_min; + galaxyFits->fRminorMax = f_max; + galaxyFits->fRminorDel = f_del; + + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGuessModels.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGuessModels.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotGuessModels.c (revision 41083) @@ -0,0 +1,259 @@ +# include "psphotInternal.h" + +// XXX : the threading here is not great. this may be due to blocks between elements, but +// the selection of the objects in a cell is not optimal. To fix: +// 1) define the boundaries of the cells up front +// 2) loop over the sources once and associate them with their cell +// 3) define the threaded function to work with sources for a given cell + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotGuessModels (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotGuessModelsReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on to guess models for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +int NpixTotal = 0; + +// construct an initial PSF model for each object (new sources only) +bool psphotGuessModelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + bool status; + + psTimerStart ("psphot.models"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping model guess"); + return true; + } + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // setup the PSF fit radius details + psphotInitRadiusPSF (recipe, readout); + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + NpixTotal = 0; + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_GUESS_MODEL"); + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, psf); + + // XXX change these to use abstract mask type info + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } +# else + if (!psphotGuessModel_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + psFree(job); +# endif + } + + + // wait for the threads to finish and manage results + // wait here for the threaded jobs to finish + // fprintf (stderr, "wait for threads (%d, %d)\n", jx, jy); + if (!psThreadPoolWait (false, true)) { + // harvest our jobs + psFree(cellGroups); + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + // we have no returned data from this operation + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + + // XXX check that all sources that should have models + int nMiss = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->tmpFlags & PM_SOURCE_TMPF_MODEL_GUESS) { + continue; + } + nMiss ++; + } + psAssert (nMiss == 0, "failed to attempt to build models for %d objects\n", nMiss); + + psFree (cellGroups); + + psLogMsg ("psphot.models", PS_LOG_WARN, "built models for %ld objects: %f sec (%d pixels)\n", sources->n, psTimerMark ("psphot.models"), NpixTotal); + return true; +} + +// construct models only for sources in the specified region +bool psphotGuessModel_Threaded (psThreadJob *job) { + + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + pmPSF *psf = job->args->data[2]; + + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[3],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[4],PS_TYPE_IMAGE_MASK_DATA); + + int nSrc = 0; + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // do not redo sources already guessed + if (source->tmpFlags & PM_SOURCE_TMPF_MODEL_GUESS) continue; + + // this is used to mark sources for which the model is measured. We check later that + // all are used. + source->tmpFlags |= PM_SOURCE_TMPF_MODEL_GUESS; + + // skip non-astronomical objects (very likely defects) and satstars with profiles subtracted + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (!source->peak) continue; + + // psAssert (source->peak->footprint, "peak without footprint??"); + + nSrc ++; + + // the guess central intensity comes from the peak: + float Io = source->peak->rawFlux; + if (!isfinite(Io) && source->moments) { + Io = source->moments->Peak; + } + + // We have two options to get a guess for the object position: the position from the + // peak and the position from the moments. Use the peak position if there are no + // moments + + bool useMoments = pmSourcePositionUseMoments(source); + + float Xo, Yo; + if (useMoments) { + Xo = source->moments->Mx; + Yo = source->moments->My; + } else { + Xo = source->peak->xf; + Yo = source->peak->yf; + } + +# if (0) + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + fprintf (stderr, "satstar: %f,%f vs %f,%f : %c\n", + source->moments->Mx, source->moments->My, + source->peak->xf, source->peak->yf, + (useMoments ? 'T' : 'F')); + } +# endif + + // set PSF parameters for this model (apply 2D shape model to coordinates Xo, Yo) + pmModel *modelPSF = pmModelFromPSFforXY(psf, Xo, Yo, Io); + + if (modelPSF == NULL) { + psWarning ("Failed to determine PSF model at (%f,%f); trying image center", Xo, Yo); + + float Xc = 0.5*readout->image->numCols; + float Yc = 0.5*readout->image->numRows; + modelPSF = pmModelFromPSFforXY(psf, Xc, Yc, Io); + if (modelPSF == NULL) { + psError(PSPHOT_ERR_PSF, false, "Failed to determine PSF model at center of image"); + return false; + } + + // Now set the object position at the expected location: + modelPSF->params->data.F32[PM_PAR_XPOS] = Xo; + modelPSF->params->data.F32[PM_PAR_YPOS] = Yo; + source->mode |= PM_SOURCE_MODE_BADPSF; + } + + // set the fit radius based on the object flux limit and the model + // this function affects the mask pixels + psphotCheckRadiusPSF (readout, source, modelPSF, markVal); + + // set the source PSF model + psAssert (source->modelPSF == NULL, "failed to free one of the models?"); + source->modelPSF = modelPSF; + source->modelPSF->residuals = psf->residuals; + + pmSourceCacheModel (source, maskVal); // ALLOC x14 (!) + NpixTotal += source->pixels->numCols * source->pixels->numRows; + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotImageLoop.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotImageLoop.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotImageLoop.c (revision 41083) @@ -0,0 +1,213 @@ +# include "psphotStandAlone.h" + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +bool psphotImageLoop (pmConfig *config, psphotImageLoopMode mode) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *load = psMetadataLookupPtr (&status, config->files, "PSPHOT.LOAD"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + pmHDU *lastHDU = NULL; // Last HDU updated + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + psImageMaskType maskTest = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, load->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + // load just the input image data (image, mask, weight) + pmFPAfileActivate (config->files, false, NULL); + pmFPAfileActivate (config->files, true, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, true, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, true, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // mosaic the cells of a chip into a single contiguous (trimmed) chip + if (!psphotMosaicChip(config, view, "PSPHOT.INPUT", "PSPHOT.LOAD")) ESCAPE ("Unable to mosaic chip."); + + // Read WCS if easy. + // XXX Since we're mosaicking cells, we ignore the case where the WCS is defined for a cell. + { + pmChip *inChip = pmFPAviewThisChip(view, input->fpa); // Mosaicked chip + pmHDU *hduLow = pmHDUGetLowest(input->fpa, inChip, NULL); + if (hduLow && !pmAstromReadWCS(input->fpa, inChip, hduLow->header, 1.0)) { + psWarning("Unable to read WCS astrometry from header."); + psErrorClear(); + pmHDU *hduHigh = pmHDUGetHighest(input->fpa, inChip, NULL); + if (hduHigh && hduHigh != hduLow && + !pmAstromReadWCS(input->fpa, chip, hduHigh->header, 1.0)) { + psWarning("Unable to read WCS astrometry from primary header."); + psErrorClear(); + } + } + } + + // try to load other supporting data (PSF, SRC, etc). + // do not re-load the following three files + pmFPAfileActivate (config->files, true, NULL); + pmFPAfileActivate (config->files, false, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, false, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, false, "PSPHOT.VARIANCE"); + // do not load the exposure number file yet file until we are done with the psphot analysis + pmFPAfileActivate (config->files, false, "PSPHOT.EXPNUM"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // re-activate files so they will be closed and freed below +// XXX: Defer this +// pmFPAfileActivate (config->files, true, NULL); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + // Update the header + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + psphotVersionHeaderFull(hdu->header); + lastHDU = hdu; + } + + // if an external mask is supplied, ensure that NAN pixels are also masked + if (readout->mask) { + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); // Mask value for saturated pixels + if (!pmReadoutMaskInvalid(readout, maskTest, maskSat)) { + psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels."); + psFree(view); + return false; + } + } + + // run the actual photometry analysis on this chip/cell/readout + switch (mode) { + case PSPHOT_SINGLE: + if (!psphotReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + break; + case PSPHOT_MINIMAL: + if (!psphotReadoutMinimal (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + break; + case PSPHOT_FORCED: + if (!psphotForcedReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + break; + case PSPHOT_FULL_FORCE: + if (!psphotFullForceReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + break; + case PSPHOT_MAKE_PSF: + if (!psphotMakePSFReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + break; + case PSPHOT_MODEL_TEST: + if (!psphotModelTestReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + break; + } + } + + // drop all versions of the internal files + status = true; + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files"); + psFree (view); + return false; + } + } + // Defer output and closing of files until we've (possibly) done the NFrames analysis below + pmFPAfileActivate (config->files, false, NULL); + + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psphot."); + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for FPA in psphot."); + + // activate the EXPNUM image, cause it to be read in (if it exists), use it to set N Frames for the sources + // and finally iterate up to trigger writing of the output files + pmFPAfileActivate (config->files, true, "PSPHOT.EXPNUM"); + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { + if (! chip->process || ! chip->file_exists) { continue; } + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed attempting to load EXPNUM input for Chip in psphot."); + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + if (! readout->data_exists) { continue; } + if (!psphotSetNFrames (config, view, "PSPHOT.INPUT")) ESCAPE ("failed to setNFrames."); + } + } + // now activate all files to trigger the output + pmFPAfileActivate (config->files, true, NULL); + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed to iterate up Chip in psphot."); + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed ouput for FPA in psphot."); + + // fail if we failed to handle an error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + return true; +} + +// I/O files related to psphot: +// PSPHOT.INPUT : input image file(s) +// PSPHOT.RESID : residual image +// PSPHOT.OUTPUT : output object tables (object) + +// PSPHOT.BACKSUB : background subtracted image +// PSPHOT.BACKGND : background model (full-scale image?) +// PSPHOT.BACKMDL : background model (binned image?) +// PSPHOT.PSF : sample PSF images + +// PSPHOT.MASK +// PSPHOT.VARIANCE +// Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotImageQuality.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotImageQuality.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotImageQuality.c (revision 41083) @@ -0,0 +1,322 @@ +#include "psphotInternal.h" +#define USE_SAMPLE 1 /* sample vs robust stats */ + +// XXX Lots of code duplication here + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotImageQuality (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotImageQualityReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on to measure image quality for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// selecting the 'good' stars (likely to be psf stars), measure the M_cn, M_sn terms for n = 2,3,4 +bool psphotImageQualityReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + bool status = true; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // if we have a PSF, skip this analysis + if (psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF")) { + return true; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping image quality"); + return true; + } + + psVector *FWHM_MAJOR = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *FWHM_MINOR = psVectorAllocEmpty(100, PS_TYPE_F32); + + psVector *M2 = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *M3 = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *M4 = psVectorAllocEmpty(100, PS_TYPE_F32); + + psVector *M2c = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *M2s = psVectorAllocEmpty(100, PS_TYPE_F32); + + int num = 0; // Number of good sources + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source) { + continue; + } + + // select by S/N? + // select by PSFSTAR mode? + // ?? + if (source->type != PM_SOURCE_TYPE_STAR || !(source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR)) { + psTrace("psphot", 10, "Ignoring source for image quality because not a good star"); + continue; + } + + pmMoments *moments = source->moments; + if (!moments) { + psTrace("psphot", 10, "Ignoring source for image quality because no moments"); + continue; + } + + // if (source->moments->SN < XXX) continue; + + // M_cn = \sum (f r^2 cos(n t)) / \sum (f) + // M_sn = \sum (f r^2 sin(n t)) / \sum (f) + + // r^2 cos(2t) = r^2 cos^2 t - r^2 sin^2 t = x^2 - y^2 + // r^2 sin(2t) = r^2 2 cos t sin t = 2 x y + + // r^2 cos(3t) = r^2 cos^3 t - r^2 3 cos t sin^2 t = (x^3 - 3 x y^2) / r + // r^2 sin(3t) = r^2 3 cos^2 t sin t - sin^3 t = (3 x^2 y - y^3) / r + + // r^2 cos(4t) = r^2 cos^4 t - r^2 6 cos^2 t sin^2 t + r^2 sin^4 t = (x^4 - 6 x^2 y^2 + y^4) / r^2 + // r^2 sin(4t) = r^2 4 cos^3 t sin t - 4 sin^3 t cos t = (4 x^3 y - 4 y^3 x) / r^2 + + // NOTE that Mxxx,etc have the factor of r in the denominator, while Mxxxx,etc have the factor of r^2. + // See pmSourceMoments.c:418 + + num++; + + psEllipseMoments emoments; + emoments.x2 = moments->Mxx; + emoments.xy = moments->Mxy; + emoments.y2 = moments->Myy; + psEllipseAxes axes = psEllipseMomentsToAxes (emoments, 20.0); + + psVectorAppend (FWHM_MAJOR, 2.355*axes.major); + psVectorAppend (FWHM_MINOR, 2.355*axes.minor); + + // the moments->Mnnn terms contain the straight sums: for example: moments->Mxxx contains \sum x^3 / r + float M_c2 = moments->Mxx - moments->Myy; + float M_s2 = 2*moments->Mxy; + psVectorAppend (M2, hypot(M_c2, M_s2)); + psVectorAppend (M2c, M_c2); + psVectorAppend (M2s, M_s2); + + float M_c3 = moments->Mxxx - 3.0*moments->Mxyy; + float M_s3 = 3.0*moments->Mxxy - moments->Myyy; + psVectorAppend (M3, hypot (M_c3, M_s3)); + + float M_c4 = moments->Mxxxx - 6.0*moments->Mxxyy + moments->Myyyy; + float M_s4 = 4.0*moments->Mxxxy - 4.0*moments->Mxyyy; + psVectorAppend (M4, hypot (M_c4, M_s4)); + } + + if (num == 0) { + psLogMsg ("psphot", PS_LOG_INFO, "no valid sources for image quality, skipping"); + psFree(FWHM_MAJOR); + psFree(FWHM_MINOR); + psFree(M2); + psFree(M2c); + psFree(M2s); + psFree(M3); + psFree(M4); + + return true; + } + + psMetadataAddS32(readout->analysis, PS_LIST_TAIL, "IQ_NSTAR", PS_META_REPLACE, "Number of stars used for IQ measurements", M2->n); + +// XXX make this a recipe option +#if (USE_SAMPLE) + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_SAMPLE_QUARTILE); + + if (!psVectorStats(stats, FWHM_MAJOR, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + + float fwhm_major = stats->sampleMean; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW1", PS_META_REPLACE, "FWHM of Major Axis from moments", stats->sampleMean); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW1_E", PS_META_REPLACE, "FWHM scatter (Major) from moments", stats->sampleStdev); + + if (!psVectorStats(stats, FWHM_MINOR, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float fwhm_minor = stats->sampleMean; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW2", PS_META_REPLACE, "FWHM of Minor Axis from moments", stats->sampleMean); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW2_E", PS_META_REPLACE, "FWHM scatter (Minor) from moments", stats->sampleStdev); + + if (!psVectorStats(stats, M2, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float vM2 = stats->sampleMean; + float dM2 = stats->sampleStdev; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2", PS_META_REPLACE, "M_2 = sqrt (M_c2^2 + M_s2^2)", vM2); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2_ER", PS_META_REPLACE, "Stdev of M_2", dM2); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2_LQ", PS_META_REPLACE, "Lower Quartile of M_2", stats->sampleLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2_UQ", PS_META_REPLACE, "Upper Quartile of M_2", stats->sampleUQ); + + if (!psVectorStats(stats, M2c, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C", PS_META_REPLACE, "M_2c = sum f r^2 cos(2phi) / sum f", stats->sampleMean); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C_E", PS_META_REPLACE, "Stdev of M_2c", stats->sampleStdev); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C_L", PS_META_REPLACE, "Lower Quartile of M_2c", stats->sampleLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C_U", PS_META_REPLACE, "Upper Quartile of M_2c", stats->sampleUQ); + + if (!psVectorStats(stats, M2s, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S", PS_META_REPLACE, "M_2s = sum f r^2 cos(2phi) / sum f", stats->sampleMean); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S_E", PS_META_REPLACE, "Stdev of M_2s", stats->sampleStdev); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S_L", PS_META_REPLACE, "Lower Quartile of M_2s", stats->sampleLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S_U", PS_META_REPLACE, "Upper Quartile of M_2s", stats->sampleUQ); + + if (!psVectorStats(stats, M3, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float vM3 = stats->sampleMean; + float dM3 = stats->sampleStdev; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3", PS_META_REPLACE, "M_3 = sqrt (M_c3^2 + M_s3^2)", vM3); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3_ER", PS_META_REPLACE, "Stdev of M_3", dM3); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3_LQ", PS_META_REPLACE, "Lower Quartile of M_3", stats->sampleLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3_UQ", PS_META_REPLACE, "Upper Quartile of M_3", stats->sampleUQ); + + if (!psVectorStats(stats, M4, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float vM4 = stats->sampleMean; + float dM4 = stats->sampleStdev; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4", PS_META_REPLACE, "M_4 = sqrt (M_c4^2 + M_s4^2)", vM4); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4_ER", PS_META_REPLACE, "Stdev of M_4", dM4); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4_LQ", PS_META_REPLACE, "Lower Quartile of M_4", stats->sampleLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4_UQ", PS_META_REPLACE, "Upper Quartile of M_4", stats->sampleUQ); + +#else + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); + + if (!psVectorStats(stats, FWHM_MAJOR, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float fwhm_major = stats->robustMedian; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW1", PS_META_REPLACE, "FWHM of Major Axis from moments", stats->robustMedian); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW1_E", PS_META_REPLACE, "FWHM scatter (Major) from moments", stats->robustStdev); + + if (!psVectorStats(stats, FWHM_MINOR, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float fwhm_minor = stats->robustMedian; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW2", PS_META_REPLACE, "FWHM of Minor Axis from moments", stats->robustMedian); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_FW2_E", PS_META_REPLACE, "FWHM scatter (Minor) from moments", stats->robustStdev); + + if (!psVectorStats(stats, M2, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float vM2 = stats->robustMedian; + float dM2 = stats->robustStdev; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2", PS_META_REPLACE, "M_2 = sqrt (M_c2^2 + M_s2^2)", vM2); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2_ER", PS_META_REPLACE, "Stdev of M_2", dM2); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2_LQ", PS_META_REPLACE, "Lower Quartile of M_2", stats->robustLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2_UQ", PS_META_REPLACE, "Upper Quartile of M_2", stats->robustUQ); + + if (!psVectorStats(stats, M2c, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C", PS_META_REPLACE, "M_2c = sum f r^2 cos(2phi) / sum f", stats->robustMedian); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C_E", PS_META_REPLACE, "Stdev of M_2c", stats->robustStdev); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C_L", PS_META_REPLACE, "Lower Quartile of M_2c", stats->robustLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2C_U", PS_META_REPLACE, "Upper Quartile of M_2c", stats->robustUQ); + + if (!psVectorStats(stats, M2s, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S", PS_META_REPLACE, "M_2s = sum f r^2 cos(2phi) / sum f", stats->robustMedian); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S_E", PS_META_REPLACE, "Stdev of M_2s", stats->robustStdev); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S_L", PS_META_REPLACE, "Lower Quartile of M_2s", stats->robustLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M2S_U", PS_META_REPLACE, "Upper Quartile of M_2s", stats->robustUQ); + + if (!psVectorStats(stats, M3, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float vM3 = stats->robustMedian; + float dM3 = stats->robustStdev; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3", PS_META_REPLACE, "M_3 = sqrt (M_c3^2 + M_s3^2)", vM3); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3_ER", PS_META_REPLACE, "Stdev of M_3", dM3); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3_LQ", PS_META_REPLACE, "Lower Quartile of M_3", stats->robustLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M3_UQ", PS_META_REPLACE, "Upper Quartile of M_3", stats->robustUQ); + + if (!psVectorStats(stats, M4, NULL, NULL, 0)) { + psError(PSPHOT_ERR_UNKNOWN, false, "Unable to perform statistics to measure image quality"); + goto FAIL; + } + float vM4 = stats->robustMedian; + float dM4 = stats->robustStdev; + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4", PS_META_REPLACE, "M_4 = sqrt (M_c4^2 + M_s4^2)", vM4); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4_ER", PS_META_REPLACE, "Stdev of M_4", dM4); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4_LQ", PS_META_REPLACE, "Lower Quartile of M_4", stats->robustLQ); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "IQ_M4_UQ", PS_META_REPLACE, "Upper Quartile of M_4", stats->robustUQ); +#endif + + psLogMsg ("psphot", PS_LOG_WARN, "Image Quality Stats from %ld psf stars : FWHM (major, minor) [pixels]: %f, %f\n", + M2->n, fwhm_major, fwhm_minor); + + psLogMsg ("psphot", PS_LOG_INFO, "M_2 : %f +/- %f, M_3 : %f +/- %f, M_4 : %f +/- %f [pixels^n]\n", + vM2, dM2, vM3, dM3, vM4, dM4); + + psFree(FWHM_MAJOR); + psFree(FWHM_MINOR); + psFree(M2); + psFree(M2c); + psFree(M2s); + psFree(M3); + psFree(M4); + psFree(stats); + + return true; + + FAIL: + psFree(FWHM_MAJOR); + psFree(FWHM_MINOR); + psFree(M2); + psFree(M2c); + psFree(M2s); + psFree(M3); + psFree(M4); + psFree(stats); + + return false; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotInternal.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotInternal.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotInternal.h (revision 41083) @@ -0,0 +1,18 @@ +/* this file is included by the internal, library functions. */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#ifndef PSPHOT_INTERNAL_H +#define PSPHOT_INTERNAL_H + +#include +#include // for strcasecmp +#include // for unlink +#include +#include +#include "psphot.h" +#include "psphotStatsFile.h" + +#endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotIsophotal.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotIsophotal.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotIsophotal.c (revision 41083) @@ -0,0 +1,78 @@ +# include "psphotInternal.h" + +bool psphotIsophotal (pmSource *source, psMetadata *recipe, psImageMaskType maskVal) { + + assert (source->extpars); + assert (source->extpars->profile); + assert (source->extpars->profile->radius); + assert (source->extpars->profile->flux); + + bool status; + + psVector *radius = source->extpars->profile->radius; + psVector *flux = source->extpars->profile->flux; + + // flux at which to measure isophotal parameters + // XXX ISOPHOTAL_FLUX should be specified in mags, need the zero point to get counts/sec + float ISOPHOT_FLUX = psMetadataLookupF32 (&status, recipe, "ISOPHOTAL_FLUX"); + assert (status); + + // find the first bin below the flux level and the last above the level + // XXX can this be done faster with bisection? + // XXX do I need to worry about crazy outliers? + // XXX should i be smoothing or fitting the curve? + int firstBelow = -1; + int lastAbove = -1; + for (int i = 0; i < flux->n; i++) { + if (flux->data.F32[i] > ISOPHOT_FLUX) lastAbove = i; + if ((firstBelow < 0) && (flux->data.F32[i] < ISOPHOT_FLUX)) firstBelow = i; + } + // if we don't go out far enough, we have a problem... + if (lastAbove == flux->n - 1) { + psTrace ("psphot", 5, "did not go out far enough to reach isophotal magnitude"); + // XXX raise a flag ? + return false; + } + if (firstBelow < 0) { + psTrace ("psphot", 5, "did not go out far enough to bound isophotal magnitude: error unmeasured"); + // XXX raise a flag ? + lastAbove = firstBelow; + return false; + } + + // need to examine pixels in this vicinity + float isophotalFluxFirst = 0; + float isophotalFluxLast = 0; + for (int i = 0; i <= PS_MAX(firstBelow, lastAbove); i++) { + if (i <= firstBelow) { + isophotalFluxFirst += flux->data.F32[i]; + } + if (i <= lastAbove) { + isophotalFluxLast += flux->data.F32[i]; + } + } + float isophotalFlux = 0.5*(isophotalFluxLast + isophotalFluxFirst); + float isophotalFluxErr = 0.5*fabs(isophotalFluxLast - isophotalFluxFirst); + + float isophotalRad = 0.5*(radius->data.F32[firstBelow] + radius->data.F32[lastAbove]); + float isophotalRadErr = 0.5*fabs(radius->data.F32[firstBelow] - radius->data.F32[lastAbove]); + + if (!source->extpars->isophot) { + source->extpars->isophot = pmSourceIsophotalValuesAlloc (); + } + + // these are uncalibrated: instrumental mags and pixel units + source->extpars->isophot->mag = -2.5*log10(isophotalFlux); + source->extpars->isophot->magErr = isophotalFluxErr / isophotalFlux; + + source->extpars->isophot->rad = isophotalRad; + source->extpars->isophot->radErr = isophotalRadErr; + + psTrace ("psphot", 5, "Isophot flux:%f +/- %f @ %f +/- %f for %f, %f\n", + source->extpars->isophot->mag, source->extpars->isophot->magErr, + source->extpars->isophot->rad, source->extpars->isophot->radErr, + source->peak->xf, source->peak->yf); + + return true; + +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKernelFromPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKernelFromPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKernelFromPSF.c (revision 41083) @@ -0,0 +1,45 @@ +# include "psphotInternal.h" + +psKernel *psphotKernelFromPSF (pmSource *source, int nPix) { + + assert (source); + assert (source->psfImage); // XXX build if needed? + + int x0 = source->peak->xf - source->psfImage->col0; + int y0 = source->peak->yf - source->psfImage->row0; + + // need to decide on the size: dynamically? statically? + psKernel *psf = psKernelAlloc (-nPix, +nPix, -nPix, +nPix); + + // XXX we should just re-construct a PSF at this location + // psModelAdd (psf->image, NULL, source->modelPSF, PM_MODEL_OP_FULL | PM_MODEL_OP_NORM | PM_MODEL_OP_CENTER); + + // if the realized PSF for this object does not cover the full kernel, give up for now + if (x0 + psf->xMin < 0) goto escape; + if (x0 + psf->xMax >= source->psfImage->numCols) goto escape; + if (y0 + psf->yMin < 0) goto escape; + if (y0 + psf->yMax >= source->psfImage->numRows) goto escape; + + double sum = 0.0; + for (int j = psf->yMin; j <= psf->yMax; j++) { + for (int i = psf->xMin; i <= psf->xMax; i++) { + double value = source->psfImage->data.F32[y0 + j][x0 + i]; + psf->kernel[j][i] = value; + sum += value; + } + } + assert (sum > 0.0); + + // psf must be normalized (integral = 1.0) + for (int i = 0; i < psf->image->numRows; i++) { + for (int j = 0; j < psf->image->numCols; j++) { + psf->image->data.F32[i][j] /= sum; + } + } + + return psf; + +escape: + psFree (psf); + return NULL; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKron.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKron.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKron.c (revision 41083) @@ -0,0 +1,8 @@ +# include "psphotInternal.h" + +bool psphotKron (pmSource *source, psMetadata *recipe, psImageMaskType maskVal) { + + psLogMsg ("psphot", PS_LOG_INFO, "not implemented\n"); + return true; + +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronFlux.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronFlux.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronFlux.c (revision 41083) @@ -0,0 +1,250 @@ +# include "psphotInternal.h" + +bool psphotKronFlux (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Kron Fluxes ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + if (!psphotKronFluxReadout (config, recipe, view, filerule, readout, sources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + + return true; +} + +bool psphotKronFluxReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char *filerule, pmReadout *readout, psArray *sources) { + + bool status = false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping kron fluxes"); + return true; + } + + psTimerStart ("psphot.kron"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend"); + return true; + } + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_KRON_FLUX"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } +# else + if (!psphotKronFlux_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + // psFree(AnalysisRegion); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + psFree (cellGroups); + + psLogMsg ("psphot.kron", PS_LOG_WARN, "measure kron fluxes : %f sec for %ld objects\n", psTimerMark ("psphot.kron"), sources->n); + return true; +} + +bool psphotKronFlux_Threaded (psThreadJob *job) { + + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[2],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[3],PS_TYPE_IMAGE_MASK_DATA); + + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + + // check status of this source's moments + if (!source->moments) continue; + if (!(source->tmpFlags & PM_SOURCE_TMPF_MOMENTS_MEASURED)) continue; + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + + // XXX where are we storing the supplied kron radius? + if (!isfinite(source->moments->Mrf)) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // is this right?? + float windowRadius = 2.5*source->moments->Mrf ; + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + pmSourceRedefinePixels (source, readout, source->peak->x, source->peak->y, windowRadius + 2); + psAssert (source->pixels, "WTF?"); + + // this function populates moments->Mrf,KronFlux,KronFluxErr + psphotKronFluxSource (source, maskVal); + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + + return true; +} + +// measure just the flux in the given aperture (this is probably a single common function -- can I use one of the pmSourcePhotometry functions?) +bool psphotKronFluxSource (pmSource *source, psImageMaskType maskVal) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + + // the peak position is less accurate but less subject to extreme deviations + // XXX do this or just use the peak position? + float dX = source->moments->Mx - source->peak->xf; + float dY = source->moments->My - source->peak->yf; + float dR = hypot(dX, dY); + float Xo = (dR < 2.0) ? source->moments->Mx : source->peak->xf; + float Yo = (dR < 2.0) ? source->moments->My : source->peak->yf; + + // center of mass in subimage. Note: the calculation below uses pixel index, so we correct + // xCM, yCM from pixel coords to pixel index here. + psF32 xCM = Xo - 0.5 - source->pixels->col0; // coord of peak in subimage + psF32 yCM = Yo - 0.5 - source->pixels->row0; // coord of peak in subimage + + // Calculate the Kron magnitude (make this block optional?) + // XXX set the aperture here + float radKron = 2.5*source->moments->Mrf; + float radKron2 = radKron*radKron; + + int nKronPix = 0; + float Sum = 0.0; + float Var = 0.0; + + // set vPix to the source pixels (it may have been set to the + // smoothed image above) + psF32 **vPix = source->pixels->data.F32; + psF32 **vWgt = source->variance->data.F32; + psImageMaskType **vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + + psF32 yDiff = row - yCM; + if (fabs(yDiff) > radKron) continue; + + for (psS32 col = 0; col < source->pixels->numCols ; col++) { + // check mask and value for this pixel + if (vMsk && (vMsk[row][col] & maskVal)) continue; + if (isnan(vPix[row][col])) continue; + + psF32 xDiff = col - xCM; + if (fabs(xDiff) > radKron) continue; + + // radKron is just a function of (xDiff, yDiff) + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > radKron2) continue; + + float pDiff = vPix[row][col]; + psF32 wDiff = vWgt[row][col]; + + Sum += pDiff; + Var += wDiff; + nKronPix ++; + } + } + + // return the flux and error to parameters? + source->moments->KronFlux = Sum; + source->moments->KronFluxErr = sqrt(Var); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronIterate.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronIterate.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronIterate.c (revision 41083) @@ -0,0 +1,657 @@ +# include "psphotInternal.h" + +bool psphotKronRadiusMeasure (pmSource *source, float radius, float minKronRadius, psImageMaskType maskVal, psImage *smoothedPixels); +bool psphotKronFluxMeasure (pmSource *source, psImageMaskType maskVal); + + +bool psphotKronIterate (pmConfig *config, const pmFPAview *view, const char *filerule, int pass) { + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Kron Iterate ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + // psAssert (psf, "missing psf?"); + + if (!psphotKronIterateReadout (config, recipe, view, filerule, readout, sources, psf, i, pass)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +int psphotKapaChannel (int channel); +bool psphotVisualShowMask (int kapaFD, psImage *inImage, const char *name, int channel); +bool psphotVisualRangeImage (int kapaFD, psImage *inImage, const char *name, int channel, float min, float max); + +#ifdef DUMP_KRS +FILE *dumpFile = NULL; +#endif + +bool psphotKronIterateReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char * filerule, pmReadout *readout, psArray *sources, pmPSF *psf, int index, int pass) { + + bool status = false; + +#ifdef DUMP_KRS + if (!dumpFile) { + dumpFile = fopen("kr.txt", "w"); + psAssert (dumpFile, "failed to open kr.txt"); + } + fprintf(dumpFile, "\n\n Input %d\n", index); +#endif + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping masked kron"); + return true; + } + + psTimerStart ("psphot.kron"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + float RADIUS = psMetadataLookupF32 (&status, readout->analysis, "PSF_MOMENTS_RADIUS"); + if (!status) { + RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_MOMENTS_RADIUS"); + } + + float MIN_KRON_RADIUS = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_MIN_KRON"); + if (!status) { + MIN_KRON_RADIUS = 0.25*RADIUS; + } + + int KRON_ITERATIONS = psMetadataLookupS32 (&status, recipe, "KRON_ITERATIONS"); + if (!status) { + KRON_ITERATIONS = 1; + } + + bool KRON_SMOOTH = psMetadataLookupBool (&status, recipe, "KRON_SMOOTH"); + if (!status) { + KRON_SMOOTH = false; + } + float KRON_SMOOTH_SIGMA = psMetadataLookupF32 (&status, recipe, "KRON_SMOOTH_SIGMA"); + if (!status) { + KRON_SMOOTH_SIGMA = 1.7; + } + float KRON_SMOOTH_NSIGMA = psMetadataLookupS32 (&status, recipe, "KRON_SMOOTH_NSIGMA"); + if (!status) { + KRON_SMOOTH_NSIGMA = 2; + } + /* + * Parameter for calculating maximum integration radius based on source's surface + * brightness + * Given minimum surface brightness SBmin and a flux the maximum radius is found from + * + * SBmin = source->flux / (pi * Rmax**2) + * Rmax = sqrt (source->flux ) / sqrt (SBmin * pi) + * + * Now what do we use for SBmin? + * + * SBmin = ( some flux ) / (some area) + * some flux ~ flux of Ns sigma PSF source + * some area ~ K times area of a PSF + * + * flux of Ns sigma source ~ Ns * SKY_STDEV * PSF_EFFECTIVE_AREA + * PSF_EFFECTIVE_AREA = 4 pi sigma_PSF^2 + * (the 4 accounts for the fact that 1 sigma is not the total area, it is + * actually a larger region). + * + * SBmin = Ns * SKY_STDEV * 4 * pi * sigma_PSF^2 / (K * pi * sigma_PSF^2) + * = Ns * SKY_STDEV * 4 / K + * + * We combine the two parameters Ns and K, and the constant 4 into a single recipe value + * KRON_SB_MIN_FACTOR with Ns = 5 and K = 2 the corresponding value + * for KRON_SB_MIN_FACTOR is 5 * 4 / 2 = 10 + */ + float KRON_SB_MIN_FACTOR = psMetadataLookupF32 (&status, recipe, "KRON_SB_MIN_FACTOR"); + if (!status) { + KRON_SB_MIN_FACTOR = 10; + } + float SKY_STDEV = psMetadataLookupF32 (&status, readout->analysis, "MSKY_DEV"); + float KRON_SB_MIN_DIVISOR = sqrt ( M_PI * KRON_SB_MIN_FACTOR * SKY_STDEV ); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend"); + return true; + } + + // pass 3 we only measure fluxes for matched sources so we don't need smooth pixels or multiple iterations + if (pass == 3) { + KRON_SMOOTH = false; + KRON_ITERATIONS = 1; + } + +# if (PS_TRACE_ON) + fprintf (stderr, "--- starting KRON ---\n"); +#endif + + // We measure the Kron Radius on a smoothed copy of the readout image + psImage *smoothedImage = NULL; + if (KRON_SMOOTH) { + psTimerStart ("psphot.kron.smooth"); + + // Build the smoothed source image + // Replace the subtracted sources + psphotReplaceAllSourcesReadout(config, view, filerule, index, recipe, false); + psLogMsg ("psphot.kron", PS_LOG_INFO, "replaced %ld raw sources %f sec\n", sources->n, psTimerMark ("psphot.kron.smooth")); + + // smoothedImage = psImageCopy(NULL, readout->image, PS_TYPE_F32); + // psImageSmooth(smoothedImage, KRON_SMOOTH_SIGMA, KRON_SMOOTH_NSIGMA); + + // Copy the image and smooth + psTimerStart ("psphot.kron.smooth"); + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading in psImageConvolve + smoothedImage = psImageSmoothNoMask_Threaded (NULL, readout->image, KRON_SMOOTH_SIGMA, KRON_SMOOTH_NSIGMA, 0.25); + psImageConvolveSetThreads(oldThreads); + psLogMsg ("psphot.kron", PS_LOG_INFO, "smoothed image %f sec\n", psTimerMark ("psphot.kron.smooth")); + + // remove the sources + psTimerStart ("psphot.kron.smooth"); + psphotRemoveAllSourcesReadout( config, view, filerule, index, recipe, false ); + psLogMsg ("psphot.kron", PS_LOG_INFO, "removed %ld raw sources %f sec\n", sources->n, psTimerMark ("psphot.kron.smooth")); + + // Now subtract smooth versions of the sources from the smoothed image + psTimerStart ("psphot.kron.smooth.sources"); + for (int i=0; i< sources->n; i++) { + pmSource *source = sources->data[i]; + // If source has been subtracted from the readout image subtract a "smoothed" version from the smoothedImage + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + // cache copy of smoothedPixels in the source + // tmpPtr is for use by a single "module" and must be null otherwise + psAssert(source->tmpPtr == NULL, "source->tmpPtr is not null!"); + + psImage *smoothedPixels = psImageSubset(smoothedImage, source->region); + source->tmpPtr = (psPtr) smoothedPixels; + pmSourceSmoothOp(source, PM_MODEL_OP_FUNC, smoothedPixels, KRON_SMOOTH_SIGMA, false, maskVal, 0, 0); + } + } + psLogMsg ("psphot.kron", PS_LOG_INFO, "removed %ld smoothed sources %f sec\n", sources->n, psTimerMark ("psphot.kron.smooth.sources")); + + } + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_KRON_ITERATE"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, smoothedImage); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, RADIUS, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, MIN_KRON_RADIUS, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, KRON_ITERATIONS, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, KRON_SMOOTH_SIGMA, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, KRON_SB_MIN_DIVISOR,PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, pass, PS_TYPE_S32); + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } +# else + if (!psphotKronIterate_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + // psFree(AnalysisRegion); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + psFree (cellGroups); + if (KRON_SMOOTH) { + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + psFree(source->tmpPtr); + source->tmpPtr = NULL; + } + } + psFree (smoothedImage); + + psLogMsg ("psphot.kron", PS_LOG_WARN, "measure masked kron magnitudes : %f sec for %ld objects\n", psTimerMark ("psphot.kron"), sources->n); + return true; +} + +bool psphotKronIterate_Threaded (psThreadJob *job) { + + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psImage *smoothedImage = job->args->data[2]; + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[3],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[4],PS_TYPE_IMAGE_MASK_DATA); + float RADIUS = PS_SCALAR_VALUE(job->args->data[5],F32); + float MIN_KRON_RADIUS = PS_SCALAR_VALUE(job->args->data[6],F32); + int KRON_ITERATIONS = PS_SCALAR_VALUE(job->args->data[7],S32); + float KRON_SMOOTH_SIGMA = PS_SCALAR_VALUE(job->args->data[8],F32); + float KRON_SB_MIN_DIVISOR = PS_SCALAR_VALUE(job->args->data[9],F32); + int pass = PS_SCALAR_VALUE(job->args->data[10],S32); + + bool measureRadius = true; + if (pass == 3) { + measureRadius = false; + } + + for (int j = 0; j < KRON_ITERATIONS; j++) { + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + if (!source->moments) continue; + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // in pass 3 we only measure the flux for matched sources + if ((pass == 3) && !(source->mode2 & PM_SOURCE_MODE2_MATCHED)) continue; + +# if (0) +# define TEST_X 653 +# define TEST_Y 466 + if ((fabs(source->peak->xf - TEST_X) < 5) && (fabs(source->peak->yf - TEST_Y) < 5)) { + fprintf (stderr, "test object\n"); + } +# undef TEST_X +# undef TEST_Y +# endif + + if (measureRadius) { + // check status of this source's moments + // XXX: I don't think that we have to apply these restrictions since we dropped the window function + if (!(source->tmpFlags & PM_SOURCE_TMPF_MOMENTS_MEASURED)) continue; + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + } + + if (!isfinite(source->moments->Mrf)) { + // Once we save a bad Mrf measurement we give up on this source + // XXX: is this the right thing to do? + continue; + } + + // replace object in image + bool reSubtract = false; + psImage *smoothedPixels = NULL; + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + smoothedPixels = (psImage *) source->tmpPtr; + if (smoothedPixels) { + pmSourceSmoothOp(source, PM_MODEL_OP_FUNC, smoothedPixels, KRON_SMOOTH_SIGMA, true, maskVal, 0, 0); + } + reSubtract = true; + } + + // On first iteration set window radius to sky radius (if valid). We also use this on subsequent + // iterations if we cannot find a better limit + float maxWindow; + if (measureRadius) { + maxWindow = isfinite(source->skyRadius) ? source->skyRadius : RADIUS; + } else { + maxWindow = source->moments->Mrf; + } + if (j > 0) { + // on subsequent iterations we use a factor times the previous radial moment value + // limited to a maximum value that depends on the surface brightness of the source + if (KRON_SB_MIN_DIVISOR) { + // Limit window radius based on surface brightness if we have a good measurement of kron flux + if (isfinite(source->moments->KronFlux) && (source->moments->KronFlux > 0)) { + float Rmax = sqrt(source->moments->KronFlux) / KRON_SB_MIN_DIVISOR; + + if (isfinite(source->moments->Mrf) && source->moments->Mrf > 0) { + maxWindow = PS_MIN(6.0*source->moments->Mrf, Rmax); + } else { + maxWindow = PS_MIN(Rmax, maxWindow); + } + } + } else { + // old recipe, no surface brightness cut + maxWindow = isfinite(source->moments->Mrf) ? 6.0*source->moments->Mrf : RADIUS; + } + } + float windowRadius = PS_MAX(RADIUS, maxWindow); + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + bool extend = pmSourceRedefinePixels (source, readout, source->peak->x, source->peak->y, windowRadius + 2); + psAssert (source->pixels, "redefine pixels failed?"); + if (extend && smoothedPixels) { + psFree(source->tmpPtr); + smoothedPixels = psImageSubset(smoothedImage, source->region); + psAssert (smoothedPixels, "redefine smoothed pixels failed?"); + source->tmpPtr = (psPtr) smoothedPixels ; + } + + bool measureFlux = true; + if (measureRadius) { + measureFlux = psphotKronRadiusMeasure (source, windowRadius, MIN_KRON_RADIUS, maskVal, smoothedPixels); + } + + if (measureFlux) { + // Make sure the sources images are large enough for the measured Kron Radius + bool extend = pmSourceRedefinePixels (source, readout, source->peak->x, source->peak->y, + 2.5 * source->moments->Mrf + 2); + + psAssert (source->pixels, "redefine pixels failed?"); + if (extend && smoothedPixels) { + psFree(source->tmpPtr); + smoothedPixels = psImageSubset(smoothedImage, source->region); + psAssert (smoothedPixels, "redefine smoothed pixels failed?"); + source->tmpPtr = (psPtr) smoothedPixels ; + } + + // this function populates moments->Mrf,KronFlux,KronFluxErr, KronFinner, and KronFouter + psphotKronFluxMeasure (source, maskVal); + } + + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + +#ifdef DUMP_KRS + fprintf(dumpFile, "%7d %6.1f %6.1f %6.1f %6.1f %6.1f %6.1f %2d\n", source->id, source->moments->Mrf, MrfPrior, maxWindow, windowRadius, source->peak->xf, source->peak->yf, source->imageID); +#endif + + // if we subtracted it above, re-subtract the object, leave local sky + if (reSubtract) { + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + if (smoothedPixels) { + pmSourceSmoothOp(source, PM_MODEL_OP_FUNC, smoothedPixels, KRON_SMOOTH_SIGMA, false, maskVal, 0, 0); + } + } + } + } + + return true; +} + +bool psphotKronRadiusMeasure (pmSource *source, float radius, float minKronRadius, psImageMaskType maskVal, + psImage *smoothedPixels) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + PS_ASSERT_FLOAT_LARGER_THAN(radius, 0.0, false); + + psF32 R2 = PS_SQR(radius); + + // a note about coordinates: coordinates of objects throughout psphot refer to the primary + // image coordinates. the source->pixels image has an offset relative to its parent of + // col0,row0: a pixel (x,y) in the primary image has coordinates of (x-col0, y-row0) in + // this subimage. we subtract off the peak coordinates, adjusted to this subimage, to have + // minimal round-off error in the sums. since these values are subtracted just to minimize + // the dynamic range and are added back below, the exact value does not matter. these are + // (int) so they can be used in the image index below. + + // Now calculate higher-order moments, using the above-calculated first moments to adjust coordinates + // Xn = SUM (x - xc)^n * (z - sky) + + psF32 RF = 0.0; + psF32 RS = 0.0; + + // the peak position is less accurate but less subject to extreme deviations + float dX = source->moments->Mx - source->peak->xf; + float dY = source->moments->My - source->peak->yf; + float dR = hypot(dX, dY); + float Xo = (dR < 2.0) ? source->moments->Mx : source->peak->xf; + float Yo = (dR < 2.0) ? source->moments->My : source->peak->yf; + + // center of mass in subimage. Note: the calculation below uses pixel index, so we correct + // xCM, yCM from pixel coords to pixel index here. + psF32 xCM = Xo - 0.5 - source->pixels->col0; // coord of peak in subimage + psF32 yCM = Yo - 0.5 - source->pixels->row0; // coord of peak in subimage + + psF32 **vPix; + if (smoothedPixels) { + vPix = smoothedPixels->data.F32; + } else { + vPix = source->pixels->data.F32; + } + + psImageMaskType **vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + { + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + + psF32 yDiff = row - yCM; + if (fabs(yDiff) > radius) continue; + + // coordinate of mirror pixel + int yFlip = yCM - yDiff; + if (yFlip < 0) continue; + if (yFlip >= source->pixels->numRows) continue; + + for (psS32 col = 0; col < source->pixels->numCols ; col++) { + // check mask and value for this pixel + if (vMsk && (vMsk[row][col] & maskVal)) continue; + if (isnan(vPix[row][col])) continue; + + psF32 xDiff = col - xCM; + if (fabs(xDiff) > radius) continue; + + // coordinate of mirror pixel + int xFlip = xCM - xDiff; + if (xFlip < 0) continue; + if (xFlip >= source->pixels->numCols) continue; + + // check mask and value for mirror pixel + if (vMsk && (vMsk[yFlip][xFlip] & maskVal)) continue; + if (isnan(vPix[yFlip][xFlip])) continue; + + // radius is just a function of (xDiff, yDiff) + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > R2) continue; + + float fDiff1 = vPix[row][col]; + float fDiff2 = vPix[yFlip][xFlip]; + + float pDiff = (fDiff1 > 0.0) ? sqrt(fabs(fDiff1*fDiff2)) : -sqrt(fabs(fDiff1*fDiff2)); + + // Kron Flux uses the 1st radial moment + psF32 rf = pDiff * sqrt(r2); + psF32 rs = 0.5 * (fDiff1 + fDiff2); + + RF += rf; + RS += rs; + } + } + + float MrfTry = RF/RS; + if (RF <= 0. || RS <= 0 || !isfinite(MrfTry)) { + // We did not get a good measurement + source->moments->Mrf = NAN; + source->moments->KronFlux = NAN; + source->moments->KronFluxErr = NAN; + source->moments->KronFinner = NAN; + source->moments->KronFouter = NAN; + return false; + } + + float Mrf = MAX(minKronRadius, MrfTry); + // Saturate the 1st radial moment + if (sqrt(source->peak->detValue) < 10.0) { + Mrf = MIN (radius, Mrf); + } + source->moments->Mrf = Mrf; + } + return true; +} + +bool psphotKronFluxMeasure(pmSource *source, psImageMaskType maskVal) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + + // a note about coordinates: coordinates of objects throughout psphot refer to the primary + // image coordinates. the source->pixels image has an offset relative to its parent of + // col0,row0: a pixel (x,y) in the primary image has coordinates of (x-col0, y-row0) in + // this subimage. we subtract off the peak coordinates, adjusted to this subimage, to have + // minimal round-off error in the sums. since these values are subtracted just to minimize + // the dynamic range and are added back below, the exact value does not matter. these are + // (int) so they can be used in the image index below. + + // Now calculate higher-order moments, using the above-calculated first moments to adjust coordinates + // Xn = SUM (x - xc)^n * (z - sky) + + // the peak position is less accurate but less subject to extreme deviations + float dX = source->moments->Mx - source->peak->xf; + float dY = source->moments->My - source->peak->yf; + float dR = hypot(dX, dY); + float Xo = (dR < 2.0) ? source->moments->Mx : source->peak->xf; + float Yo = (dR < 2.0) ? source->moments->My : source->peak->yf; + + // center of mass in subimage. Note: the calculation below uses pixel index, so we correct + // xCM, yCM from pixel coords to pixel index here. + psF32 xCM = Xo - 0.5 - source->pixels->col0; // coord of peak in subimage + psF32 yCM = Yo - 0.5 - source->pixels->row0; // coord of peak in subimage + + psF32 **vPix = source->pixels->data.F32; + psF32 **vWgt = source->variance->data.F32; + + psImageMaskType **vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + source->moments->KronFlux = NAN; + source->moments->KronFluxErr = NAN; + source->moments->KronFinner = NAN; + source->moments->KronFouter = NAN; + + // Calculate the Kron fluxes + float radKinner = 1.0 * source->moments->Mrf; + float radKron = 2.5 * source->moments->Mrf; + float radKouter = 4.0 * source->moments->Mrf; + + float limitRadius = MIN (radKouter, source->windowRadius); + if (radKouter > source->windowRadius) { + // This happens but the measurement isn't important enough to allocate the extra pixels + // psWarning ("outer kron radius: %f is larger than windowRadius: %f for %d\n", + // radKouter, source->windowRadius, source->id); + limitRadius = MIN (radKron, source->windowRadius); + } + if (radKron > source->windowRadius) { + // caller should have prevented this from happening + psWarning ("kron radius: %f is larger than windowRadius: %f for %d\n", + radKron, source->windowRadius, source->id); + return false; + } + + float Sum = 0.0; + float Var = 0.0; + float SumInner = 0.0; + float SumOuter = 0.0; + + // set vPix to the source pixels (it may have been set to the smoothed image above) + vPix = source->pixels->data.F32; + + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + + psF32 yDiff = row - yCM; + if (fabs(yDiff) > limitRadius) continue; + + for (psS32 col = 0; col < source->pixels->numCols ; col++) { + // check mask and value for this pixel + if (vMsk && (vMsk[row][col] & maskVal)) continue; + if (isnan(vPix[row][col])) continue; + + psF32 xDiff = col - xCM; + if (fabs(xDiff) > limitRadius) continue; + + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + psF32 r = sqrt(r2); + + float pDiff = vPix[row][col]; + psF32 wDiff = vWgt[row][col]; + + if (r > radKinner && r < radKron) { + SumInner += pDiff; + } + if (r < radKron) { + Sum += pDiff; + Var += wDiff; + } + if (r > radKron && r < radKouter) { + SumOuter += pDiff; + } + } + } + + source->moments->KronFlux = Sum; + source->moments->KronFluxErr = sqrt(Var); + source->moments->KronFinner = SumInner; + + // only save radKouter if the radius is inside the integration radius limit + if (radKouter <= limitRadius) { + source->moments->KronFouter = SumOuter; + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronMasked.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronMasked.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotKronMasked.c (revision 41083) @@ -0,0 +1,487 @@ +# include "psphotInternal.h" +# ifndef ROUND +# define ROUND(X) ((int) ((X) + 0.5*SIGN(X))) +# endif + +bool psphotKronMask (pmSource *source, psImage *kronMask, psImageMaskType markVal); +bool psphotKronMag (pmSource *source, float radius, float minKronRadius, psImageMaskType maskVal); + +bool psphotKronMasked (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + // psAssert (psf, "missing psf?"); + + if (!psphotKronMaskedReadout (config, recipe, view, readout, sources, psf)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +int psphotKapaChannel (int channel); +bool psphotVisualShowMask (int kapaFD, psImage *inImage, const char *name, int channel); +bool psphotVisualRangeImage (int kapaFD, psImage *inImage, const char *name, int channel, float min, float max); + +bool psphotKronMaskedReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf) { + + bool status = false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping masked kron"); + return true; + } + + psTimerStart ("psphot.kron"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + float RADIUS = psMetadataLookupF32 (&status, readout->analysis, "PSF_MOMENTS_RADIUS"); + if (!status) { + RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_MOMENTS_RADIUS"); + } + + float MIN_KRON_RADIUS = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_MIN_KRON"); + if (!status) { + MIN_KRON_RADIUS = 0.25*RADIUS; + } + + float EXT_FIT_MAX_RADIUS = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_RADIUS"); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // psphotSaveImage (NULL, readout->mask, "kron.unmasked.fits"); + + pmSourcePhotometryMode photMode = PM_SOURCE_PHOT_PSFONLY; + + // XXX tmp visualization + // int kapa = psphotKapaChannel (1); + + // generate the mask image: increment counter for every source overlapping the pixel + psImage *kronMask = psImageAlloc (readout->image->numCols, readout->image->numRows, PS_TYPE_S32); + psImageInit (kronMask, 0); + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + + // allocate space for moments + if (!source->moments) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // save the PSF-based values + // source->moments->KronFluxPSF = source->moments->KronFlux; + // source->moments->KronFluxPSFErr = source->moments->KronFluxErr; + // source->moments->KronRadiusPSF = source->moments->Mrf; + + // iterate to the window radius + float windowRadius = RADIUS; + for (int j = 0; j < 4; j++) { + // XXX use some S/N criterion to limit? + // if (source->moments->KronFlux / source->moments->KronFluxErr > 10.0) { + // windowRadius = PS_MAX (RADIUS, 10*source->moments->Mrf); + // } + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + pmSourceRedefinePixels (source, readout, source->peak->x, source->peak->y, windowRadius + 2); + + // mask the pixels not contained by the footprint + // if (psphotMaskFootprint (readout, source, markVal)) { + // // psphotVisualShowMask (kapa, source->maskObj, "mask", 2); + // // psphotVisualRangeImage (kapa, source->pixels, "image", 1, -50, 300); + // // fprintf (stderr, "masked\n"); + // } + + // mask the pixels associated with near neighbors + if (psphotKronMask (source, kronMask, markVal)) { + // psphotVisualShowMask (kapa, source->maskObj, "mask", 2); + // psphotVisualRangeImage (kapa, source->pixels, "image", 1, -50, 300); + // fprintf (stderr, "masked\n"); + } + // this function populates moments->Mrf,KronFlux,KronFluxErr + // XXX what about KronInner, KronOuter, etc? + psphotKronMag (source, windowRadius, MIN_KRON_RADIUS, maskVal); + windowRadius = PS_MIN(PS_MAX(RADIUS, 4.0*source->moments->Mrf), EXT_FIT_MAX_RADIUS); + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + } + pmSourceMagnitudes (source, psf, photMode, maskVal, markVal, source->apRadius); + float kmag = -2.5*log10(source->moments->KronFlux); + if (source->psfMag - kmag > 0.25) { + // psphotVisualShowMask (kapa, source->maskObj, "mask", 1); + // psphotVisualRangeImage (kapa, source->pixels, "image", 0, -50, 300); + // fprintf (stderr, "psf: %f, kron: %f, dmag: %f\n", source->psfMag, kmag, source->psfMag - kmag); + // fprintf (stderr, "continue\n"); + } + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + + // psphotSaveImage (NULL, readout->mask, "kron.masked.fits"); + // psphotSaveImage (NULL, kronMask, "kronmask.fits"); + + psLogMsg ("psphot.kron", PS_LOG_DETAIL, "measure masked kron magnitudes : %f sec for %ld objects\n", psTimerMark ("psphot.kron"), sources->n); + + psFree (kronMask); + return true; +} + +# define WEIGHTED 0 + +bool psphotKronMag (pmSource *source, float radius, float minKronRadius, psImageMaskType maskVal) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + PS_ASSERT_FLOAT_LARGER_THAN(radius, 0.0, false); + + psF32 R2 = PS_SQR(radius); + +# if (WEIGHTED) + float rsigma2 = 0.5 / PS_SQR(radius / 4.0); // use a Gaussian window with sigma = R_window / 2 +# endif + + // a note about coordinates: coordinates of objects throughout psphot refer to the primary + // image coordinates. the source->pixels image has an offset relative to its parent of + // col0,row0: a pixel (x,y) in the primary image has coordinates of (x-col0, y-row0) in + // this subimage. we subtract off the peak coordinates, adjusted to this subimage, to have + // minimal round-off error in the sums. since these values are subtracted just to minimize + // the dynamic range and are added back below, the exact value does not matter. these are + // (int) so they can be used in the image index below. + + // Now calculate higher-order moments, using the above-calculated first moments to adjust coordinates + // Xn = SUM (x - xc)^n * (z - sky) + + psF32 RF = 0.0; + psF32 RS = 0.0; + + // the peak position is less accurate but less subject to extreme deviations + float dX = source->moments->Mx - source->peak->xf; + float dY = source->moments->My - source->peak->yf; + float dR = hypot(dX, dY); + float Xo = (dR < 2.0) ? source->moments->Mx : source->peak->xf; + float Yo = (dR < 2.0) ? source->moments->My : source->peak->yf; + + // center of mass in subimage. Note: the calculation below uses pixel index, so we correct + // xCM, yCM from pixel coords to pixel index here. + psF32 xCM = Xo - 0.5 - source->pixels->col0; // coord of peak in subimage + psF32 yCM = Yo - 0.5 - source->pixels->row0; // coord of peak in subimage + + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + + psF32 yDiff = row - yCM; + if (fabs(yDiff) > radius) continue; + + psF32 *vPix = source->pixels->data.F32[row]; + + psImageMaskType *vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[row]; + + for (psS32 col = 0; col < source->pixels->numCols ; col++, vPix++) { + if (vMsk) { + if (*vMsk & maskVal) { + vMsk++; + continue; + } + vMsk++; + } + if (isnan(*vPix)) continue; + + psF32 xDiff = col - xCM; + if (fabs(xDiff) > radius) continue; + + // radius is just a function of (xDiff, yDiff) + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > R2) continue; + +# if (WEIGHTED) + float z = r2 * rsigma2; + assert (z >= 0.0); + float weight = exp(-z); + psF32 pDiff = *vPix * weight; +# else + psF32 pDiff = *vPix; +# endif + + // Kron Flux uses the 1st radial moment (maybe Gaussian windowed?) + psF32 rf = pDiff * sqrt(r2); + psF32 rs = pDiff; + + RF += rf; + RS += rs; + } + } + + // Saturate the 1st radial moment + float Mrf = MAX(minKronRadius, RF/RS); + if (sqrt(source->peak->detValue) < 10.0) { + Mrf = MIN (radius, Mrf); + } + + // Calculate the Kron magnitude (make this block optional?) + float radKron = 2.5*Mrf; + float radKron2 = radKron*radKron; + + int nKronPix = 0; + float Sum = 0.0; + float Var = 0.0; + + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + + psF32 yDiff = row - yCM; + if (fabs(yDiff) > radKron) continue; + + psF32 *vPix = source->pixels->data.F32[row]; + psF32 *vWgt = source->variance->data.F32[row]; + + psImageMaskType *vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[row]; + + for (psS32 col = 0; col < source->pixels->numCols ; col++, vPix++, vWgt++) { + if (vMsk) { + if (*vMsk & maskVal) { + vMsk++; + continue; + } + vMsk++; + } + if (isnan(*vPix)) continue; + + psF32 xDiff = col - xCM; + if (fabs(xDiff) > radKron) continue; + + // radKron is just a function of (xDiff, yDiff) + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > radKron2) continue; + +# if (WEIGHTED) + float z = r2 * rsigma2; + assert (z >= 0.0); + float weight = exp(-z); + psF32 pDiff = *vPix * weight; + psF32 wDiff = *vWgt * weight; +# else + psF32 pDiff = *vPix; + psF32 wDiff = *vWgt; +# endif + + Sum += pDiff; + Var += wDiff; + nKronPix ++; + } + } + + source->moments->Mrf = Mrf; + source->moments->KronFlux = Sum; + source->moments->KronFluxErr = sqrt(Var); + // source->moments->KronFlux = Sum * M_PI * PS_SQR(radKron) / nKronPix; + // source->moments->KronFluxErr = sqrt(Var) * M_PI * PS_SQR(radKron) / nKronPix; + + return true; +} + +enum {Y_P, Y_M, X_P, X_M}; + +bool psphotKronMask (pmSource *source, psImage *kronMask, psImageMaskType markVal) { + + int X, Y, e, dXs, dYs; + + if (!source->peak) return false; + if (!source->peak->saddlePoints) return false; + if (!source->peak->saddlePoints->n) return false; + + // return true; + + int xMax = source->maskObj->numCols; + int yMax = source->maskObj->numRows; + // int xOff = source->maskObj->col0; + // int yOff = source->maskObj->row0; + + int Xo = ROUND(source->peak->xf) - source->maskObj->col0; + int Yo = ROUND(source->peak->yf) - source->maskObj->row0; + + for (int i = 0; i < source->peak->saddlePoints->n; i++) { + + psVector *saddlePoint = source->peak->saddlePoints->data[i]; + int Xs = ROUND(saddlePoint->data.S32[0]) - source->maskObj->col0; + int Ys = ROUND(saddlePoint->data.S32[1]) - source->maskObj->row0; + + // fprintf (stderr, "mask %d,%d @ %d,%d (%d,%d)\n", Xo, Yo, Xs, Ys, saddlePoint->data.S32[0], saddlePoint->data.S32[1]); + + // We want to mask the pixels between the edge of the image and the line perpendicular + // to the connecting line. Call dX,dY = (Xs-Xo,Ys-Yo). There are 4 cases: + // +y : dY > 0, fabs(dY) > fabs(dX) + // -y : dY < 0, fabs(dY) > fabs(dX) + // +x : dX > 0, fabs(dX) > fabs(dY) + // -x : dX < 0, fabs(dX) > fabs(dY) + + int dX = Xs - Xo; + int dY = Ys - Yo; + + // +y : dY > 0, fabs(dY) > fabs(dX) + // -y : dY < 0, fabs(dY) > fabs(dX) + if (fabs(dY) > fabs(dX)) { + // points to right of saddle + Y = Ys; + e = 0; + dXs = (dY > 0) ? +dY : -dY; // this forces dXs > 0 + dYs = (dY > 0) ? -dX : +dX; + for (X = Xs; X < xMax; X++) { + if (dY > 0) { + for (int Ym = Y; Ym < yMax; Ym++) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Ym][X] |= markVal; + // kronMask->data.S32[Ym+yOff][X+xOff] = source->id; + } + } else { + for (int Ym = Y; Ym >= 0; Ym--) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Ym][X] |= markVal; + // kronMask->data.S32[Ym+yOff][X+xOff] = source->id; + } + } + e += dYs; + int e2 = 2 * e; + if (e2 > dXs) { + Y ++; + e -= dXs; + } + if (e2 < -dXs) { + Y --; + e += dXs; + } + if (Y >= yMax) break; + if (Y < 0) break; + } + // points to left of saddle + Y = Ys; + e = 0; + for (X = Xs; X >= 0; X--) { + if (dY > 0) { + for (int Ym = Y; Ym < yMax; Ym++) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Ym][X] |= markVal; + // kronMask->data.S32[Ym+yOff][X+xOff] = source->id; + } + } else { + for (int Ym = Y; Ym >= 0; Ym--) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Ym][X] |= markVal; + // kronMask->data.S32[Ym+yOff][X+xOff] = source->id; + } + } + e -= dYs; + int e2 = 2 * e; + if (e2 > dXs) { + Y ++; + e -= dXs; + } + if (e2 < -dXs) { + Y --; + e += dXs; + } + if (Y >= yMax) break; + if (Y < 0) break; + } + } else { + // points to right of saddle + X = Xs; + e = 0; + dXs = (dX > 0) ? -dY : +dY; + dYs = (dX > 0) ? +dX : -dX; // this forces dYs > 0 + for (Y = Ys; Y < yMax; Y++) { + if (dX > 0) { + for (int Xm = X; Xm < xMax; Xm++) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Y][Xm] |= markVal; + // kronMask->data.S32[Y+yOff][Xm+xOff] = source->id; + } + } else { + for (int Xm = X; Xm >= 0; Xm--) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Y][Xm] |= markVal; + // kronMask->data.S32[Y+yOff][Xm+xOff] = source->id; + } + } + e += dXs; + int e2 = 2 * e; + if (e2 > dYs) { + X ++; + e -= dYs; + } + if (e2 < -dYs) { + X --; + e += dYs; + } + if (X >= yMax) break; + if (X < 0) break; + } + // points to left of saddle + X = Xs; + e = 0; + for (Y = Ys; Y >= 0; Y--) { + if (dX > 0) { + for (int Xm = X; Xm < xMax; Xm++) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Y][Xm] |= markVal; + // kronMask->data.S32[Y+yOff][Xm+xOff] = source->id; + } + } else { + for (int Xm = X; Xm >= 0; Xm--) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[Y][Xm] |= markVal; + // kronMask->data.S32[Y+yOff][Xm+xOff] = source->id; + } + } + e -= dXs; + int e2 = 2 * e; + if (e2 > dYs) { + X ++; + e -= dYs; + } + if (e2 < -dYs) { + X --; + e += dYs; + } + if (X >= yMax) break; + if (X < 0) break; + } + } + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLensing.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLensing.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLensing.c (revision 41083) @@ -0,0 +1,356 @@ +# include "psphotInternal.h" + +// calculate lensing parameters (we have the moments already) +bool psphotLensing (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Lensing Parameters ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full extended source non-linear fits? + if (!psMetadataLookupBool (&status, recipe, "LENSING_PARAMETERS")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping extended source fits\n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotLensingReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure lensing parameters for %s entry %d", filerule, i); + return false; + } + if (!psphotLensingPSFtrendsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure lensing parameters for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// this block measures the lensing parameters for all objects +bool psphotLensingReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + bool status; + pmSource *source; + + psTimerStart ("psphot.lensing"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // if we have measured the window, we will be saving the modified version of these recipe values on readout->analysis + float SIGMA = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + SIGMA = psMetadataLookupF32 (&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping lensing"); + return true; + } + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + + pmMoments *moments = source->moments; + + if (!source->lensingOBJ) { + source->lensingOBJ = pmSourceLensingAlloc (); + } + + // XXX big objects (eg, saturated stars) use 3*SIGMA. I'm ignoring this for now + pmSourceLensingShearFromMoments (source->lensingOBJ, moments, SIGMA); + pmSourceLensingSmearFromMoments (source->lensingOBJ, moments, SIGMA); + } + + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "LENS_OBJ", PS_META_REPLACE, "per-object lensing stats measured", true); + psLogMsg ("psphot.lensing", PS_LOG_DETAIL, "calculate lensing parameters for %d objects\n", (int) sources->n); + + return true; +} + +// generate image maps for the Xij,ei elements based only on good stars +bool psphotLensingPSFtrendsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + int Nfail = 0; + bool status; + pmSource *source; + + psTimerStart ("psphot.lensing"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping lensing psf trends"); + return true; + } + + // gather the stats to assess the aperture residuals + int Npsf = 0; + psVector *psfX11sm = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psfX12sm = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psfX22sm = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psf_e1sm = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psf_e2sm = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psfX11sh = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psfX12sh = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psfX22sh = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psf_e1sh = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psf_e2sh = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psf_e1 = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *psf_e2 = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *xPos = psVectorAllocEmpty (300, PS_TYPE_F32); + psVector *yPos = psVectorAllocEmpty (300, PS_TYPE_F32); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // only use good stars: + if (source->type != PM_SOURCE_TYPE_STAR) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + if (source->mode & PM_SOURCE_MODE_BLEND) continue; + if (source->mode & PM_SOURCE_MODE_FAIL) continue; + if (source->mode & PM_SOURCE_MODE_POOR) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) continue; + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode2 & PM_SOURCE_MODE2_MATCHED) continue; // XXX ??? + + // use psf - ap to limit to good stars? + // XXX need to be sure if this is before or after ap corrections are done + if (!isfinite(source->apMag)) continue; + if (!isfinite(source->psfMag)) continue; + + // XXX make this user-configurable? + if (source->psfMagErr > 0.03) continue; + + // aperture residual for this source + float dap = source->apMag - source->psfMag; + + // good stars have |dap| < 0.2 + if (fabs(dap) > 0.2) { + Nfail ++; + psTrace ("psphot", 3, "fail : bad dap: %f", dap); + continue; + } + + pmModel *model = source->modelPSF; + + float R = source->moments->Mxx + source->moments->Myy; + source->lensingOBJ->e1 = (source->moments->Mxx - source->moments->Myy) / R; + source->lensingOBJ->e2 = 2*source->moments->Mxy / R; + + psVectorAppend (xPos, model->params->data.F32[PM_PAR_XPOS]); + psVectorAppend (yPos, model->params->data.F32[PM_PAR_YPOS]); + psVectorAppend (psfX11sm, source->lensingOBJ->smear->X11); + psVectorAppend (psfX12sm, source->lensingOBJ->smear->X12); + psVectorAppend (psfX22sm, source->lensingOBJ->smear->X22); + psVectorAppend (psf_e1sm, source->lensingOBJ->smear->e1); + psVectorAppend (psf_e2sm, source->lensingOBJ->smear->e2); + psVectorAppend (psfX11sh, source->lensingOBJ->shear->X11); + psVectorAppend (psfX12sh, source->lensingOBJ->shear->X12); + psVectorAppend (psfX22sh, source->lensingOBJ->shear->X22); + psVectorAppend (psf_e1sh, source->lensingOBJ->shear->e1); + psVectorAppend (psf_e2sh, source->lensingOBJ->shear->e2); + psVectorAppend (psf_e1, source->lensingOBJ->e1); + psVectorAppend (psf_e2, source->lensingOBJ->e2); + Npsf ++; + } + + // this is a bit tricky, because we have two cases (MAP vs POLY), and they have a different + // definition for 'order' (order_MAP = order_POLY + 1). in addition, we have a + // user-specified MAX order, which we should respect, regardless of the mode + + // set the max order (0 = constant) which the number of psf stars can support: + // we require only 3 stars for n = 0, increase stars / cell for higher order + if (Npsf < 3) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources passed selection criteria, skipping lensing psf trends"); + return true; + } + int LensingTrendOrder = 0; + if (Npsf >= 16) LensingTrendOrder = 1; // 4 cells + if (Npsf >= 54) LensingTrendOrder = 2; // 9 cells + if (Npsf >= 128) LensingTrendOrder = 3; // 16 cells + if (Npsf >= 300) LensingTrendOrder = 4; // 25 cells + if (Npsf > 576) LensingTrendOrder = 5; // 36 cells + + pmTrend2DMode mode = PM_TREND_MAP; + if (mode == PM_TREND_MAP) { + LensingTrendOrder ++; + } + + // Nx,Ny are the superpixel size; but I need to choose them to allow a max of LensingTrendOrder values + // I'm not sure this is really sensible, but... + int NX = readout->image->numCols; + int NY = readout->image->numRows; + int Nx, Ny; + if (NX > NY) { + Nx = LensingTrendOrder; + Ny = PS_MAX (1, (int)(LensingTrendOrder * (NY / (float)(NX)) + 0.5)); + } else { + Ny = LensingTrendOrder; + Nx = PS_MAX (1, (int)(LensingTrendOrder * (NX / (float)(NY)) + 0.5)); + } + + psLogMsg ("psphot.lensing", PS_LOG_DETAIL, "generate lensing maps using %d objects (%d x %d grid)\n", Npsf, Nx, Ny); + + // XXX allow user to set this optionally? + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); + + // measure Trend2D for the current spatial scale + pmTrend2D *trendX11sm = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trendX12sm = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trendX22sm = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trend_e1sm = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trend_e2sm = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + + pmTrend2D *trendX11sh = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trendX12sh = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trendX22sh = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trend_e1sh = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trend_e2sh = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + + pmTrend2D *trend_e1 = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + pmTrend2D *trend_e2 = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats); + + bool goodFit = false; + if (!pmTrend2DFit (&goodFit, trendX11sm, NULL, 0xff, xPos, yPos, psfX11sm, NULL)) { psWarning ("failed to measure X11 smear trend"); } + if (!pmTrend2DFit (&goodFit, trendX12sm, NULL, 0xff, xPos, yPos, psfX12sm, NULL)) { psWarning ("failed to measure X12 smear trend"); } + if (!pmTrend2DFit (&goodFit, trendX22sm, NULL, 0xff, xPos, yPos, psfX22sm, NULL)) { psWarning ("failed to measure X22 smear trend"); } + if (!pmTrend2DFit (&goodFit, trend_e1sm, NULL, 0xff, xPos, yPos, psf_e1sm, NULL)) { psWarning ("failed to measure e1 smear trend"); } + if (!pmTrend2DFit (&goodFit, trend_e2sm, NULL, 0xff, xPos, yPos, psf_e2sm, NULL)) { psWarning ("failed to measure e2 smear trend"); } + + if (!pmTrend2DFit (&goodFit, trendX11sh, NULL, 0xff, xPos, yPos, psfX11sh, NULL)) { psWarning ("failed to measure X11 shear trend"); } + if (!pmTrend2DFit (&goodFit, trendX12sh, NULL, 0xff, xPos, yPos, psfX12sh, NULL)) { psWarning ("failed to measure X12 shear trend"); } + if (!pmTrend2DFit (&goodFit, trendX22sh, NULL, 0xff, xPos, yPos, psfX22sh, NULL)) { psWarning ("failed to measure X22 shear trend"); } + if (!pmTrend2DFit (&goodFit, trend_e1sh, NULL, 0xff, xPos, yPos, psf_e1sh, NULL)) { psWarning ("failed to measure e1 shear trend"); } + if (!pmTrend2DFit (&goodFit, trend_e2sh, NULL, 0xff, xPos, yPos, psf_e2sh, NULL)) { psWarning ("failed to measure e2 shear trend"); } + + if (!pmTrend2DFit (&goodFit, trend_e1, NULL, 0xff, xPos, yPos, psf_e1, NULL)) { psWarning ("failed to measure e1 trend"); } + if (!pmTrend2DFit (&goodFit, trend_e2, NULL, 0xff, xPos, yPos, psf_e2, NULL)) { psWarning ("failed to measure e2 trend"); } + + // evaluate the PSF trend maps at the location of all sources + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + pmPeak *peak = source->peak; + if (!peak) continue; + + float xPos = peak->xf; + float yPos = peak->yf; + + if (!source->lensingPSF) { + source->lensingPSF = pmSourceLensingAlloc (); + } + pmSourceLensing *lensing = source->lensingPSF; + + if (!lensing->shear) { + lensing->shear = pmLensingParsAlloc(); + } + pmLensingPars *shear = lensing->shear; + + shear->X11 = pmTrend2DEval (trendX11sh, xPos, yPos); + shear->X12 = pmTrend2DEval (trendX12sh, xPos, yPos); + shear->X22 = pmTrend2DEval (trendX22sh, xPos, yPos); + shear->e1 = pmTrend2DEval (trend_e1sh, xPos, yPos); + shear->e2 = pmTrend2DEval (trend_e2sh, xPos, yPos); + + if (!lensing->smear) { + lensing->smear = pmLensingParsAlloc(); + } + pmLensingPars *smear = lensing->smear; + + smear->X11 = pmTrend2DEval (trendX11sm, xPos, yPos); + smear->X12 = pmTrend2DEval (trendX12sm, xPos, yPos); + smear->X22 = pmTrend2DEval (trendX22sm, xPos, yPos); + smear->e1 = pmTrend2DEval (trend_e1sm, xPos, yPos); + smear->e2 = pmTrend2DEval (trend_e2sm, xPos, yPos); + + lensing->e1 = pmTrend2DEval (trend_e1, xPos, yPos); + lensing->e2 = pmTrend2DEval (trend_e2, xPos, yPos); + } + + psFree (trendX11sm); + psFree (trendX12sm); + psFree (trendX22sm); + psFree (trend_e1sm); + psFree (trend_e2sm); + + psFree (trendX11sh); + psFree (trendX12sh); + psFree (trendX22sh); + psFree (trend_e1sh); + psFree (trend_e2sh); + + psFree (trend_e1); + psFree (trend_e2); + + psFree (psfX11sm); + psFree (psfX12sm); + psFree (psfX22sm); + psFree (psf_e1sm); + psFree (psf_e2sm); + + psFree (psfX11sh); + psFree (psfX12sh); + psFree (psfX22sh); + psFree (psf_e1sh); + psFree (psf_e2sh); + + psFree (psf_e1); + psFree (psf_e2); + + psFree (xPos ); + psFree (yPos ); + + psFree (stats); + + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "LENS_PSF", PS_META_REPLACE, "psf-trend lensing stats measured", true); + psLogMsg ("psphot.lensing", PS_LOG_DETAIL, "calculate lensing parameters for %d objects: %f sec\n", (int) sources->n, psTimerMark ("psphot.lensing")); + + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLoadPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLoadPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLoadPSF.c (revision 41083) @@ -0,0 +1,76 @@ +# include "psphotInternal.h" + +// PSPHOT.PSF.LOAD vs input file -- see note at top +bool psphotLoadPSF (pmConfig *config, const pmFPAview *view, const char *filerule) { + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // Generate the mask and weight images, including the user-defined analysis region of interest + if (!psphotLoadPSFReadout (config, view, filerule, "PSPHOT.PSF.LOAD", i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to load PSF model for PSPHOT.PSF.LOAD entry %d", i); + return false; + } + } + return true; +} + +// NOTE : pmPSF_IO.c functions must load the psf model onto the chip->analysis metadata because +// the I/O operation likely occurs before the readout exists. This implementation assumes that +// a single psf model is valid for the entire set of readouts (not valid for a time series of readouts) + +// XXX for now (2010.01.27), the supporting programs do not define multiple PSPHOT.PSF.LOAD +// files to go with multiple input files. as a result, the implementation below is +// currently going to work for the case of a single input file, but will fail if we try with a +// stack of images. + +// load an externally supplied psf model +bool psphotLoadPSFReadout (pmConfig *config, const pmFPAview *view, const char *outFilename, const char *inFilename, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *inFile = pmFPAfileSelectSingle(config->files, inFilename, index); // File of interest + if (inFile == NULL) { + psLogMsg ("psphot", 3, "no psf supplied for this chip"); + return true; + } + + // find the currently selected readout + pmFPAfile *outFile = pmFPAfileSelectSingle(config->files, outFilename, index); // File of interest + if (outFile == NULL) { + psLogMsg ("psphot", 3, "no psf supplied for this chip"); + return true; + } + + // find the currently selected chip + pmChip *chip = pmFPAviewThisChip (view, inFile->fpa); + if (!chip) return false; + + // find the currently selected readout + pmReadout *readout = pmFPAviewThisReadout (view, outFile->fpa); + if (!readout) return false; + + // check if a PSF model is supplied by the user + pmPSF *psf = psMetadataLookupPtr (&status, chip->analysis, "PSPHOT.PSF"); + if (psf == NULL) { + psLogMsg ("psphot", 3, "no psf supplied for this chip"); + return true; + } + + if (!psphotPSFstats (readout, psf)) { + psAbort("cannot measure PSF shape terms"); + } + + psLogMsg ("psphot", 3, "using externally supplied PSF model for this readout"); + + // save PSF on readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot psf model", psf)) { + psError (PSPHOT_ERR_UNKNOWN, false, "problem saving sources on readout"); + return false; + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLoadSRCTEXT.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLoadSRCTEXT.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotLoadSRCTEXT.c (revision 41083) @@ -0,0 +1,137 @@ +# include "psphotInternal.h" + +bool psphotLoadSRCTEXT (pmFPA *fpa, pmConfig *config) { + + pmChip *chip; + + // we have a psArray of filenames, but there is no information to tie the filenames to the + // corresponding chip/cell/readout. Fortunately: (a) psphot only really operates on chips + // and (b) we can make some basic assumptions: + + // 1) if there is only a single entry in the array, read the file and attach the results to + // each of the chip->analysis metadata entries (just an extra ref counter). + + // 2) if there are multiple entries, the number of filenames must match the number of + // chips. + + psArray *files = psMetadataLookupPtr(NULL, config->arguments, "SRCTEXT"); + + // XXX is this allowed? how do we get an empty entry in arguments? + if (!files->n) return true; + + // we use a default model type of GAUSS to define the sources + pmModelType modelType = pmModelClassGetType("PS_MODEL_GAUSS"); + + psArray *sourceArrays = psArrayAllocEmpty(files->n); + for (int i = 0; i < files->n; i++) { + + char *filename = files->data[i]; + + // each file contains a list of X Y coordinates and nothing else? + FILE *f = fopen (filename, "r"); + if (f == NULL) { + psError(PSPHOT_ERR_CONFIG, true, "Failed to read source text list %s", filename); + return false; + } + + double X, Y; + + psArray *sources = psArrayAllocEmpty (100); + while (fscanf (f, "%lf %lf", &X, &Y) != EOF) { + + psEllipseAxes axes; + + pmSource *source = pmSourceAlloc (); + pmModel *model = pmModelAlloc (modelType); + source->modelPSF = model; + source->type = PM_SOURCE_TYPE_STAR; + + // NOTE: A SEGV here because "model" is NULL is probably caused by not initialising the models. + psF32 *PAR = model->params->data.F32; + psF32 *dPAR = model->dparams->data.F32; + + // NOTE: most of these values are irrelevant for loaded source positions + source->seq = source->id; + PAR[PM_PAR_XPOS] = X; + PAR[PM_PAR_YPOS] = Y; + dPAR[PM_PAR_XPOS] = 0.0; + dPAR[PM_PAR_YPOS] = 0.0; + axes.major = 1.0; + axes.minor = 1.0; + axes.theta = 0.0; + + PAR[PM_PAR_SKY] = 0.0; + dPAR[PM_PAR_SKY] = 0.0; + source->sky = PAR[PM_PAR_SKY]; + source->skyErr = dPAR[PM_PAR_SKY]; + + source->psfMag = 0.0; + source->psfMagErr = 0.0; + source->apMag = 0.0; + + PAR[PM_PAR_I0] = 1.0; + dPAR[PM_PAR_I0] = 0.0; + + pmPSF_AxesToModel (PAR, axes, model->class->useReff); + + float peakFlux = 1.0; + + source->peak = pmPeakAlloc(PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], peakFlux, PM_PEAK_LONE); + source->peak->dx = dPAR[PM_PAR_XPOS]; + source->peak->dy = dPAR[PM_PAR_YPOS]; + source->peak->xf = PAR[PM_PAR_XPOS]; // the alloc function takes the pixel index, + source->peak->yf = PAR[PM_PAR_YPOS]; // but we know the pixel coordinate + + source->pixWeightNotBad = 1.0; + source->pixWeightNotPoor = 1.0; + source->crNsigma = 0.0; + source->extNsigma = 0.0; + source->apRadius = 0.0; + + model->chisq = 0.0; + model->nDOF = 0; + model->nPix = 0; + + source->moments = pmMomentsAlloc (); + source->moments->Mxx = 0.0; + source->moments->Mxy = 0.0; + source->moments->Myy = 0.0; + + //source->mode = 0; + // MEH -- has to be PSFSTAR to accommodate changed made in psphot:psphotReadoutCleanupReadout and psphotPSFstats to psphotPSFstatsSources + source->mode = 64; + + psArrayAdd (sources, 100, source); + psFree(source); + } + psArrayAdd (sourceArrays, 100, sources); + psFree (sources); + } + + if (files->n == 1) { + psArray *sources = sourceArrays->data[0]; + pmFPAview *view = pmFPAviewAlloc (0); + while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) { + if (!chip->process) { continue; } + psMetadataAddPtr (chip->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES.TEXT", PS_DATA_ARRAY | PS_META_REPLACE, "loaded source from textfile", sources); + } + psFree (view); + } else { + int nSrc = 0; + pmFPAview *view = pmFPAviewAlloc (0); + while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) { + if (!chip->process) { continue; } + + if (nSrc >= sourceArrays->n) { + psError(PSPHOT_ERR_CONFIG, true, "mismatch between number of source lists and number of chips (readouts)"); + return false; + } + psArray *sources = sourceArrays->data[nSrc]; + psMetadataAddPtr (chip->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES.TEXT", PS_DATA_ARRAY | PS_META_REPLACE, "loaded source from textfile", sources); + nSrc ++; + } + psFree (view); + } + psFree (sourceArrays); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMagnitudes.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMagnitudes.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMagnitudes.c (revision 41083) @@ -0,0 +1,369 @@ +# include "psphotInternal.h" + +bool psphotMagnitudes (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Magnitudes ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + if (!psphotMagnitudesReadout (config, recipe, view, readout, sources, psf, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotMagnitudesReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources, pmPSF *psf, int index) { + + bool status = false; + int Nap = 0; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source magnitudes"); + return true; + } + + psTimerStart ("psphot.mags"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // if backModel or backStdev are missing, the values of sky and/or skyErr will be set to NAN + pmReadout *backModel = psphotSelectBackground (config, view, index); + pmReadout *backStdev = psphotSelectBackgroundStdev (config, view, index); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + pmSourceMagnitudesInit (config, recipe); + + // the binning details are saved on the analysis metadata + psImageBinning *binning = NULL; + if (backModel) { + binning = psMetadataLookupPtr(&status, backModel->analysis, "PSPHOT.BACKGROUND.BINNING"); + } + + bool IGNORE_GROWTH = psMetadataLookupBool (&status, recipe, "IGNORE_GROWTH"); + bool INTERPOLATE_AP = psMetadataLookupBool (&status, recipe, "INTERPOLATE_AP"); + bool DIFF_STATS = psMetadataLookupBool (&status, recipe, "DIFF_STATS"); + + pmSourcePhotometryMode photMode = PM_SOURCE_PHOT_APCORR | PM_SOURCE_PHOT_WEIGHT; + if (!IGNORE_GROWTH) photMode |= PM_SOURCE_PHOT_GROWTH; + if (INTERPOLATE_AP) photMode |= PM_SOURCE_PHOT_INTERP; + if (DIFF_STATS) photMode |= PM_SOURCE_PHOT_DIFFSTATS; + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_MAGNITUDES"); + + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, psf); + psArrayAdd(job->args, 1, binning); + psArrayAdd(job->args, 1, backModel); + psArrayAdd(job->args, 1, backStdev); + + PS_ARRAY_ADD_SCALAR(job->args, photMode, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for nAp + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = job->args->data[8]; + Nap += scalar->data.S32; + } + psFree(job); + } + } + + psFree (cellGroups); + + psLogMsg ("psphot.magnitudes", PS_LOG_WARN, "measure magnitudes : %f sec for %ld objects (%d with apertures)\n", psTimerMark ("psphot.mags"), sources->n, Nap); + return true; +} + +bool psphotMagnitudes_Threaded (psThreadJob *job) { + + bool status; + int Nap = 0; + + psArray *sources = job->args->data[0]; + pmPSF *psf = job->args->data[1]; + psImageBinning *binning = job->args->data[2]; + pmReadout *backModel = job->args->data[3]; + pmReadout *backStdev = job->args->data[4]; + pmSourcePhotometryMode photMode = PS_SCALAR_VALUE(job->args->data[5],S32); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[6],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[7],PS_TYPE_IMAGE_MASK_DATA); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = (pmSource *) sources->data[i]; + + bool saveTest = false; + psImage *testImage = NULL; +# if (0) + if ((fabs(source->peak->xf-3518) < 5) && (fabs(source->peak->yf-3178) < 5)) { + saveTest = true; + psRegion subregion = psRegionSet (source->peak->xf - 200, source->peak->xf + 200, source->peak->yf - 200, source->peak->yf + 200); + testImage = psImageSubset ((psImage *) source->pixels->parent, subregion); + } +# endif + + if (saveTest) { + psphotSaveImage(NULL, testImage, "test.image.1.fits"); + } + + // satstars modeled as a radial profile need special handling + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) { + psphotSatstarPhotometry (source); + continue; + } + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + if (saveTest) { + psphotSaveImage(NULL, testImage, "test.image.2.fits"); + } + + float xPos = source->peak->xf; + float yPos = source->peak->yf; + + pmModel *model = source->modelPSF; + if (model) { + xPos = model->params->data.F32[PM_PAR_XPOS]; + yPos = model->params->data.F32[PM_PAR_YPOS]; + } else { + bool useMoments = pmSourcePositionUseMoments(source); + if (useMoments) { + xPos = source->moments->Mx; + yPos = source->moments->My; + } + } + + // clear the mask bit and set the circular mask pixels + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + psImageKeepCircle (source->maskObj, xPos, yPos, source->apRadius, "OR", markVal); + + status = pmSourceMagnitudes (source, psf, photMode, maskVal, markVal, source->apRadius); + if (status && isfinite(source->apFlux)) { + Nap ++; + } + + // clear the mask bit + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + if (saveTest) { + psphotSaveImage(NULL, testImage, "test.image.3.fits"); + } + + if (backModel) { + psAssert (binning, "if backModel is defined, so should binning be"); + source->sky = psImageUnbinPixel(xPos, yPos, backModel->image, binning); + if (isnan(source->sky) && false) { + psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.sky"); + } + } else { + source->sky = NAN; + } + + if (backStdev) { + psAssert (binning, "if backStdev is defined, so should binning be"); + source->skyErr = psImageUnbinPixel(xPos, yPos, backStdev->image, binning); + if (isnan(source->skyErr) && false) { + psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.skyErr"); + } + } else { + source->skyErr = NAN; + } + } + + // change the value of a scalar on the array (wrap this and put it in psArray.h) + psScalar *scalar = job->args->data[8]; + scalar->data.S32 = Nap; + + return true; +} + +bool psphotPSFWeights(pmConfig *config, pmReadout *readout, const pmFPAview *view, psArray *sources) { + + bool status = false; + + psTimerStart ("psphot.mags"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + nThreads = 0; // XXX until testing is complete, do not thread this function + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_PSF_WEIGHTS"); + + psArrayAdd(job->args, 1, cells->data[j]); // sources + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + + psFree (cellGroups); + + psLogMsg ("psphot", PS_LOG_DETAIL, "measure psf weights : %f sec for %ld objects\n", psTimerMark ("psphot.mags"), sources->n); + return true; +} + +bool psphotPSFWeights_Threaded (psThreadJob *job) { + + bool status; + bool isPSF; + + psArray *sources = job->args->data[0]; + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[1],PS_TYPE_IMAGE_MASK_DATA); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = (pmSource *) sources->data[i]; + + // we must have a valid model + pmModel *model = pmSourceGetModel (&isPSF, source); + if (model == NULL) { + psTrace ("psphot", 3, "fail mag : no valid model"); + source->pixWeightNotBad = NAN; + source->pixWeightNotPoor = NAN; + continue; + } + + status = pmSourcePixelWeight (source, model, source->maskObj, maskVal, source->apRadius); + if (!status) { + psTrace ("psphot", 3, "fail to measure pixel weight"); + source->pixWeightNotBad = NAN; + source->pixWeightNotPoor = NAN; + continue; + } + + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeFluxScale.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeFluxScale.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeFluxScale.c (revision 41083) @@ -0,0 +1,89 @@ +# include "psphotInternal.h" + +bool psphotMakeFluxScale (psImage *image, psMetadata *recipe, pmPSF *psf) { + + bool status = false; + + psTimerStart ("psphot.fluxscale"); + + int Nx = psMetadataLookupS32(&status, recipe, "PSF.FLUXSCALE.NX"); + int Ny = psMetadataLookupS32(&status, recipe, "PSF.FLUXSCALE.NY"); + + // stats structure for use by ApTrend : XXX make parameters user setable + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); + + pmTrend2D *trend = pmTrend2DAlloc (PM_TREND_MAP, image, Nx, Ny, stats); + + psVector *xPts = psVectorAlloc (Nx*Ny, PS_TYPE_F32); + psVector *yPts = psVectorAlloc (Nx*Ny, PS_TYPE_F32); + psVector *fPts = psVectorAlloc (Nx*Ny, PS_TYPE_F32); + + int Npts = 0; + bool success = true; // Function succeeded? + + // generate a set of test normalized PSF fluxes filling the grid + for (int ix = 0; ix < Nx; ix++) { + for (int iy = 0; iy < Ny; iy++) { + + float x = psImageBinningGetFineX (trend->map->binning, ix + 0.5); + float y = psImageBinningGetFineY (trend->map->binning, iy + 0.5); + + float fitSum = 0; + + // create normalized model object at xc,yc + pmModel *model = pmModelFromPSFforXY (psf, x, y, 1.0); + if (!model) { + psTrace ("psphot", 3, "Unable to generate model for grid point %d,%d", ix, iy); + fitSum = NAN; + } else { + // measure the fitMag for this model + fitSum = model->class->modelFlux (model->params); + } + if (fitSum < 1.e-6) continue; + if (!isfinite(fitSum)) continue; + + xPts->data.F32[Npts] = x; + yPts->data.F32[Npts] = y; + fPts->data.F32[Npts] = fitSum; + Npts ++; + assert (Npts <= xPts->nalloc); + psFree (model); + } + } + xPts->n = Npts; + yPts->n = Npts; + fPts->n = Npts; + + // XXX we should allow the spatial sampling to decrease if the fit fails + bool goodFit = false; + if (!pmTrend2DFit (&goodFit, trend, NULL, 0xff, xPts, yPts, fPts, NULL)) { + psError(PS_ERR_UNKNOWN, false, "Unable to fit trend"); + success = false; + goto DONE; + } + if (!goodFit) { + psError(PS_ERR_UNKNOWN, false, "poor fit to flux-scale trend"); + success = false; + goto DONE; + } + if (trend->mode == PM_TREND_MAP) { + // p_psImagePrint (2, trend->map->map, "FluxScale Before"); // XXX TEST: + psImageMapRepair (trend->map->map); + // p_psImagePrint (2, trend->map->map, "FluxScale After"); // XXX TEST: + } + + // XXX do something useful to measure residual statistics + + psf->FluxScale = psMemIncrRefCounter(trend); + + DONE: + psFree(xPts); + psFree(yPts); + psFree(fPts); + psFree(stats); + psFree(trend); + + psLogMsg ("psphot", PS_LOG_MINUTIA, "built flux scale: %f sec\n", psTimerMark ("psphot.fluxscale")); + + return success; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeGrowthCurve.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeGrowthCurve.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeGrowthCurve.c (revision 41083) @@ -0,0 +1,62 @@ +# include "psphotInternal.h" + +bool psphotMakeGrowthCurve (pmReadout *readout, psMetadata *recipe, pmPSF *psf, psArray *sources) { + + bool status; + + psTimerStart ("psphot.growth"); + + // set limits on the aperture magnitudes + pmSourceMagnitudesInit (NULL, recipe); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // measure the aperture loss as a function of radius for PSF + float REF_RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_REF_RADIUS"); + float PSF_FIT_PAD = psMetadataLookupF32 (&status, recipe, "PSF_FIT_PADDING"); + + float gaussSigma = psMetadataLookupF32(&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + gaussSigma = psMetadataLookupF32(&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float apScale = psMetadataLookupF32(&status, recipe, "PSF_APERTURE_SCALE"); + float PSF_APERTURE = (int)(apScale*gaussSigma); + + psf->growth = pmGrowthCurveAlloc (PSF_FIT_PAD, 100.0, REF_RADIUS); + + bool GROWTH_FROM_SOURCES = psMetadataLookupBool (&status, recipe, "GROWTH_FROM_SOURCES"); + if (!status) GROWTH_FROM_SOURCES = false; + + if (GROWTH_FROM_SOURCES) { + bool INTERPOLATE_AP = psMetadataLookupBool (&status, recipe, "INTERPOLATE_AP"); + if (!pmGrowthCurveGenerateFromSources (readout, psf, sources, INTERPOLATE_AP, maskVal, markVal)) { + psError(PSPHOT_ERR_APERTURE, false, "Fitting aperture corrections"); + psFree(psf->growth); psf->growth = NULL; + return false; + } + + } else { + bool IGNORE_GROWTH = psMetadataLookupBool (&status, recipe, "IGNORE_GROWTH"); + if (!pmGrowthCurveGenerate (readout, psf, IGNORE_GROWTH, maskVal, markVal)) { + psError(PSPHOT_ERR_APERTURE, false, "Fitting aperture corrections"); + psFree(psf->growth); psf->growth = NULL; + return false; + } + } + + psLogMsg ("psphot", PS_LOG_MINUTIA, "built growth curve: %f sec\n", psTimerMark ("psphot.growth")); + + float offset = pmGrowthCurveCorrect (psf->growth, PSF_APERTURE); + psLogMsg ("psphot", PS_LOG_DETAIL, "correction from %f to %f: %f mags\n", PSF_APERTURE, REF_RADIUS, offset); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSF.c (revision 41083) @@ -0,0 +1,35 @@ +# include "psphotStandAlone.h" + +int main (int argc, char **argv) { + + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotMakePSFArguments (argc, argv); + assert(config); + + psphotVersionPrint(); + + // load input data (config and images (signal, noise, mask) + if (!psphotParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotImageLoop (config, PSPHOT_MAKE_PSF)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", 3, "complete psphot run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFArguments.c (revision 41083) @@ -0,0 +1,189 @@ +# include "psphotStandAlone.h" + +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile) +{ + fprintf(ofile, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "%s -help\n" + "\n" + "%s -version\n" + "\n" + "where:\n" + " FileNameList is a text file containing filenames, one per line\n" + " MaskFileNameList is a text file of mask filenames, one per line\n" + " VarFileNameList is a text file of variance filenames, one per line\n" + " OutFileBaseName is the 'root name' for output files\n" + "\n" + "additional options:\n" + " -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n" + " specify PSF rather than letting %s estimate it\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify additional sources for PSF generation\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -photcode PhotoCodeName\n" + " specify photocode\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -nthreads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n", + program,program,program,program,program); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "Try '%s -help' for more options and explanation\n", + program,program,program); + if (exitCode != PS_EXIT_SUCCESS) + psErrorStackPrint(stderr, "Error reading arguments\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + +pmConfig *psphotMakePSFArguments(int argc, char **argv) { + + int N; + bool status; + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + // Number of threads is handled + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + // visual : interactive display mode + if ((N = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (N, &argc, argv); + pmVisualSetVisual(true); + } + + // break : used from recipe throughout psphotReadout + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF + if ((N = psArgumentGet (argc, argv, "-region"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + // + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); + + // an input list of sources to use is allowed, but not required + pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC", "-src", "-srclist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "SRCTEXT", "-srctext", "-srctextlist"); + + if (argc == 1) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (psArgumentGet(argc, argv, "-help") || + psArgumentGet(argc, argv, "-h")) + writeHelpInfo(argv[0], config, stdout); + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotMakePSFArguments...\n"); + return (config); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFImageLoop.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFImageLoop.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFImageLoop.c (revision 41083) @@ -0,0 +1,129 @@ +# include "psphotStandAlone.h" + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ +} + +bool psphotMakePSFImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + pmFPAfile *load = psMetadataLookupPtr (&status, config->files, "PSPHOT.LOAD"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.INPUT"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + pmFPAview *view = pmFPAviewAlloc (0); + pmHDU *lastHDU = NULL; // Last HDU updated + + // files associated with the science image + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, load->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + + // load just the input image data (image, mask, weight) + pmFPAfileActivate (config->files, false, NULL); + pmFPAfileActivate (config->files, true, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, true, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, true, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // mosaic the cells of a chip into a single contiguous (trimmed) chip + if (!psphotMosaicChip(config, view, "PSPHOT.INPUT", "PSPHOT.LOAD")) ESCAPE ("Unable to mosaic chip."); + + // try to load other supporting data (PSF, SRC, etc). + // do not re-load the following three files + pmFPAfileActivate (config->files, true, NULL); + pmFPAfileActivate (config->files, false, "PSPHOT.LOAD"); + pmFPAfileActivate (config->files, false, "PSPHOT.MASK"); + pmFPAfileActivate (config->files, false, "PSPHOT.VARIANCE"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphot."); + + // re-activate files so they will be closed and freed below + pmFPAfileActivate (config->files, true, NULL); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + // Update the header + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + psphotVersionHeaderFull(hdu->header); + lastHDU = hdu; + } + + // if an external mask is supplied, ensure that NAN pixels are also masked + if (readout->mask) { + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); // Mask value for saturated pixels + if (!pmReadoutMaskNonfinite(readout, maskSat)) { + psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels."); + psFree(view); + return false; + } + } + + // run the actual photometry analysis on this chip/cell/readout + if (!psphotMakePSFReadout (config, view, "PSPHOT.INPUT")) { + psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + } + + status = true; + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV"); + status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND"); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files"); + psFree (view); + return false; + } + } + + // save output which is saved at the chip level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed output for Chip in psphot."); + } + // save output which is saved at the fpa level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed ouput for FPA in psphot."); + + // fail if we failed to handle an error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + return true; +} + +// I/O files related to psphot: +// PSPHOT.INPUT : input image file(s) +// PSPHOT.RESID : residual image +// PSPHOT.OUTPUT : output object tables (object) + +// PSPHOT.BACKSUB : background subtracted image +// PSPHOT.BACKGND : background model (full-scale image?) +// PSPHOT.BACKMDL : background model (binned image?) +// PSPHOT.PSF : sample PSF images + +// PSPHOT.MASK +// PSPHOT.VARIANCE +// Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakePSFReadout.c (revision 41083) @@ -0,0 +1,76 @@ +# include "psphotInternal.h" + +bool psphotMakePSFReadout(pmConfig *config, const pmFPAview *view, const char *filerule) { + + // measure the total elapsed time in psphotReadout. XXX the current threading plan + // for psphot envisions threading within psphotReadout, not multiple threads calling + // the same psphotReadout. In the current plan, this dtime is the elapsed time used + // jointly by the multiple threads, not the total time used by all threads. + psTimerStart ("psphotReadout"); + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + // set the photcode for this image + if (!psphotAddPhotcode (config, view, filerule)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // optional break-point for processing + char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); + PS_ASSERT_PTR_NON_NULL (breakPt, false); + + // Generate the mask and weight images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + if (!strcasecmp (breakPt, "NOTHING")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!psphotSubtractBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "BACKMDL")) { + return psphotReadoutCleanup (config, view, filerule); + } + + psphotLoadExtSources (config, view, filerule); + + // If sources have been supplied, then these should be used to measure the PSF include + // externally-supplied sources; if not, we need to generate a set of possible PSF sources. + // This function updates the SN entries for the loaded sources or generates a set of + // detections from the image, if no external ones have been supplied. Sources loaded from + // a text file have no valid SN values, but psphotChoosePSF needs to select the top + // PSF_MAX_NSTARS to generate the PSF. + if (!psphotCheckExtSources (config, view, filerule)) { + psLogMsg ("psphot", 3, "failure to select possible PSF sources (external or internal)"); + return psphotReadoutCleanup (config, view, filerule); + } + + // Use bright stellar objects to measure PSF. If we do not have enough stars to generate + // the PSF, build one from the SEEING guess and model class + if (!psphotChoosePSF (config, view, filerule, true)) { + psLogMsg ("psphot", 3, "failure to construct a psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // measure aperture photometry corrections +# if 0 + if (!psphotApResid (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed on psphotApResid"); + return psphotReadoutCleanup (config, view, filerule); + } +# endif + + return psphotReadoutCleanup (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeResiduals.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeResiduals.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMakeResiduals.c (revision 41083) @@ -0,0 +1,344 @@ +# include "psphotInternal.h" + +# define RESIDUAL_SOFTENING 0.005 + +bool psphotMakeResiduals (psArray *sources, psMetadata *recipe, pmPSF *psf, psImageMaskType maskVal) { + + bool status, isPSF; + double flux, dflux; + psImageMaskType mflux; + + psTimerStart ("psphot.residuals"); + + if (!psMetadataLookupBool(&status, recipe, "PSF.RESIDUALS")) return true; + + int SPATIAL_ORDER = psMetadataLookupS32(&status, recipe, "PSF.RESIDUALS.SPATIAL_ORDER"); + PS_ASSERT (status, false); + if (SPATIAL_ORDER != 0 && SPATIAL_ORDER != 1) { + psError(PSPHOT_ERR_CONFIG, true, "PSF.RESIDUALS.SPATIAL_ORDER must be 0 or 1 (not %d)", + SPATIAL_ORDER); + return false; + } + + int xBin = psMetadataLookupS32(&status, recipe, "PSF.RESIDUALS.XBIN"); + PS_ASSERT (status, false); + PS_ASSERT (xBin != 0, false); + + int yBin = psMetadataLookupS32(&status, recipe, "PSF.RESIDUALS.YBIN"); + PS_ASSERT (status, false); + PS_ASSERT (yBin != 0, false); + + float nSigma = psMetadataLookupF32(&status, recipe, "PSF.RESIDUALS.NSIGMA"); + PS_ASSERT (status, false); + + float pixelSN = psMetadataLookupF32(&status, recipe, "PSF.RESIDUALS.PIX.SN"); + PS_ASSERT (status, false); + + float radiusMax = psMetadataLookupF32(&status, recipe, "PSF.RESIDUALS.RADIUS"); + PS_ASSERT (status, false); + + char *modeString = psMetadataLookupStr(&status, recipe, "PSF.RESIDUALS.INTERPOLATION"); + PS_ASSERT (status, false); + + psImageInterpolateMode mode = psImageInterpolateModeFromString (modeString); + if (mode == PS_INTERPOLATE_NONE) { + psError(PSPHOT_ERR_CONFIG, false, "invalid interpolation in psphot.config"); + return false; + } + + char *statString = psMetadataLookupStr(&status, recipe, "PSF.RESIDUALS.STATISTIC"); + PS_ASSERT (status, false); + + psStatsOptions statOption = psStatsOptionFromString (statString); + if (!statOption) { + psError(PSPHOT_ERR_CONFIG, false, "invalid residual statistic in psphot.config"); + return false; + } + + // user parameters: + // size of aperture (determine from source images?) + // binning factor + + // select the subset of sources which are the PSFSTARs + // for each input source: + // - construct a residual image, renormalized + // - construct a renormalized variance image + // - construct a new mask image + + // construct the output residual table (Nx*DX,Ny*DY) + // for each output pixel: + // - construct a histogram of the values & variances (interpolate to the common pixel coordinate) + // - measure the robust median & sigma + // - reject (mask) input pixels which are outliers + // - re-measure the robust median & sigma + // - set output pixel, variance, and mask + + // these mask values do not correspond to the recipe values: they + // are not propagated to images: they just need to fit in an 8-bit + // value. they are supplied to psImageInterpolate, which takes a + // psImageMaskType; the mask portion of the result from + // psImageInterpolate is supplied to fmasks, which is then used by + // psVectorStats + + const psImageMaskType badMask = 0x01; // mask bits + const psImageMaskType poorMask = 0x02; // from psImageInterpolate + const psImageMaskType clippedMask = 0x04; // mask bit set for clipped values + const psVectorMaskType fmaskVal = badMask | poorMask | clippedMask; + + // determine the maximum image size from the input sources + int xSize = 0; + int ySize = 0; + + psVector *xC = psVectorAllocEmpty (100, PS_TYPE_F32); + psVector *yC = psVectorAllocEmpty (100, PS_TYPE_F32); + + // build (DATA - MODEL) [an image] for each psf star + psArray *input = psArrayAllocEmpty (100); + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + + // which model to use? + pmModel *model = pmSourceGetModel (&isPSF, source); + if (model == NULL) continue; // model must be defined + + psImage *image = psImageCopy (NULL, source->pixels, PS_TYPE_F32); + psImage *mask = psImageCopy (NULL, source->maskView ? source->maskView : source->maskObj, PS_TYPE_IMAGE_MASK); + psImage *variance = psImageCopy (NULL, source->variance ? source->variance : source->pixels, PS_TYPE_F32); + pmModelSub (image, mask, model, PM_MODEL_OP_FUNC, maskVal); + + // re-normalize image and variance + float Io = model->params->data.F32[PM_PAR_I0]; + psBinaryOp (image, image, "/", psScalarAlloc(Io, PS_TYPE_F32)); + psBinaryOp (variance, variance, "/", psScalarAlloc(Io*Io, PS_TYPE_F32)); + + // we interpolate the image and variance - include the mask or not? + // XXX why not the mask? + // psImageInterpolation *interp = psImageInterpolationAlloc(mode, image, variance, mask, maskVal, 0.0, 0.0, badMask, poorMask, 0.0, 0); + psImageInterpolation *interp = psImageInterpolationAlloc(mode, image, variance, NULL, 0xff, 0.0, 0.0, badMask, poorMask, 0.0, 0); + psArrayAdd (input, 100, interp); + + // save the X,Y position for future reference + xC->data.F32[xC->n] = model->params->data.F32[PM_PAR_XPOS]; + yC->data.F32[yC->n] = model->params->data.F32[PM_PAR_YPOS]; + psVectorExtend (xC, 100, 1); + psVectorExtend (yC, 100, 1); + + xSize = PS_MAX (xSize, image->numCols); + ySize = PS_MAX (ySize, image->numRows); + + // free up the excess references + psFree (mask); + psFree (image); + psFree (variance); + psFree (interp); + } + xSize = PS_MIN(xSize, 2*radiusMax+3); + ySize = PS_MIN(ySize, 2*radiusMax+3); + + pmResiduals *resid = pmResidualsAlloc (xSize, ySize, xBin, yBin); + psImageInit (resid->mask, 0); + + // x(resid) = (x(image) - Xo)*xBin + xCenter + + psVector *fluxes = psVectorAlloc (input->n, PS_TYPE_F32); + psVector *dfluxes = psVectorAlloc (input->n, PS_TYPE_F32); + psVector *fmasks = psVectorAlloc (input->n, PS_TYPE_VECTOR_MASK); + + // statistic to use to determine baseline for clipping + psStats *fluxClip = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psStats *fluxClipDef = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + // statistic to use to determine output flux + // XXX make API to convert statOption for MEAN/MEDIAN in to corresponding STDEV? + psStats *fluxStats = psStatsAlloc (statOption | PS_STAT_SAMPLE_STDEV); + psStats *fluxStatsDef = psStatsAlloc (statOption | PS_STAT_SAMPLE_STDEV); + + // Use psF64 to minimize overflow problems? + psImage *A = psImageAlloc(3, 3, PS_TYPE_F64); // Least-squares matrix + psVector *B = psVectorAlloc(3, PS_TYPE_F64); // Least-squares vector + + // Solve MODEL = R + x R_x + y R_y in pixel-by-pixel a least-squares sense + // (If SPATIAL_ORDER == 0, just solve MODEL = R) + for (int oy = 0; resid != NULL && oy < resid->Ro->numRows; oy++) { + for (int ox = 0; ox < resid->Ro->numCols; ox++) { + + int nGoodPixel = 0; // pixel is off the image + + // build the vector of data values for this output pixel + for (int i = 0; i < input->n; i++) { + + psImageInterpolation *interp = input->data[i]; + + // fractional image position + float ix = (ox + 0.5 - resid->xCenter) / (float) xBin + xC->data.F32[i] - interp->image->col0; + float iy = (oy + 0.5 - resid->yCenter) / (float) yBin + yC->data.F32[i] - interp->image->row0; + + mflux = 0; + if (psImageInterpolate (&flux, &dflux, &mflux, ix, iy, interp) == PS_INTERPOLATE_STATUS_OFF) { + // This pixel is off the image + fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i] = badMask; + } else { + fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i] = mflux; // XXX is mflux IMAGE or VECTOR type? + } + fluxes->data.F32[i] = flux; + dfluxes->data.F32[i] = hypot(dflux, RESIDUAL_SOFTENING); + if (isnan(flux)) { + fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i] = badMask; + } + if (isnan(dflux)) { + fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i] = badMask; + } + if (fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i] == 0) { + nGoodPixel ++; + } + } + + // skip pixels with insufficient data + bool validPixel = (SPATIAL_ORDER == 0) ? (nGoodPixel > 1) : (nGoodPixel > 3); + if (!validPixel) { + resid->Ro->data.F32[oy][ox] = 0.0; + resid->Rx->data.F32[oy][ox] = 0.0; + resid->Ry->data.F32[oy][ox] = 0.0; + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = badMask; + continue; + } + + // measure the robust median to determine a baseline reference value + *fluxClip = *fluxClipDef; + if (!psVectorStats (fluxClip, fluxes, NULL, fmasks, fmaskVal)) { + psError(PSPHOT_ERR_CONFIG, false, "Error calculating residual stats"); + return false; + } + if (isnan(fluxClip->robustMedian)) { + resid->Ro->data.F32[oy][ox] = 0.0; + resid->Rx->data.F32[oy][ox] = 0.0; + resid->Ry->data.F32[oy][ox] = 0.0; + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = badMask; + continue; + } + + // mark input pixels which are more than N sigma from the median + int nKeep = 0; + for (int i = 0; i < fluxes->n; i++) { + float delta = fluxes->data.F32[i] - fluxClip->robustMedian; + float sigma = sqrt (dfluxes->data.F32[i]); + float swing = fabs(delta) / sigma; + + // mask pixels which are out of range + if (swing > nSigma) { + fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i] = clippedMask; + } + if (!fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i]) nKeep++; + } + + if (SPATIAL_ORDER == 0) { + // measure the desired statistic on the unclipped pixels + *fluxStats = *fluxStatsDef; + if (!psVectorStats (fluxStats, fluxes, NULL, fmasks, fmaskVal)) { + psError(PSPHOT_ERR_CONFIG, false, "Error calculating residual stats"); + return false; + } + + float radius = hypot((ox - 0.5*resid->Ro->numCols), (oy - 0.5*resid->Ro->numRows)); + if (radius > radiusMax) { + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = 1; + continue; + } + + resid->Ro->data.F32[oy][ox] = psStatsGetValue(fluxStats, statOption); + resid->Rx->data.F32[oy][ox] = resid->Ry->data.F32[oy][ox] = 0.0; + + if (isnan(resid->Ro->data.F32[oy][ox])) { + resid->Ro->data.F32[oy][ox] = 0.0; + resid->Rx->data.F32[oy][ox] = 0.0; + resid->Ry->data.F32[oy][ox] = 0.0; + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = badMask; + continue; + } + + if (fabs(resid->Ro->data.F32[oy][ox]) < pixelSN*fluxStats->sampleStdev/sqrt(nKeep)) { + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = 1; + } + } else { + assert (SPATIAL_ORDER == 1); + + float radius = hypot((ox - 0.5*resid->Ro->numCols), (oy - 0.5*resid->Ro->numRows)); + if (radius > radiusMax) { + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = 1; + continue; + } + + psImageInit(A, 0.0); + psVectorInit(B, 0.0); + for (int i = 0; i < fluxes->n; i++) { + if (fmasks->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue; + B->data.F64[0] += fluxes->data.F32[i]/dfluxes->data.F32[i]; + B->data.F64[1] += fluxes->data.F32[i]*xC->data.F32[i]/dfluxes->data.F32[i]; + B->data.F64[2] += fluxes->data.F32[i]*yC->data.F32[i]/dfluxes->data.F32[i]; + + A->data.F64[0][0] += 1.0/dfluxes->data.F32[i]; + A->data.F64[1][0] += xC->data.F32[i]/dfluxes->data.F32[i]; + A->data.F64[2][0] += yC->data.F32[i]/dfluxes->data.F32[i]; + + A->data.F64[1][1] += PS_SQR(xC->data.F32[i])/dfluxes->data.F32[i]; + A->data.F64[2][2] += PS_SQR(yC->data.F32[i])/dfluxes->data.F32[i]; + A->data.F64[1][2] += xC->data.F32[i]*yC->data.F32[i]/dfluxes->data.F32[i]; + } + + A->data.F64[0][1] = A->data.F64[1][0]; + A->data.F64[0][2] = A->data.F64[2][0]; + A->data.F64[2][1] = A->data.F64[1][2]; + + if (!psMatrixGJSolve(A, B)) { + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = 1; + psWarning("Singular matrix solving for (y,x) = (%d,%d)'s residuals, masking", oy, ox); + continue; + } + + resid->Ro->data.F32[oy][ox] = B->data.F64[0]; + resid->Rx->data.F32[oy][ox] = B->data.F64[1]; + resid->Ry->data.F32[oy][ox] = B->data.F64[2]; + + float dRo = sqrt(A->data.F32[0][0]); + + if (fabs(resid->Ro->data.F32[oy][ox]) < pixelSN*dRo/sqrt(nKeep)) { + resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = 1; + resid->Ro->data.F32[oy][ox] = 0.0; + resid->Rx->data.F32[oy][ox] = 0.0; + resid->Ry->data.F32[oy][ox] = 0.0; + } + } + } + } + + psFree (A); + psFree (B); + + psLogMsg ("psphot.pspsf", PS_LOG_DETAIL, "generate residuals for %ld objects: %f sec\n", input->n, psTimerMark ("psphot.residuals")); + + psFree (xC); + psFree (yC); + psFree (input); + + psFree (fluxes); + psFree (dfluxes); + psFree (fmasks); + + psFree (fluxStats); + psFree (fluxStatsDef); + psFree (fluxClip); + psFree (fluxClipDef); + + if (resid != NULL && psTraceGetLevel("psphot") > 5) { + psphotSaveImage (NULL, resid->Ro, "resid.ro.fits"); + psphotSaveImage (NULL, resid->Rx, "resid.rx.fits"); + psphotSaveImage (NULL, resid->Ry, "resid.ry.fits"); + psphotSaveImage (NULL, resid->variance, "resid.wt.fits"); + psphotSaveImage (NULL, resid->mask, "resid.mk.fits"); + } + + psf->residuals = resid; + return (resid != NULL) ? true : false; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMaskBackground.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMaskBackground.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMaskBackground.c (revision 41083) @@ -0,0 +1,102 @@ +# include "psphotInternal.h" +# define NSIGMA 2.0 + +// mask pixel in the input image above model + N*stdev (using mark) +bool psphotMaskBackgroundReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + bool status = true; + + psTimerStart ("psphot.background"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout (view, file->fpa); + psAssert (readout, "missing readout?"); + + // find the currently selected readout (XXX note that the model is saved on PSPHOT.BACKMDL regardless of 'filename' + pmFPAfile *modelFile = pmFPAfileSelectSingle(config->files, psphotGetFilerule("PSPHOT.BACKMDL"), index); // File of interest + assert (modelFile); + + pmFPAfile *stdevFile = pmFPAfileSelectSingle(config->files, psphotGetFilerule("PSPHOT.BACKMDL.STDEV"), index); + assert (stdevFile); + + float skyMean = psMetadataLookupF32(&status, readout->analysis, "SKY_MEAN"); + float skyStdv = psMetadataLookupF32(&status, readout->analysis, "SKY_STDEV"); + + pmReadout *model = READOUT_OR_INTERNAL(view, modelFile); + psAssert (model, "this must exist"); + + pmReadout *stdev = READOUT_OR_INTERNAL(view, stdevFile); + psAssert (stdev, "this must exist"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + assert (markVal); + + psImageBinning *binning = psMetadataLookupPtr(&status, model->analysis, "PSPHOT.BACKGROUND.BINNING"); + assert (binning); + + // create a binned version of the threshold image (= model + NSIGMA*stdev) + psImage *threshBinned = psImageCopy (NULL, model->image, PS_TYPE_F32); + for (int iy = 0; iy < model->image->numRows; iy++) { + for (int ix = 0; ix < model->image->numCols; ix++) { + // threshBinned->data.F32[iy][ix] = model->image->data.F32[iy][ix] + NSIGMA*stdev->image->data.F32[iy][ix]; + threshBinned->data.F32[iy][ix] = skyMean + NSIGMA*skyStdv; + } + } + psphotSaveImage (NULL, threshBinned, "threshbin.fits"); + + // create a threshold image + psImage *threshold = psImageCopy (NULL, readout->image, PS_TYPE_F32); + + // linear interpolation to full-scale + if (!psImageUnbin (threshold, threshBinned, binning)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning"); + return false; + } + psphotSaveImage (NULL, threshold, "threshold.fits"); + + psLogMsg ("psphot", PS_LOG_MINUTIA, "build threshold image: %f sec\n", psTimerMark ("psphot.background")); + + // raise the 'markVal' mask bit for pixels above threshold + for (int iy = 0; iy < readout->image->numRows; iy++) { + for (int ix = 0; ix < readout->image->numCols; ix++) { + if (readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal) continue; + if (readout->image->data.F32[iy][ix] < threshold->data.F32[iy][ix]) continue; + readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= markVal; + } + } + psphotSaveImage (NULL, readout->mask, "newmask.fits"); + psLogMsg ("psphot", PS_LOG_INFO, "masked image based on 1st pass background model: %f sec\n", psTimerMark ("psphot.background")); + + psFree (threshold); + psFree (threshBinned); + + return true; +} + +bool psphotMaskBackground (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotMaskBackgroundReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to subtract background for %s entry %d", filerule, i); + return false; + } + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMaskReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMaskReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMaskReadout.c (revision 41083) @@ -0,0 +1,215 @@ +# include "psphotInternal.h" + +bool psphotSetMaskAndVariance (pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // Generate the mask and weight images, including the user-defined analysis region of interest + if (!psphotSetMaskAndVarianceReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to generate mask for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// generate mask and variance if not defined, additional mask for restricted subregion +bool psphotSetMaskAndVarianceReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + // find the currently selected readout + pmReadout *readout = pmFPAviewThisReadout (view, file->fpa); + psAssert (readout, "missing readout?"); + + // save maskSat and maskBad on the psphot recipe (mostly for psphotRoughClass) + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); // Mask value for saturated pixels + psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MASK.SAT", PS_META_REPLACE, "user-defined mask", maskSat); + + psImageMaskType maskBad = pmConfigMaskGet("LOW", config); // Mask value for low pixels + if (!maskBad) { + // for backward compatability look up old name + maskBad = pmConfigMaskGet("BAD", config); + } + psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MASK.BAD", PS_META_REPLACE, "user-defined mask", maskBad); + + // generate mask & variance images if they don't already exit + if (!readout->mask) { + if (!pmReadoutGenerateMask(readout, maskSat, maskBad)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble creating mask"); + return false; + } + } + if (!readout->variance) { + if (!pmReadoutGenerateVariance(readout, NULL, true)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble creating variance"); + return false; + } + } + + // insure than any non-finite pixels in image or variance are masked + // get the PSPHOT.MASK value from the config + psImageMaskType maskValue; + if (!pmConfigMaskSetBits (&maskValue, NULL, config)) { + psError (PS_ERR_UNKNOWN, false, "Unable to define the mask bit values"); + return false; + } + if (!pmReadoutMaskInvalid(readout, maskValue, maskSat)) { + psError (PS_ERR_UNKNOWN, false, "Unable to mask invalid pixels in readout."); + return false; + } + + bool softenVariance = psMetadataLookupBool (&status, recipe, "SOFTEN.VARIANCE"); + float softenFraction = psMetadataLookupF32 (&status, recipe, "SOFTEN.VARIANCE.FRACTION"); + + // make this an option via the recipe + if (softenVariance) { + psImage *im = readout->image; + psImage *wt = readout->variance; + for (int j = 0; j < im->numRows; j++) { + for (int i = 0; i < im->numCols; i++) { + if (!isfinite(im->data.F32[j][i])) continue; + if (!isfinite(wt->data.F32[j][i])) continue; + float sysError = softenFraction * im->data.F32[j][i]; + wt->data.F32[j][i] += PS_SQR(sysError); + } + } + } + + // mask the excluded outer pixels + // these coordinates refer to the parent image + // these bounds will saturate on the subimage + // negative upper bounds will subtract from the *subimage* + float XMIN = psMetadataLookupF32 (&status, recipe, "XMIN"); + float XMAX = psMetadataLookupF32 (&status, recipe, "XMAX"); + float YMIN = psMetadataLookupF32 (&status, recipe, "YMIN"); + float YMAX = psMetadataLookupF32 (&status, recipe, "YMAX"); + psRegion valid = psRegionSet (XMIN, XMAX, YMIN, YMAX); + + // restrict the supplied region above to the valid area on the image + psRegion keep = psRegionForImage (readout->image, valid); + + // psImageKeepRegion assumes the region refers to the parent coordinates + psImageKeepRegion (readout->mask, keep, "OR", maskBad); + + // test output of files at this stage + if (psTraceGetLevel("psphot.imsave") >= 5) { + psphotSaveImage (NULL, readout->image, "image.fits"); + psphotSaveImage (NULL, readout->mask, "mask.fits"); + psphotSaveImage (NULL, readout->variance, "variance.fits"); + } + + // display the image, weight, mask (ch 1,2,3) + psphotVisualShowImage (readout); + + return true; +} + +// XXX this function and support below was created to test the theory that the faint-end +// bias results from the Poisson variation of the background pixels. This is NOT the +// case. Using the code below maintains the faint-end bias. +bool psphotUpdateVariance (pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // Generate the mask and weight images, including the user-defined analysis region of interest + if (!psphotUpdateVarianceReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to generate mask for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// determine the mean variance image (equivalent to the background model, but for the variance image) +// set the variance image to the MAX(input, mean) +bool psphotUpdateVarianceReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + // find the currently selected readout + pmReadout *readout = pmFPAviewThisReadout (view, file->fpa); + psAssert (readout, "missing readout?"); + + pmSourceFitVarMode varMode = psphotGetFitVarMode (recipe); + if (varMode == PM_SOURCE_PHOTFIT_NONE) { + psError (PSPHOT_ERR_CONFIG, false, "need valid LINEAR_FIT_VARIANCE_MODE"); + return false; + } + + // make this an option via the recipe + if (varMode != PM_SOURCE_PHOTFIT_MODEL_SKY) return true; + + // create a model variance image (full-scale image to take result of psImageUnbin below) + psImage *modelVar = psImageCopy (NULL, readout->variance, PS_TYPE_F32); + + // find the binning information + psImageBinning *backBinning = psphotBackgroundBinning (modelVar, config); + assert (backBinning); + + psImage *varModel = psImageAlloc(backBinning->nXruff, backBinning->nYruff, PS_TYPE_F32); // Background model + psImage *varModelStdev = psImageAlloc(backBinning->nXruff, backBinning->nYruff, PS_TYPE_F32); // Background model + + if (!psphotModelBackgroundReadout(varModel, varModelStdev, NULL, readout, backBinning, config, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate background model"); + psFree (varModel); + psFree (varModelStdev); + return false; + } + + // linear interpolation to full-scale + if (!psImageUnbin (modelVar, varModel, backBinning)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning"); + psFree (varModel); + psFree (varModelStdev); + return false; + } + + // XXX save these? + psFree (varModel); + psFree (varModelStdev); + + psImage *im = readout->image; + psImage *wt = readout->variance; + for (int j = 0; j < im->numRows; j++) { + for (int i = 0; i < im->numCols; i++) { + if (!isfinite(im->data.F32[j][i])) continue; + if (!isfinite(wt->data.F32[j][i])) continue; + // XXX for a test, make variance constant wt->data.F32[j][i] = PS_MAX(wt->data.F32[j][i], modelVar->data.F32[j][i]); + wt->data.F32[j][i] = modelVar->data.F32[j][i]; + } + } + + // test output of files at this stage + if (psTraceGetLevel("psphot.imsave") >= 5) { + psphotSaveImage (NULL, readout->image, "image.varsky.fits"); + psphotSaveImage (NULL, readout->mask, "mask.varsky.fits"); + psphotSaveImage (NULL, readout->variance, "variance.varsky.fits"); + } + + psFree (modelVar); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMergeSources.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMergeSources.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMergeSources.c (revision 41083) @@ -0,0 +1,1097 @@ +# include "psphotInternal.h" + +// Mask to apply for PSF sources : only exclude bad sources -- we will re-test for extendedness +#define PSF_SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_SATSTAR | PM_SOURCE_MODE_BLEND | \ + PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_SATURATED | \ + PM_SOURCE_MODE_CR_LIMIT) + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotMergeSources (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotMergeSourcesReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to merge sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// add newly selected sources to the existing list of sources +bool psphotMergeSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *newSources = detections->newSources; + psAssert (newSources, "missing sources?"); + + if (!detections->allSources) { + detections->allSources = psArrayAllocEmpty(newSources->n); + } + psArray *allSources = detections->allSources; + + for (int i = 0; i < newSources->n; i++) { + pmSource *source = newSources->data[i]; + psArrayAdd (allSources, 100, source); + } + + psFree (detections->newSources); + detections->newSources = NULL; + + return true; +} + +// Merge the externally supplied sources with the existing sources. Mark them as having mode +// PM_SOURCE_MODE_EXTERNAL. + +// XXX this function needs to be updated slightly for psphotFullForce: +// * load the additional parameters to guide the new concepts + +// XXX This function needs to be updated to loop over set of input files. At the moment, we +// only expect a single entry for PSPHOT.INPUT.CMF and PSPHOT.SOURCES.TEXT, so we can only +// associate input sources with a single entry for the filerule +bool psphotLoadExtSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + bool status; + pmDetections *extCMF = NULL; + pmDetections *extCFF = NULL; + psArray *extSourcesTXT = NULL; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + detections = pmDetectionsAlloc(); + detections->newSources = psArrayAllocEmpty (100); + // save detections on the readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + } else { + psMemIncrRefCounter(detections); // so we can free the detections below + } + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + // load data from input CMF file: + { + pmReadout *readoutCMF = pmFPAfileThisReadout (config->files, view, "PSPHOT.INPUT.CMF"); + if (!readoutCMF) goto loadCFF; + + extCMF = psMetadataLookupPtr (NULL, readoutCMF->analysis, "PSPHOT.DETECTIONS"); + if (extCMF) { + for (int i = 0; i < extCMF->allSources->n; i++) { + pmSource *source = extCMF->allSources->data[i]; + source->mode |= PM_SOURCE_MODE_EXTERNAL; + + // the supplied peak flux needs to be re-normalized + source->peak->rawFlux = 1.0; + source->peak->smoothFlux = 1.0; + source->peak->detValue = 1.0; + + // drop the loaded source modelPSF + psFree (source->modelPSF); + source->modelPSF = NULL; + source->imageID = index; + + psArrayAdd (detections->newSources, 100, source); + } + } + } + +loadCFF: + // load data from input CFF file: + { + pmReadout *readoutCFF = pmFPAfileThisReadout (config->files, view, "PSPHOT.INPUT.CFF"); + if (!readoutCFF) goto loadTXT; + + extCFF = psMetadataLookupPtr (NULL, readoutCFF->analysis, "PSPHOT.DETECTIONS"); + if (extCFF) { + psF32 exptime = psMetadataLookupF32(NULL, readout->parent->concepts, "CELL.EXPOSURE"); + for (int i = 0; i < extCFF->allSources->n; i++) { + pmSource *source = extCFF->allSources->data[i]; + + // setting this bit not only tracks the inputs, it makes pmSourceMoments + // keep the Mx,My values for the centroid. + source->mode |= PM_SOURCE_MODE_EXTERNAL; + + // source->peak->detValue,rawFlux,smoothFlux all set to input flux value which is scaled + // to 1 second exposure time. Scale to this image's exposure. + source->peak->rawFlux *= exptime; + source->peak->smoothFlux *= exptime; + source->peak->detValue *= exptime; + // source->peak->xf,yf, moments->Mx,My all set to input position + + // drop the loaded source modelPSF + psFree (source->modelPSF); + source->modelPSF = NULL; + source->imageID = index; + + psArrayAdd (detections->newSources, 100, source); + } + } + } + +loadTXT: + + // load data from input TXT file: + { + pmChip *chipTXT = pmFPAfileThisChip (config->files, view, filerule); + if (!chipTXT) goto finish; + + extSourcesTXT = psMetadataLookupPtr (NULL, chipTXT->analysis, "PSPHOT.SOURCES.TEXT"); + if (extSourcesTXT) { + for (int i = 0; i < extSourcesTXT->n; i++) { + pmSource *source = extSourcesTXT->data[i]; + source->mode |= PM_SOURCE_MODE_EXTERNAL; + + // the supplied peak flux needs to be re-normalized + source->peak->rawFlux = 1.0; + source->peak->smoothFlux = 1.0; + source->peak->detValue = 1.0; + + // drop the loaded source modelPSF + psFree (source->modelPSF); + source->modelPSF = NULL; + source->imageID = index; + + psArrayAdd (detections->newSources, 100, source); + } + } + } + +finish: + + psFree (detections); + + if (!(extCMF || extCFF || extSourcesTXT)) { + psLogMsg ("psphot", 3, "no external sources for this readout"); + return true; + } + + int nCMF = extCMF ? extCMF->allSources->n : 0; + int nCFF = extCFF ? extCFF->allSources->n : 0; + int nTXT = extSourcesTXT ? extSourcesTXT->n : 0; + + psLogMsg ("psphot", 3, "%d external sources (%d cmf, %d cff, %d text) merged to yield %ld total sources", + nCMF + nCFF + nTXT, nCMF, nCFF, nTXT, sources->n); + return true; +} +bool psphotLoadExtSources(pmConfig *config, const pmFPAview *view, const char *filerule) { + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotLoadExtSourcesReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to load sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// copy the known sources (as external) to the detection list of the given filerule +bool psphotAddKnownSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources) { + + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // determine properties (sky, moments) of initial sources + float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + psAssert (status, "missing SKY_OUTER_RADIUS in recipe?"); + + // XXX this seems like an arbitrary number... + OUTER = PS_MAX(OUTER, 20.0); // XXX Guarantee that we can encompass the max moments radius + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + detections = pmDetectionsAlloc(); + detections->newSources = psArrayAllocEmpty (100); + // save detections on the readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + } else { + psMemIncrRefCounter(detections); // so we can free the detections below + } + + // copy the sources from inSources to the new detection structure + for (int i = 0; i < inSources->n; i++) { + pmSource *inSource = inSources->data[i]; + + pmSource *newSource = pmSourceCopy(inSource); + newSource->mode |= PM_SOURCE_MODE_EXTERNAL; + + // drop the loaded source modelPSF + psFree (newSource->modelPSF); + // source->modelPSF = NULL; check this! + + // drop the references to the original image pixels: + pmSourceFreePixels (newSource); + + // allocate image, weight, mask for the new image for each peak (square of radius OUTER) + pmSourceDefinePixels (newSource, readout, newSource->peak->x, newSource->peak->y, OUTER); + + newSource->imageID = 0; + // XXX reset the source ID? raised questions about the meaning of this ID... + // P_PM_SOURCE_SET_ID(source, i); + + psArrayAdd (detections->newSources, 100, newSource); + } + psLogMsg ("psphot", 3, "%ld known sources supplied", detections->newSources->n); + + psFree (detections); + return true; +} + +// extract the input sources corresponding to this readout +// XXX this function needs to be updated to work with the new context of psphot inputs +psArray *psphotLoadPSFSources (pmConfig *config, const pmFPAview *view) { + + bool status; + + // find the currently selected readout + pmReadout *readout = pmFPAfileThisReadout (config->files, view, "PSPHOT.INPUT.CMF"); + if (!readout) { + psLogMsg ("psphot", 3, "readout not found"); + return NULL; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + psLogMsg ("psphot", 3, "no psf sources for this readout"); + return NULL; + } + + psArray *sources = detections->allSources; + if (!sources) { + psLogMsg ("psphot", 3, "no psf sources for this readout"); + return NULL; + } + + return sources; +} + +// this function is used to fix sources which were loaded externally, but have passed from +// psphotDetectionsFromSources to psphotSourceStats and are now stored on +// detections->newSources. +bool psphotRepairLoadedSources (pmConfig *config, const pmFPAview *view, const char *filerule) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (NULL, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + psError(PSPHOT_ERR_CONFIG, false, "missing detections"); + return false; + } + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + // peak flux is wrong : set based on previous image + // use the peak measured in the moments analysis: + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + source->peak->rawFlux = source->moments->Peak; + source->peak->smoothFlux = source->moments->Peak; + } + + return true; +} + +// generate the detection structure for the supplied array of sources +// XXX this currently assumes there is a single input file +bool psphotDetectionsFromSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *sources) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = pmDetectionsAlloc(); + + detections->peaks = psArrayAllocEmpty(100); + + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + float snMin = psMetadataLookupF32(NULL, recipe, "MOMENTS_SN_MIN"); + if (!isfinite(snMin)) { + return false; + } + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + pmModel *model = source->modelPSF; + + if (source->mode & PSF_SOURCE_MASK || !isfinite(source->psfMag)) { + continue; + } + + // use the existing peak information, otherwise generate a new peak + if (source->peak) { + source->peak->assigned = false; // So the moments will be measured + psArrayAdd (detections->peaks, 100, source->peak); + continue; + } + + float flux = powf(10.0, -0.4 * source->psfMag); + float xpos = model->params->data.F32[PM_PAR_XPOS]; + float ypos = model->params->data.F32[PM_PAR_YPOS]; + + pmPeak *peak = pmPeakAlloc(xpos, ypos, flux, PM_PEAK_LONE); + peak->xf = xpos; + peak->yf = ypos; + + psArrayAdd (detections->peaks, 100, peak); + psFree (peak); + } + + psLogMsg ("psphot", 3, "%ld PSF sources loaded", detections->peaks->n); + psphotVisualShowSources (sources); + psphotVisualShowPeaks (detections); + + // save detections on the readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detectinos", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + psFree (detections); + + return true; +} + +// generate the detection structure for the supplied array of sources +// XXX this function is currently unused +bool psphotSetSourceParams (pmConfig *config, psArray *sources, pmPSF *psf) { + + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + pmModel *model = source->modelPSF; + + if (source->mode & PSF_SOURCE_MASK || !isfinite(source->psfMag)) { + continue; + } + + float flux = powf(10.0, -0.4 * source->psfMag); + float xpos = model->params->data.F32[PM_PAR_XPOS]; + float ypos = model->params->data.F32[PM_PAR_YPOS]; + + pmPeak *peak = pmPeakAlloc(xpos, ypos, flux, PM_PEAK_LONE); + peak->xf = xpos; + peak->yf = ypos; + peak->rawFlux = flux; // this are being set wrong, but does it matter? + peak->smoothFlux = flux; // this are being set wrong, but does it matter? + + source->peak = peak; + } + + psLogMsg ("psphot", 3, "%ld PSF sources loaded", sources->n); + + return true; +} + +bool psphotCheckExtSources (pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status; + + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + // XXX allSources of newSources? + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (sources->n) { + // the user wants to make the psf from these stars; define them as psf stars: + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + source->mode |= PM_SOURCE_MODE_PSFSTAR; + } + // force psphotChoosePSF to use all loaded sources + psMetadataAddS32 (recipe, PS_LIST_TAIL, "PSF_MAX_NSTARS", PS_META_REPLACE, "max number of sources for PSF model", sources->n); + + // measure stats of externally specified sources + if (!psphotSourceStatsUpdate (sources, config, readout)) { + psError(PSPHOT_ERR_CONFIG, false, "failure to measure stats of existing sources"); + return false; + } + } else { + + // find the detections (by peak and/or footprint) in the image. + if (!psphotFindDetections (config, view, filerule, true)) { + psError(PSPHOT_ERR_CONFIG, false, "unable to find detections in this image"); + return psphotReadoutCleanup (config, view, filerule); + } + + // construct sources and measure basic stats + psphotSourceStats (config, view, filerule, true); + + // find blended neighbors of very saturated stars + psphotDeblendSatstars (config, view, filerule); + + // mark blended peaks PS_SOURCE_BLEND + if (!psphotBasicDeblend (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed on deblend analysis"); + return psphotReadoutCleanup (config, view, filerule); + } + + // classify sources based on moments, brightness + if (!psphotRoughClass (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed to find a valid PSF clump for image"); + return psphotReadoutCleanup (config, view, filerule); + } + } + + return true; +} + +// copy the detections from one pmFPAfile to another +bool psphotCopySources (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc) +{ + bool status = true; + + int num = psphotFileruleCount(config, ruleSrc); + + // skip the chisq image because it is a duplicate of the detection version + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotCopySourcesReadout (config, view, ruleOut, ruleSrc, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to copy sources from %s to %s entry %d", ruleSrc, ruleOut, i); + return false; + } + } + return true; +} + +// add newly selected sources to the existing list of sources +bool psphotCopySourcesReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, ruleSrc, index); // File of interest + psAssert (fileSrc, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readoutSrc->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, ruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + // save detections on the readout->analysis + // XXX this replaced any existing entry; allow this operation to merge? + if (!psMetadataAddPtr (readoutOut->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + + + return true; +} + +// copy the newPeaks from the detections of one pmFPAfile to another +bool psphotCopyPeaks (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc) +{ + bool status = true; + + int num = psphotFileruleCount(config, ruleSrc); + + // skip the chisq image because it is a duplicate of the detection version + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotCopyPeaksReadout (config, view, ruleOut, ruleSrc, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to copy peaks from %s to %s entry %d", ruleSrc, ruleOut, i); + return false; + } + } + return true; +} + +// add newly detected peaks to the existing list of sources +bool psphotCopyPeaksReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, ruleSrc, index); // File of interest + psAssert (fileSrc, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readoutSrc->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, ruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + // generate a new detection structure for the output filerule + pmDetections *detectionsOut = psMetadataLookupPtr (&status, readoutOut->analysis, "PSPHOT.DETECTIONS"); + psAssert (detectionsOut, "missing PSPHOT.DETECTIONS?"); + + psAssert (detectionsOut->peaks, "programming error"); + psAssert (!detectionsOut->oldPeaks, "programming error"); + psAssert (detections->peaks, "programming error"); + + // save the OUT existing peaks on oldPeaks + detectionsOut->oldPeaks = detectionsOut->peaks; + detectionsOut->peaks = psArrayAllocEmpty(detections->peaks->n); + + for (int i = 0; i < detections->peaks->n; i++) { + psAssert (detections->peaks->data[i], "programming error"); + pmPeak *peak = pmPeakAlloc (0, 0, 0.0, PM_PEAK_LONE); + pmPeakCopy(peak, detections->peaks->data[i]); + psArrayAdd (detectionsOut->peaks, 100, peak); + psFree (peak); + } + return true; +} + +// create source parents children from ruleSrc for ruleOut for orphans +bool psphotSourceParents (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc) +{ + bool status = true; + + int num = psphotFileruleCount(config, ruleSrc); + + // skip the chisq image because it is a duplicate of the detection version + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotSourceParentsReadout (config, view, ruleOut, ruleSrc, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to copy sources from %s to %s entry %d", ruleSrc, ruleOut, i); + return false; + } + } + return true; +} + +// create source parents from ruleSrc for ruleOut for orphaned children for this readout. +bool psphotSourceParentsReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index) { + + bool status; + int nParents = 0; + int nNonOrphans = 0; + + // find the currently selected readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, ruleSrc, index); // File of interest + psAssert (fileSrc, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmDetections *detectionsSrc = psMetadataLookupPtr (&status, readoutSrc->analysis, "PSPHOT.DETECTIONS"); + psAssert (detectionsSrc, "missing detections?"); + + psArray *sourcesSrc = detectionsSrc->allSources; + psAssert (sourcesSrc, "missing sources?"); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, ruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + // generate a new detection structure for the output filerule + pmDetections *detectionsOut = psMetadataLookupPtr (&status, readoutOut->analysis, "PSPHOT.DETECTIONS"); + psAssert (detectionsOut, "missing PSPHOT.DETECTIONS?"); + + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + + // copy the sources from sourceSrcs to the new detection structure + for (int i = 0; i < sourcesSrc->n; i++) { + pmSource *sourceSrc = sourcesSrc->data[i]; + if (sourceSrc->parent) { + nNonOrphans ++; + continue; // Not an orphan + } + + pmSource *sourceOut = pmSourceCopy(sourceSrc); + sourceOut->parent = sourceSrc; + + // keep the original source flags + sourceOut->seq = sourceSrc->seq; + sourceOut->type = sourceSrc->type; + sourceOut->mode = sourceSrc->mode; + sourceOut->mode2 = sourceSrc->mode2; + sourceOut->tmpFlags = sourceSrc->tmpFlags; + + // does this copy all model data? (NO) + sourceOut->modelPSF = pmModelCopy(sourceSrc->modelPSF); + sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); + + if (sourceSrc->modelFits) { + sourceOut->modelFits = psArrayAlloc(sourceSrc->modelFits->n); + for (int j = 0; j < sourceSrc->modelFits->n; j++) { + sourceOut->modelFits->data[j] = pmModelCopy(sourceSrc->modelFits->data[j]); + } + } + + // drop the references to the original image pixels: + pmSourceFreePixels (sourceOut); + + // allocate image, weight, mask for the new image for each peak + if (sourceOut->modelPSF) { + pmSourceRedefinePixels (sourceOut, readoutOut, sourceOut->peak->x, sourceOut->peak->y, sourceOut->modelPSF->fitRadius); + } else { + // if we have no pixels we can't use it to determine the psf so make sure this bit is off + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; + } + + // child sources have not been subtracted in this image, but this flag may be raised if + // they were subtracted in the parent's image + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + + nParents ++; + psArrayAdd (detectionsOut->allSources, 100, sourceOut); + psFree (sourceOut); + } + psLogMsg ("psphot", 3, "%d parents created, %d unorphaned children, %ld input vs %ld output", nParents, nNonOrphans, sourcesSrc->n, detectionsOut->allSources->n); + + return true; +} + +// create source children from ruleSrc for ruleOut +bool psphotSourceChildren (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc) +{ + bool status = true; + + int num = psphotFileruleCount(config, ruleSrc); + + // skip the chisq image because it is a duplicate of the detection version + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotSourceChildrenReadout (config, view, ruleOut, ruleSrc, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to copy sources from %s to %s entry %d", ruleSrc, ruleOut, i); + return false; + } + } + return true; +} + +// Create source children from ruleSrc for ruleOut for this entry. Currently, this is only +// used by psphotStackReadout (sources go on allSources so that psphotChoosePSF can be called +// repeatedly). +bool psphotSourceChildrenReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, ruleSrc, index); // File of interest + psAssert (fileSrc, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmDetections *detectionsSrc = psMetadataLookupPtr (&status, readoutSrc->analysis, "PSPHOT.DETECTIONS"); + psAssert (detectionsSrc, "missing detections?"); + + psArray *sourcesSrc = detectionsSrc->allSources; + psAssert (sourcesSrc, "missing sources?"); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, ruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + pmDetections *detectionsOutOld = psMetadataLookupPtr (&status, readoutOut->analysis, "PSPHOT.DETECTIONS"); + psArray *oldFootprints = detectionsOutOld ? detectionsOutOld->footprints : NULL; + + // replace any existing DETECTION container on readoutOut->analysis with the new one + pmDetections *detectionsOut = pmDetectionsAlloc(); + if (oldFootprints) { + // ... but hang on to any existing footprints so that they can be merged with new footprints in pass 2 + detectionsOut->footprints = psMemIncrRefCounter(oldFootprints); + } + detectionsOut->allSources = psArrayAllocEmpty (100); + if (!psMetadataAddPtr (readoutOut->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detectionsOut)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return false; + } + + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + + // copy the sources from sourceSrcs to the new detection structure + for (int i = 0; i < sourcesSrc->n; i++) { + pmSource *sourceSrc = sourcesSrc->data[i]; + + pmSource *sourceOut = pmSourceCopy(sourceSrc); + sourceOut->parent = sourceSrc; + + // keep the original source flags + sourceOut->seq = sourceSrc->seq; + sourceOut->type = sourceSrc->type; + sourceOut->mode = sourceSrc->mode; + sourceOut->mode2 = sourceSrc->mode2; + sourceOut->tmpFlags = sourceSrc->tmpFlags; + + // does this copy all model data? (NO) + sourceOut->modelPSF = pmModelCopy(sourceSrc->modelPSF); + sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); + + if (sourceSrc->modelFits) { + sourceOut->modelFits = psArrayAlloc(sourceSrc->modelFits->n); + for (int j = 0; j < sourceSrc->modelFits->n; j++) { + sourceOut->modelFits->data[j] = pmModelCopy(sourceSrc->modelFits->data[j]); + } + } + + // drop the references to the original image pixels: + pmSourceFreePixels (sourceOut); + + // XXX do we need to skip the Chisq image sources? + + // allocate image, weight, mask for the new image for each peak + pmSourceRedefinePixels (sourceOut, readoutOut, sourceOut->peak->x, sourceOut->peak->y, sourceOut->windowRadius); + + // child sources have not been subtracted in this image, but this flag may be raised if + // they were subtracted in the parent's image + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + + psArrayAdd (detectionsOut->allSources, 100, sourceOut); + psFree (sourceOut); + } + psLogMsg ("psphot", 3, "created %ld children", detectionsOut->allSources->n); + + psFree(detectionsOut); // a copy remains on the analysis metadata + + return true; +} + +// create source children associated with 'filerule' from the objectsSrc. returns a new object +// array containing the child sources. XXX currently, this is only used by psphotStackReadout +// (sources go on allSources so that psphotChoosePSF can be called repeatedly) +psArray *psphotSourceChildrenByObject (pmConfig *config, const pmFPAview *view, const char *fileruleOut, const char *fileruleSrc, psArray *objectsSrc, bool sourcesSubtracted) { + + bool status; + + int nImages = psphotFileruleCount(config, fileruleOut); + + // generate look-up arrays for detections and readouts + psArray *detArrays = psArrayAlloc(nImages); + psArray *readouts = psArrayAlloc(nImages); + psArray *fitOptionsArray = psArrayAlloc(nImages); + + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + for (int i = 0; i < nImages; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, fileruleOut, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, fileruleSrc, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + + // create DETECTIONS containers for each image, in case one lacks it + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + detections = pmDetectionsAlloc(); + detections->allSources = psArrayAllocEmpty (100); + detections->peaks = psArrayAllocEmpty (100); + // save detections on the readout->analysis + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot detections", detections)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving detections on readout"); + return NULL; + } + psFree(detections); // a copy remains on the analysis metadata + psAssert (detections, "missing detections?"); + } + pmSourceFitOptions *fitOptions = NULL; + if (psMetadataLookupBool(&status, recipe, "EXTENDED_SOURCE_FITS")) { + fitOptions = psMetadataLookupPtr (&status, readoutSrc->analysis, "PCM_FIT_OPTIONS"); + psAssert (fitOptions, "missing pcm fit options"); + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PCM_FIT_OPTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "pcm fit options", fitOptions); + } + + // we need to save the new sources on the detection arrays of the appropriate image + detArrays->data[i] = psMemIncrRefCounter(detections); + readouts->data[i] = psMemIncrRefCounter(readout); + if (fitOptions) { + fitOptionsArray->data[i] = psMemIncrRefCounter(fitOptions); + } + } + + psArray *objectsOut = psArrayAlloc(objectsSrc->n); + + // copy all sources for each object + for (int k = 0; k < objectsSrc->n; k++) { + + pmPhotObj *objectSrc = objectsSrc->data[k]; + if (!objectSrc) continue; + if (!objectSrc->sources) continue; + + pmPhotObj *objectOut = pmPhotObjAlloc(); + objectsOut->data[k] = objectOut; + + objectOut->flux = objectSrc->flux; + objectOut->x = objectSrc->x; + objectOut->y = objectSrc->y; + + objectOut->sources = psArrayAlloc(objectSrc->sources->n); + + // copy the sources from sourceSrcs to the new detection structure + // loop over the sources, redefine their pixels to point at the new filerule image, + // copy the source data, and add a reference back to the original source + for (int i = 0; i < objectSrc->sources->n; i++) { + + pmSource *sourceSrc = objectSrc->sources->data[i]; + + pmSource *sourceOut = pmSourceCopy(sourceSrc); + sourceOut->parent = sourceSrc; + + // save on the output object array at the same location + objectOut->sources->data[i] = sourceOut; + + // keep the original source flags and sequence ID (if set) + sourceOut->seq = sourceSrc->seq; + sourceOut->type = sourceSrc->type; + sourceOut->mode = sourceSrc->mode; + sourceOut->mode2 = sourceSrc->mode2; + sourceOut->tmpFlags = sourceSrc->tmpFlags; + + // does this copy all model data? (NO) + sourceOut->modelPSF = pmModelCopy(sourceSrc->modelPSF); + + bool foundModelEXT = false; + if (sourceSrc->modelFits) { + sourceOut->modelFits = psArrayAlloc(sourceSrc->modelFits->n); + for (int j = 0; j < sourceSrc->modelFits->n; j++) { + pmModel *modelSrc = sourceSrc->modelFits->data[j]; + pmModel *modelOut = sourceOut->modelFits->data[j] = pmModelCopy(modelSrc); + if (modelSrc == sourceSrc->modelEXT) { + foundModelEXT = true; + sourceOut->modelEXT = psMemIncrRefCounter (modelOut); + } + modelOut->isPCM = modelSrc->isPCM; + } + } + if (!foundModelEXT && sourceSrc->modelEXT) { + // Will this ever happen? + sourceOut->modelEXT = pmModelCopy(sourceSrc->modelEXT); + } + + // drop the references to the original image pixels: + pmSourceFreePixels (sourceOut); + + // set the output readout + int index = sourceOut->imageID; + if (index >= readouts->n) continue; // skip the sources generated by the chisq image + pmReadout *readout = readouts->data[index]; + + pmSourceFitOptions *fitOptions = fitOptionsArray->data[index]; + + // allocate image, weight, mask for the new image for each peak + if (sourceOut->modelPSF) { + pmSourceRedefinePixels (sourceOut, readout, sourceOut->peak->x, sourceOut->peak->y, + sourceSrc->windowRadius); + } else { + // if we have no pixels we can't use it to determine the psf so make sure this bit is off + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; + } + + // child sources have not been subtracted in this image, but this flag may be raised if + // they were subtracted in the parent's image + // XXX NOTE : in the pre-20130914 version of psphotStack, we carried a copy of the pixels + // generated before the subtraction took place (and then we smoothed to match the desired PSF). + // in the new version, we copy the image after subtraction; we need to distinguish these cases + if (!sourcesSubtracted) { + sourceOut->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + } else { + bool isPSF = false; + pmModel *model = pmSourceGetModel (&isPSF, sourceOut); + if (model && sourceSrc->modelFlux) { + if (model->isPCM) { + pmPCMdata *pcm = pmPCMinit (sourceOut, fitOptions, model, maskVal, psfSize); + if (pcm) { + // pmPCMMakeModel (sourceOut, model, pcm->nsigma, maskVal, psfSize); + pmPCMCacheModel (sourceOut, maskVal, psfSize, pcm->nsigma); + psFree(pcm); + } else { + // What to do here? + // psAssert (pcm, "pmPCMinit failed!"); + psFree (sourceOut->modelEXT); + sourceOut->modelEXT = NULL; + } + } else { + pmSourceCacheModel (sourceOut, maskVal); + } + } + } + + // set the output detections: + pmDetections *detectionsOut = detArrays->data[index]; + psArrayAdd (detectionsOut->allSources, 100, sourceOut); + psArrayAdd (detectionsOut->peaks, 100, sourceOut->peak); + } + } + + for (int i = 0; i < nImages; i++) { + pmDetections *detections = detArrays->data[i]; + psLogMsg ("psphot", 3, "%ld source children for image %d", detections->allSources->n, i); + } + + psFree (detArrays); + psFree (readouts); + psFree (fitOptionsArray); + + return objectsOut; +} + +bool psphotCopyEfficiency (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc) +{ + bool status = true; + + int num = psphotFileruleCount(config, ruleSrc); + + // skip the chisq image because it is a duplicate of the detection version + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotCopyEfficiencyReadout (config, view, ruleOut, ruleSrc, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to copy detection efficiency from %s to %s entry %d", ruleSrc, ruleOut, i); + return false; + } + } + return true; +} + +// add newly selected sources to the existing list of sources +bool psphotCopyEfficiencyReadout (pmConfig *config, const pmFPAview *view, const char *ruleOut, const char *ruleSrc, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, ruleSrc, index); // File of interest + psAssert (fileSrc, "missing file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, ruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + pmDetEff *de = psMetadataLookupPtr(&status, readoutSrc->analysis, PM_DETEFF_ANALYSIS); // Detection efficiency + if (!status || !de) { + // nothing there + return true; + } + + // save DetEff on the readoutOut->analysis + if (!psMetadataAddPtr (readoutOut->analysis, PS_LIST_TAIL, PM_DETEFF_ANALYSIS, PS_META_REPLACE | PS_DATA_UNKNOWN, "Detection efficiency", de)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving Detection efficiency on readout"); + return false; + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMinimal.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMinimal.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMinimal.c (revision 41083) @@ -0,0 +1,36 @@ +# include "psphotStandAlone.h" + +int main (int argc, char **argv) { + + psMemInit(); // needed if USE_SPINLOCK is set in psMemory.c + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotMinimalArguments (argc, argv); + assert(config); + + psphotVersionPrint(); + + // load input data (config and images (signal, noise, mask) + if (!psphotParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotImageLoop (config, PSPHOT_MINIMAL)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", PS_LOG_WARN, "complete psphot run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMinimalArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMinimalArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMinimalArguments.c (revision 41083) @@ -0,0 +1,200 @@ +# include "psphotStandAlone.h" + +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile) +{ + fprintf(ofile, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "%s -help\n" + "\n" + "%s -version\n" + "\n" + "where:\n" + " FileNameList is a text file containing filenames, one per line\n" + " MaskFileNameList is a text file of mask filenames, one per line\n" + " VarFileNameList is a text file of variance filenames, one per line\n" + " OutFileBaseName is the 'root name' for output files\n" + "\n" + "additional options:\n" + " -modeltest xObj yObj [-model DEFAULT|ModelName] \n" + " [-fitmode DEFAULT|PSF|CONV] [-fitset FitFileName]\n" + " Test fit for object at the given coordinates. ModelName\n" + " is one of PS_MODEL_GAUSS, PS_MODEL_PGAUSS, PS_MODEL_QGAUSS,\n" + " PS_MODEL_RGAUSS, PS_MODEL_PS1_V1, or PS_MODEL_SERSIC.\n" + " FitFileName is a file of x,y,Io triples.\n" + " -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n" + " specify PSF rather than letting %s estimate it\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify additional sources for PSF generation\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -photcode PhotoCodeName\n" + " specify photocode\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -nthreads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n", + program,program,program,program,program); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "Try '%s -help' for more options and explanation\n", + program,program,program); + if (exitCode != PS_EXIT_SUCCESS) + psErrorStackPrint(stderr, "Error reading arguments\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + +pmConfig *psphotMinimalArguments(int argc, char **argv) { + + int N; + bool status; + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + // generic arguments (version, dumpconfig) + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // thread arguments + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + // photcode : used in output to supplement header data (argument or recipe?) + if ((N = psArgumentGet (argc, argv, "-photcode"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "PHOTCODE", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // break : used from recipe throughout psphotReadout + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF + if ((N = psArgumentGet (argc, argv, "-region"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + // + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "PSPHOT.PSF", "-psf", "-psflist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC", "-src", "-srclist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "EXPNUM", "-expnum", "-expnumlist"); + + if (argc == 1) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (psArgumentGet(argc, argv, "-help") || + psArgumentGet(argc, argv, "-h")) + writeHelpInfo(argv[0], config, stdout); + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotArguments...\n"); + return (config); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelBackground.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelBackground.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelBackground.c (revision 41083) @@ -0,0 +1,595 @@ +# include "psphotInternal.h" +static int npass = 0; +static char *defaultStatsName = "FITTED_MEAN"; + +// Determine the binning characteristics for the background model +// Sets the ruff image size and skip based on recipe values for the binning +psImageBinning *psphotBackgroundBinning(const psImage *image, // Image for which to generate a bg model + const pmConfig *config // Configuration + ) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // I have the fine image size, I know the binning factor, determine the ruff image size + psImageBinning *binning = psImageBinningAlloc(); + binning->nXfine = image->numCols; + binning->nYfine = image->numRows; + binning->nXbin = psMetadataLookupS32(&status, recipe, "BACKGROUND.XBIN"); + binning->nYbin = psMetadataLookupS32(&status, recipe, "BACKGROUND.YBIN"); + + psImageBinningSetRuffSize(binning, PS_IMAGE_BINNING_CENTER); + psImageBinningSetSkip(binning, image); + + return binning; +} + +// track background cell failures for log reporting purposes +static int nFailures = 0; + +// Generate the background model +// generate the median in NxN boxes, clipping heavily +// linear interpolation to generate full-scale model +// +// NOTE that the 'analysis' metedata passed in here is used to store the binning information. +// This may be the analysis for this readout, but it may be the analysis for the pmFPAfile +// corresponding to the model. Other information about the background model is saved on the +// readout->analysis +bool psphotModelBackgroundReadout(psImage *model, // Model image + psImage *modelStdev, // Model stdev image + psMetadata *analysis, // Analysis metadata for outputs + pmReadout *readout, // Readout for which to generate a background model + psImageBinning *binning, // Binning parameters + const pmConfig *config,// Configuration + bool useVarianceImage + ) +{ + psTimerStart ("psphot.background"); + + bool status = true; + + psImage *image = useVarianceImage ? readout->variance : readout->image; + psImage *mask = readout->mask; // Image and mask for readout + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // mark pixel may be used for optional iteration -- is not required by all processes + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + + // apply both MASK and MARK to make background model + maskVal |= markVal; + + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + + float dXsample = psMetadataLookupF32(&status, recipe, "BACKGROUND.XSAMPLE"); + if (!status) dXsample = 1.0; + float dYsample = psMetadataLookupF32(&status, recipe, "BACKGROUND.YSAMPLE"); + if (!status) dYsample = 1.0; + + // subtract this amount extra from the sky + float SKY_BIAS = psMetadataLookupF32 (&status, recipe, "SKY_BIAS"); + if (!status) { + SKY_BIAS = 0; + } + + // supply the sky background statistics options + char *statsName = psMetadataLookupStr (&status, recipe, "SKY_STAT"); + if (statsName == NULL) { + statsName = defaultStatsName; + } + psStatsOptions statsOptionLocation = psStatsOptionFromString(statsName); + if (!(statsOptionLocation & (PS_STAT_SAMPLE_MEAN | + PS_STAT_SAMPLE_MEDIAN | + PS_STAT_ROBUST_MEDIAN | + PS_STAT_ROBUST_QUARTILE | + PS_STAT_CLIPPED_MEAN | + PS_STAT_FITTED_MEAN))) { + statsOptionLocation = PS_STAT_FITTED_MEAN; + } + + psStatsOptions statsOptionWidth = PS_STAT_NONE; + if (statsOptionLocation & (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_MEDIAN)) { + statsOptionWidth = PS_STAT_SAMPLE_STDEV; + } else if (statsOptionLocation & (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_QUARTILE)) { + statsOptionWidth = PS_STAT_ROBUST_STDEV; + } else if (statsOptionLocation & PS_STAT_FITTED_MEAN) { + statsOptionWidth = PS_STAT_FITTED_STDEV; + } else if (statsOptionLocation & PS_STAT_CLIPPED_MEAN) { + statsOptionWidth = PS_STAT_CLIPPED_STDEV; + } else { + psAbort("Unable to estimate variance of selected statsOptionLocations 0x%x", statsOptionLocation); + } + + const psStatsOptions statsOption = statsOptionLocation | statsOptionWidth; + psStats *statsDefaults = psStatsAlloc (statsOption); + + // set range for old-version of sky statistic + if (statsOptionLocation & PS_STAT_ROBUST_QUARTILE) { + statsDefaults->min = 0.25; + statsDefaults->max = 0.75; + } + + // set user-option for number of pixels per region + statsDefaults->nSubsample = psMetadataLookupF32 (&status, recipe, "IMSTATS_NPIX"); + if (!status) { + statsDefaults->nSubsample = 1000; + } + + // optionally set the binsize + statsDefaults->binsize = psMetadataLookupF32 (&status, recipe, "SKY_HISTOGRAM_BINS"); + if (status) { + statsDefaults->options |= PS_STAT_USE_BINSIZE; + } + + // optionally set the sigma clipping + statsDefaults->clipSigma = psMetadataLookupF32 (&status, recipe, "SKY_CLIP_SIGMA"); + if (!status) { + if (statsDefaults->options & PS_STAT_FITTED_MEAN) { + statsDefaults->clipSigma = 1.0; + } else { + statsDefaults->clipSigma = 3.0; + } + } + + // we save the binning structure for use in psphotMagnitudes + if (!useVarianceImage) { + psMetadataAddPtr(analysis, PS_LIST_TAIL, "PSPHOT.BACKGROUND.BINNING", PS_DATA_UNKNOWN | PS_META_REPLACE, "Background binning", binning); + } + + psVector *dQ = psVectorAllocEmpty (100, PS_TYPE_F32); + psF32 **modelData = model->data.F32; + psF32 **modelStdevData = modelStdev->data.F32; + + // XXXX we can thread this here by running blocks in parallel + psImageBackgroundInit(); + nFailures = 0; // reset for this pass + + // we have Nx * Ny model points, but we can use a window which is larger (or smaller) than + // 1 superpixel. If we have a window of size dXsample * dYsample, then the regions run from: + // (ix + 0.5) - dX to (ix + 0.5) + dX + + // measure clipped median for subimages + for (int iy = 0; iy < model->numRows; iy++) { + for (int ix = 0; ix < model->numCols; ix++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_MODEL_BACKGROUND"); + + psArrayAdd(job->args, 1, image); + psArrayAdd(job->args, 1, mask); + psArrayAdd(job->args, 1, binning); + psArrayAdd(job->args, 1, rng); + psArrayAdd(job->args, 1, statsDefaults); + + psArrayAdd(job->args, 1, modelData); + psArrayAdd(job->args, 1, modelStdevData); + + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + + PS_ARRAY_ADD_SCALAR(job->args, ix, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, iy, PS_TYPE_S32); + + PS_ARRAY_ADD_SCALAR(job->args, dXsample, PS_TYPE_F32); + PS_ARRAY_ADD_SCALAR(job->args, dYsample, PS_TYPE_F32); + + PS_ARRAY_ADD_SCALAR(job->args, statsOptionLocation, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, statsOptionWidth, PS_TYPE_S32); + + PS_ARRAY_ADD_SCALAR(job->args, NAN, PS_TYPE_F32); // this is used as a return value for dQvalue + +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return NULL; + } +# else + if (!psphotModelBackground_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Failure to model background."); + return NULL; + } + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = job->args->data[14]; + float dQvalue = scalar->data.F32; + psVectorAppend (dQ, dQvalue); + } + psFree(job); +# endif + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return NULL; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = job->args->data[14]; + float dQvalue = scalar->data.F32; + psVectorAppend (dQ, dQvalue); + } + psFree(job); + } + + if (nFailures) { + psLogMsg ("psphot", PS_LOG_WARN, "Failed to estimate background for %d of %d subimages", nFailures, (model->numRows*model->numCols)); + } + + if (psTraceGetLevel("psphot") > 5) { + char name[256]; + sprintf (name, "backraw.%02d.fits", npass); + psphotSaveImage (NULL, model, name); + } + + // patch over bad regions (use average of 8 possible neighbor pixels) + // XXX consider testing all pixels against the 8 neighbors and replacing outliers... + double Count = 0; // number of good pixels + double Value = 0; // sum of good pixel's value + double ValueStdev = 0; // sum of good pixel's standard deviations + for (int iy = 0; iy < model->numRows; iy++) { + for (int ix = 0; ix < model->numCols; ix++) { + if (isfinite(modelData[iy][ix])) { + Value += modelData[iy][ix]; + ValueStdev += modelStdevData[iy][ix]; + Count++; + continue; + } + + double value = 0; + double count = 0; + for (int jy = iy - 1; jy <= iy + 1; jy++) { + if (jy < 0) continue; + if (jy >= model->numRows) continue; + for (int jx = ix - 1; jx <= ix + 1; jx++) { + if (!jx && !jy) continue; + if (jx < 0) continue; + if (jx >= model->numCols) continue; + if (!isfinite(modelData[jy][jx])) continue; + value += modelData[jy][jx]; + count += 1.0; + } + } + if (count > 0) { + psLogMsg ("psphot", PS_LOG_DETAIL, "patching background %d, %d: %f (%d pts)\n", ix, iy, (value / count), (int) count); + modelData[iy][ix] = value / count; + } + } + } + if (Count == 0) { + psError (PSPHOT_ERR_DATA, true, "failed to build background image"); + psFree(statsDefaults); + psFree(binning); + psFree(rng); + psFree(dQ); + return false; + } + + Value /= Count; + ValueStdev /= Count; + + // patch over remaining bad regions (use global average) + for (int iy = 0; iy < model->numRows; iy++) { + for (int ix = 0; ix < model->numCols; ix++) { + if (!isnan(modelData[iy][ix])) continue; + modelData[iy][ix] = Value; + modelStdevData[iy][ix] = ValueStdev; + } + } + + // apply artificial offset if desired + for (int iy = 0; iy < model->numRows; iy++) { + for (int ix = 0; ix < model->numCols; ix++) { + modelData[iy][ix] += SKY_BIAS; + } + } + + if (psTraceGetLevel("psphot") > 5) { + char name[256]; + sprintf (name, "backfill.%02d.fits", npass); + psphotSaveImage (NULL, model, name); + sprintf (name, "backfist.%02d.fits", npass); + psphotSaveImage (NULL, modelStdev, name); + } + + psLogMsg ("psphot", PS_LOG_WARN, "built background image: %f sec\n", psTimerMark ("psphot.background")); + + psStats *statsDQ = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN); + psVectorStats (statsDQ, dQ, NULL, NULL, 0); + + if (!useVarianceImage) { + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_MEAN", PS_META_REPLACE, "sky mean", Value); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_STDEV", PS_META_REPLACE, "sky stdev", ValueStdev); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SKY_DQ", PS_META_REPLACE, "sky quartile slope", statsDQ->sampleMedian); + psLogMsg ("psphot", PS_LOG_INFO, "image sky : mean %f, stdev %f, dQ %f", Value, ValueStdev, statsDQ->sampleMedian); + + // measure image and background stats and save for later output + psStats *statsBck = psStatsAlloc (PS_STAT_SAMPLE_MEAN | + PS_STAT_SAMPLE_STDEV | + PS_STAT_MIN | + PS_STAT_MAX); + psImageStats (statsBck, model, NULL, 0); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MN", PS_META_REPLACE, "sky model mean", statsBck->sampleMean); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_SIG", PS_META_REPLACE, "sky model stdev", statsBck->sampleStdev); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_DEV", PS_META_REPLACE, "sky stdev", ValueStdev); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_DQ", PS_META_REPLACE, "sky quartile slope", statsDQ->sampleMedian); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MAX", PS_META_REPLACE, "sky model maximum value", statsBck->max); + psMetadataAddF32 (readout->analysis, PS_LIST_TAIL, "MSKY_MIN", PS_META_REPLACE, "sky model minimum value", statsBck->min); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "MSKY_NX", PS_META_REPLACE, "sky model size (x)", model->numCols); + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "MSKY_NY", PS_META_REPLACE, "sky model size (y)", model->numRows); + psLogMsg ("psphot", PS_LOG_INFO, "background sky : min %f mean %f max %f stdev %f", + statsBck->min, statsBck->sampleMean, statsBck->max, statsBck->sampleStdev); + psFree(statsBck); + } else { + psLogMsg ("psphot", PS_LOG_INFO, "variance data : mean %f, stdev %f, dQ %f", Value, ValueStdev, statsDQ->sampleMedian); + } + + psFree(statsDQ); + psFree(dQ); + + psFree(statsDefaults); + psFree(binning); + psFree(rng); + + return model; +} + +// generate a background model for a single readout. do not save an associated pmFPAfile for possible output +psImage *psphotModelBackgroundReadoutNoFile(pmReadout *readout, const pmConfig *config) +{ + PM_ASSERT_READOUT_NON_NULL(readout, NULL); + PM_ASSERT_READOUT_IMAGE(readout, NULL); + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psImageBinning *binning = psphotBackgroundBinning(readout->image, config); // Image binning parameters + psImage *model = psImageAlloc(binning->nXruff, binning->nYruff, PS_TYPE_F32); // Background model + psImage *modelStdev = psImageAlloc(binning->nXruff, binning->nYruff, PS_TYPE_F32); // Standard deviation + + if (!psphotModelBackgroundReadout(model, modelStdev, readout->analysis, readout, binning, config, false)) { + psFree(model); + psFree(modelStdev); + psError(PS_ERR_UNKNOWN, false, "Unable to generate background model"); + return NULL; + } + psFree(modelStdev); + return model; +} + +// generate a background model for readout number index; save an associated pmFPAfile for possible output +bool psphotModelBackgroundReadoutFileIndex (pmConfig *config, const pmFPAview *view, const char *filename, int index) +{ + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filename, index); // File of interest + psAssert (file, "missing file?"); + + pmFPA *inFPA = file->fpa; + pmReadout *readout = pmFPAviewThisReadout(view, inFPA); + psAssert (readout, "missing readout?"); + + psImageBinning *binning = psphotBackgroundBinning(readout->image, config); // Image binning parameters + pmReadout *model = pmFPAGenerateReadout(config, view, psphotGetFilerule("PSPHOT.BACKMDL"), inFPA, binning, index); + pmReadout *modelStdev = pmFPAGenerateReadout(config, view, psphotGetFilerule("PSPHOT.BACKMDL.STDEV"), inFPA, binning, index); + + if (!psphotModelBackgroundReadout(model->image, modelStdev->image, model->analysis, readout, binning, config, false)) { + int lastError = psErrorCodeLast(); + if (lastError == PSPHOT_ERR_DATA) { + // depending on context this value may or may not be used + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "PSPHOT_QUALITY", PS_META_REPLACE, "quality error due to poor data", lastError); + } + psError(lastError, false, "Unable to generate background model"); + return false; + } + + npass ++; + return true; +} + +bool psphotModelBackground (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + int num = psphotFileruleCount(config, filerule); + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Model Background ---"); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotModelBackgroundReadoutFileIndex(config, view, filerule, i)) { + int lastError = psErrorCodeLast(); + psError (lastError ? lastError : PSPHOT_ERR_CONFIG, false, "failed to model background for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotModelBackground_Threaded (psThreadJob *job) { + + psImage *image = job->args->data[0]; + psImage *mask = job->args->data[1]; + + psImageBinning *binning = job->args->data[2]; + + psRandom *rng = job->args->data[3]; + psStats *statsDefaults = job->args->data[4]; + + psF32 **modelData = job->args->data[5]; + psF32 **modelStdevData = job->args->data[6]; + + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[7],PS_TYPE_IMAGE_MASK_DATA); + + int ix = PS_SCALAR_VALUE(job->args->data[8],S32); + int iy = PS_SCALAR_VALUE(job->args->data[9],S32); + + float dXsample = PS_SCALAR_VALUE(job->args->data[10],F32); + float dYsample = PS_SCALAR_VALUE(job->args->data[11],F32); + + psStatsOptions statsOptionLocation = PS_SCALAR_VALUE(job->args->data[12],S32); + psStatsOptions statsOptionWidth = PS_SCALAR_VALUE(job->args->data[13],S32); + + // convert the ruff grid cell to the equivalent fine grid cell + psRegion ruffRegion = psRegionSet (ix + 0.5 - 0.5*dXsample, ix + 0.5 + 0.5*dXsample, iy + 0.5 - 0.5*dYsample, iy + 0.5 + 0.5*dYsample); + psRegion fineRegion = psImageBinningSetFineRegion (binning, ruffRegion); + fineRegion = psRegionForImage (image, fineRegion); + + psImage *subset = psImageSubset (image, fineRegion); + if (!subset->numCols || !subset->numRows) { + psFree (subset); + return false; // XXX do we / should we fail on this? + } + psImage *submask = psImageSubset (mask, fineRegion); + + psStats *stats = psStatsAlloc (PS_STAT_NONE); + *stats = *statsDefaults; + + // Use the selected background statistic for the first pass + // If it fails, fall back on the "ROBUST_MEDIAN" version + // If both fail, set the pixel to NAN and (later) interpolate + + psVector *sample = NULL; + float dQvalue = NAN; + + if (psImageBackground(stats, &sample, subset, submask, maskVal, rng)) { + if (stats->options & PS_STAT_ROBUST_QUARTILE) { + modelData[iy][ix] = stats->robustMedian; + } else { + modelData[iy][ix] = psStatsGetValue(stats, statsOptionLocation); + } + modelStdevData[iy][ix] = psStatsGetValue(stats, statsOptionWidth); + + // fprintf (stderr, "dQ : %f - %f - %f = %f\n", stats->robustLQ, stats->robustMedian, stats->robustUQ, stats->robustUQ + stats->robustLQ - 2*stats->robustMedian); + // XXX this operation is not thread safe -- move out somehow... + // psVectorAppend (dQ, stats->robustUQ + stats->robustLQ - 2*stats->robustMedian); + dQvalue = stats->robustUQ + stats->robustLQ - 2*stats->robustMedian; + // return dQvalue to main thread + + // supply sample to plotting routing + // only allow in a non-threaded context -- can plot more than one cell + // psphotDiagnosticPlots (config, "IMAGE.BACKGROUND.CELL.HISTOGRAM", ix, iy, modelData[iy][ix], modelStdevData[iy][ix], sample); + } else { + // psStatsOptions currentOptions = stats->options; + stats->options = PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV; + if (!psImageBackground(stats, &sample, subset, submask, maskVal, rng)) { + if ((nFailures < 3) || (nFailures % 100 == 0)) { + psLogMsg ("psphot", PS_LOG_WARN, "Failed to estimate background using ROBUST_MEDIAN for " + "(%dx%d, (row0,col0) = (%d,%d)", + subset->numRows, subset->numCols, subset->row0, subset->col0); + } + nFailures ++; // static + modelData[iy][ix] = modelStdevData[iy][ix] = NAN; + } else { + modelData[iy][ix] = psStatsGetValue (stats, PS_STAT_ROBUST_MEDIAN); + modelStdevData[iy][ix] = psStatsGetValue(stats, PS_STAT_ROBUST_STDEV); + + // supply sample to plotting routing + // only allow in a non-threaded context -- can plot more than one cell + // psphotDiagnosticPlots (config, "IMAGE.BACKGROUND.CELL.HISTOGRAM", ix, iy, modelData[iy][ix], modelStdevData[iy][ix], sample); + } + // drop errors caused by psImageBackground failures + // NOTE : psStats raises errors when it cannot find a solution; this + // probably should not be errors but exit stats info only, but c'est la code + // XXX we probably should trap and exit on serious failures + psErrorClear(); + // stats->options = currentOptions; // this is not needed (set by init above) + } + psFree (stats); + psFree (sample); + psFree (subset); + psFree (submask); + + // return the dQvalue to the calling thread + psScalar *scalar = job->args->data[14]; + scalar->data.F32 = dQvalue; + + return true; +} + +// load a background model for readout number index; save an associated pmFPAfile for possible output +bool psphotLoadBackgroundModelReadoutFileIndex (pmConfig *config, const pmFPAview *view, const char *filename, int index) +{ + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filename, index); // File of interest + psAssert (file, "missing file?"); + + pmFPA *inFPA = file->fpa; + pmReadout *readout = pmFPAviewThisReadout(view, inFPA); + psAssert (readout, "missing readout?"); + + // Set up the background model file + // Here we are assuming that the recipe being used is the same as when the original analysis that generated + // the background model was done + psImageBinning *binning = psphotBackgroundBinning(readout->image, config); // Image binning parameters + psMetadataAddPtr(readout->analysis, PS_LIST_TAIL, "PSPHOT.BACKGROUND.BINNING", PS_DATA_UNKNOWN | PS_META_REPLACE, "Background binning", binning); + psFree(binning); + pmReadout *model = pmFPAGenerateReadout(config, view, psphotGetFilerule("PSPHOT.BACKMDL"), inFPA, binning, index); + pmReadout *modelStdev = pmFPAGenerateReadout(config, view, psphotGetFilerule("PSPHOT.BACKMDL.STDEV"), inFPA, binning, index); + + // Copy the image from the input model file to the model file + // XXX: Check for nulls and that the image sizes match + pmFPAfile *bgFile = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.BACKMDL.RAW", 0); // File with bg + pmChip *bgChip = pmFPAviewThisChip(view, bgFile->fpa); + pmReadout *bgReadout = pmFPAviewThisReadout(view, bgChip->parent); + + // Copy the image from the input model readout to the model readout + model->image = psImageCopy(model->image, bgReadout->image, model->image->type.type); + // For now also copy the input into the stdev image. Is this used for anything? + modelStdev->image = psImageCopy(modelStdev->image, bgReadout->image, modelStdev->image->type.type); + + return true; +} + +bool psphotLoadBackgroundModel (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + int num = psphotFileruleCount(config, filerule); + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Load Background Model ---"); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotLoadBackgroundModelReadoutFileIndex(config, view, filerule, i)) { + int lastError = psErrorCodeLast(); + psError (lastError ? lastError : PSPHOT_ERR_CONFIG, false, "failed to model background for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// code for in-line plotting from above +// turn on stats tracing in desired cells +// XXX this code may need to be re-worked for a threaded context +# if (0) + psMetadata *plots = psMetadataLookupPtr (&status, recipe, "DIAGNOSTIC.PLOTS"); + assert (plots); + + int xPlot = psMetadataLookupS32 (&status, plots, "IMAGE.BACKGROUND.CELL.HISTOGRAM.X"); + assert (status); + int yPlot = psMetadataLookupS32 (&status, plots, "IMAGE.BACKGROUND.CELL.HISTOGRAM.Y"); + assert (status); + + bool gotX = (xPlot < 0) || (xPlot == ix); + bool gotY = (yPlot < 0) || (yPlot == iy); + + if (gotX && gotY) { + (void) psTraceSetLevel ("psLib.math.vectorFittedStats", 6); + (void) psTraceSetLevel ("psLib.math.vectorRobustStats", 6); + } else { + (void) psTraceSetLevel ("psLib.math.vectorFittedStats", 0); + (void) psTraceSetLevel ("psLib.math.vectorRobustStats", 0); + } +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelGroupInit.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelGroupInit.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelGroupInit.c (revision 41083) @@ -0,0 +1,25 @@ +# include "psphotInternal.h" + +// Add locally-defined models here. As these mature, they can be moved to +// psModule/src/objects/models + +// # include "models/pmModel_TEST1.c" +// # include "models/pmModel_STRAIL.c" + +static pmModelClass userModels[] = { + // {"PS_MODEL_TEST1", 7, pmModelFunc_TEST1, pmModelFlux_TEST1, pmModelRadius_TEST1, pmModelLimits_TEST1, pmModelGuess_TEST1, pmModelFromPSF_TEST1, pmModelParamsFromPSF_TEST1, pmModelFitStatus_TEST1, NULL}, + // {"PS_MODEL_STRAIL", 9, pmModelFunc_STRAIL, pmModelFlux_STRAIL, pmModelRadius_STRAIL, pmModelLimits_STRAIL, pmModelGuess_STRAIL, pmModelFromPSF_STRAIL, pmModelParamsFromPSF_STRAIL, pmModelFitStatus_STRAIL, NULL}, +}; + +void psphotModelClassInit (void) +{ + + // if pmModelClassInit returns false, we have already init'ed + if (!pmModelClassInit ()) return; + + int Nmodels = sizeof (userModels) / sizeof (pmModelClass); + for (int i = 0; i < Nmodels; i++) { + pmModelClassAdd (&userModels[i]); + } + return; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTest.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTest.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTest.c (revision 41083) @@ -0,0 +1,36 @@ +# include "psphotStandAlone.h" + +int main (int argc, char **argv) { + + psMemInit(); + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotModelTestArguments (argc, argv); + assert(config); + + psphotVersionPrint(); + + // load input data (config and images (signal, noise, mask) + if (!psphotParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotImageLoop (config, PSPHOT_MODEL_TEST)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", 3, "complete psphot run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTestArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTestArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTestArguments.c (revision 41083) @@ -0,0 +1,197 @@ +# include "psphotStandAlone.h" + +static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile) +{ + fprintf(ofile, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "%s -help\n" + "\n" + "%s -version\n" + "\n" + "where:\n" + " FileNameList is a text file containing filenames, one per line\n" + " MaskFileNameList is a text file of mask filenames, one per line\n" + " VarFileNameList is a text file of variance filenames, one per line\n" + " OutFileBaseName is the 'root name' for output files\n" + "\n" + "additional options:\n" + " -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n" + " specify PSF rather than letting %s estimate it\n" + " -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n" + " specify additional sources for PSF generation\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -photcode PhotoCodeName\n" + " specify photocode\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -nthreads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n", + program,program,program,program,program); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: one of the following\n" + "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n" + " -variance varfile1[,varfile2,...] OutFileBaseName\n" + "\n" + "%s -list FileNameList [-masklist MaskFileNameList] \n" + " -variancelist VarFileNameList OutFileBaseName\n" + "\n" + "Try '%s -help' for more options and explanation\n", + program,program,program); + if (exitCode != PS_EXIT_SUCCESS) + psErrorStackPrint(stderr, "Error reading arguments\n"); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + +pmConfig *psphotModelTestArguments(int argc, char **argv) { + + int N; + bool status; + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + // Number of threads is handled + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + // visual : interactive display mode + if ((N = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (N, &argc, argv); + pmVisualSetVisual(true); + } + + // break : used from recipe throughout psphotReadout + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF + if ((N = psArgumentGet (argc, argv, "-region"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-chip"))) { + if (argc<=N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + + // chip selection is used to limit chips to be processed + if ((N = psArgumentGet (argc, argv, "-moments-radius"))) { + psArgumentRemove (N, &argc, argv); + psMetadataAddBool (config->arguments, PS_LIST_TAIL, "TEST_MOMENTS_RADIUS", PS_DATA_BOOL, "", true); + } + + // if these command-line options are supplied, load the file name lists into config->arguments + // override any configuration-specified source for these files + // + pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); + + // an input list of sources to use is allowed, but not required + pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC", "-src", "-srclist"); + pmConfigFileSetsMD (config->arguments, &argc, argv, "SRCTEXT", "-srctext", "-srctextlist"); + + pmConfigFileSetsMD (config->arguments, &argc, argv, "PSPHOT.PSF", "-psf", "-psflist"); + + if (argc == 1) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (psArgumentGet(argc, argv, "-help") || + psArgumentGet(argc, argv, "-h")) + writeHelpInfo(argv[0], config, stdout); + + // the input file is a required argument; if not found, we will exit + status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + if (!status) { + psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments"); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotModelTestArguments...\n"); + return (config); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTestReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTestReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelTestReadout.c (revision 41083) @@ -0,0 +1,298 @@ +# include "psphotInternal.h" +# define PM_SOURCE_FIT_PSF_X_EXT PM_SOURCE_FIT_PSF_AND_SKY + +bool psphotModelTestReadout (pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status; + + psTimerStart ("modelTest"); + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + // remove cruft from the input analysis structure + if (!psphotCleanInputs (config, view, filerule)) { + psError (PSPHOT_ERR_PROG, false, "trouble setting up the inputs"); + return false; + } + + // set the photcode for this image + if (!psphotAddPhotcode (config, view, filerule)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images, including the user-defined analysis region of interest + if (!psphotSetMaskAndVariance (config, view, filerule)) { + return psphotReadoutCleanup(config, view, filerule); + } + + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + + if (!psphotSubtractBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + + // load the psf model, if suppled. FWHM_MAJ,FWHM_MIN,etc are determined and saved on + // readout->analysis. NOTE: this function currently only loads from PSPHOT.PSF.LOAD + if (!psphotLoadPSF (config, view, filerule)) { // ??? need to supply 2 ? + psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + float MIN_KRON_RADIUS = 5.0; + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // what fraction of the PSF is used? (radius in pixels : 2 -> 5x5 box) + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // use poissonian errors or local-sky errors + bool POISSON_ERRORS = psMetadataLookupBool (&status, recipe, filerule); + if (!status) POISSON_ERRORS = true; + + // find the various fitting parameters (try test values first) + float INNER = psMetadataLookupF32 (&status, recipe, "TEST_FIT_INNER_RADIUS"); + if (!status || !isfinite(INNER)) { + INNER = psMetadataLookupF32 (&status, recipe, "SKY_INNER_RADIUS"); + } + float OUTER = psMetadataLookupF32 (&status, recipe, "TEST_FIT_OUTER_RADIUS"); + if (!status || !isfinite(OUTER)) { + OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + } + float RADIUS = psMetadataLookupF32 (&status, recipe, "TEST_FIT_RADIUS"); + if (!status || !isfinite(RADIUS)) { + RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_FIT_RADIUS"); + } + float mRADIUS = psMetadataLookupF32 (&status, recipe, "TEST_MOMENTS_RADIUS"); + if (!status || !isfinite(mRADIUS)) { + mRADIUS = psMetadataLookupF32 (&status, recipe, "PSF_MOMENTS_RADIUS"); + } + + // define the source of interest + float xObj = psMetadataLookupF32 (&status, recipe, "TEST_FIT_X"); + float yObj = psMetadataLookupF32 (&status, recipe, "TEST_FIT_Y"); + if (!isfinite(xObj) || !isfinite(yObj)) psAbort ("object position is not defined"); + + // construct the source structures + pmSource *source = pmSourceAlloc(); + source->peak = pmPeakAlloc (xObj, yObj, 0, 0); + pmSourceDefinePixels (source, readout, xObj, yObj, OUTER); + + // find the model: supplied by user or first in the PSF_MODEL list + char *modelName = psMetadataLookupStr (&status, recipe, "TEST_FIT_MODEL"); + int modelType = pmModelClassGetType (modelName); + if (modelType < 0) psAbort("unknown model %s", modelName); + source->type = PM_SOURCE_TYPE_EXTENDED; + + bool TEST_FIT_CONVOLVED = psMetadataLookupBool (&status, recipe, "TEST_FIT_CONVOLVED"); + + // find the local sky + status = pmSourceLocalSky (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal); + if (!status) psAbort("pmSourceLocalSky error"); + + bool testMomentsRadius = psMetadataLookupBool (&status, config->arguments, "TEST_MOMENTS_RADIUS"); + if (testMomentsRadius) { + // XXX I want to test an iterative aperture for brighter sources + float radius = mRADIUS; + for (int i = 0; i < 10; i++) { + + // get the source moments + status = pmSourceMoments (source, radius, 0.25*radius, 0.0, MIN_KRON_RADIUS, maskVal); + if (!status) psAbort("psSourceMoments error"); + + float oldRadius = radius; + radius = source->moments->Mrf * RADIUS; + + fprintf (stderr, "%d %f %f %f\n", i, oldRadius, radius, source->moments->Mrf); + } + exit (0); + } + + // get the source moments + status = pmSourceMoments (source, mRADIUS, 0.25*mRADIUS, 0.0, MIN_KRON_RADIUS, maskVal); + if (!status) psAbort("psSourceMoments error"); + + fprintf (stderr, "sum: %f @ (%f, %f)\n", source->moments->Sum, source->moments->Mx, source->moments->My); + fprintf (stderr, "moments: %f, %f - %f\n", source->moments->Mxx, source->moments->Myy, source->moments->Mxy); + + psEllipseMoments moments; + moments.x2 = source->moments->Mxx; + moments.y2 = source->moments->Myy; + moments.xy = source->moments->Mxy; + psEllipseAxes axes = psEllipseMomentsToAxes (moments, 20.0); + + fprintf (stderr, "axes: %f @ (%f, %f)\n", axes.theta*180/M_PI, axes.major, axes.minor); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + if (psf) { + // set PSF parameters for this model (apply 2D shape model to coordinates Xo, Yo) + source->peak->rawFlux = source->moments->Peak; + float Io = source->moments->Peak; + source->modelPSF = pmModelFromPSFforXY(psf, xObj, yObj, Io); + } + + // get the initial model parameter guess + pmModel *model = pmSourceModelGuess (source, modelType, maskVal, markVal); + if (!model) { + fprintf (stderr, "failed to generate model guess\n"); + exit (2); + } + + source->modelEXT = model; + + // if any parameters are defined by the user, take those values + int nParams = pmModelClassParameterCount (modelType); + psF32 *params = model->params->data.F32; + params[PM_PAR_XPOS] = xObj; // XXX use the user-supplied value, + params[PM_PAR_YPOS] = yObj; // XXX or use the centroid + for (int i = 0; i < nParams; i++) { + if (i == PM_PAR_XPOS) continue; + if (i == PM_PAR_YPOS) continue; + + char name[32]; + sprintf (name, "TEST_FIT_PAR%d", i); + float value = psMetadataLookupF32 (&status, recipe, name); + if (status && isfinite (value)) { + params[i] = value; + } + } + + float area = params[4]*params[5]; + fprintf (stderr, "peak: %f @ (%f, %f)\n", source->moments->Sum*area, (double)source->peak->x, (double)source->peak->y); + + if (modelType == pmModelClassGetType("PS_MODEL_TRAIL")) { + fprintf (stderr, "guess: %f @ (%f, %f)\n", params[6]*180/M_PI, params[4], params[5]); + } else { + bool useReff = pmModelUseReff (modelType); + pmModelParamsToAxes (&axes, params[PM_PAR_SXX], params[PM_PAR_SXY], params[PM_PAR_SYY], useReff); + fprintf (stderr, "guess: %f @ (%f, %f) : %f\n", axes.theta*180/M_PI, axes.major, axes.minor, params[PM_PAR_SXY]); + } + + fprintf (stderr, "input parameters: \n"); + for (int i = 0; i < nParams; i++) { + fprintf (stderr, "%d : %f\n", i, params[i]); + } + + // define the pixels used for the fit + psImageKeepCircle (source->maskObj, xObj, yObj, RADIUS, "OR", markVal); + psphotSaveImage (NULL, source->maskObj, "mask1.fits"); + + float SKY_SIG = psMetadataLookupF32(&status, readout->analysis, "SKY_STDEV"); + + // options which modify the behavior of the model fitting + pmSourceFitOptions *fitOptions = pmSourceFitOptionsAlloc(); + fitOptions->nIter = psMetadataLookupS32(&status, recipe, "PSF_FIT_ITER"); // Maximum number of fit iterations + fitOptions->minTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_MIN_TOL"); // Fit tolerance + fitOptions->maxTol = psMetadataLookupF32 (&status, recipe, "PSF_FIT_MAX_TOL"); // Fit tolerance + fitOptions->maxChisqDOF = psMetadataLookupF32 (&status, recipe, "PSF_FIT_MAX_CHISQ"); // Fit tolerance + fitOptions->poissonErrors = POISSON_ERRORS; + fitOptions->weight = PS_SQR(SKY_SIG); + fitOptions->mode = PM_SOURCE_FIT_EXT; + fitOptions->covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + + bool chisqConvergence = psMetadataLookupBool (&status, recipe, "LMM_FIT_CHISQ_CONVERGENCE"); // Fit tolerance + if (!status) chisqConvergence = true; + fitOptions->chisqConvergence = chisqConvergence; + + int gainFactorMode = psMetadataLookupS32 (&status, recipe, "LMM_FIT_GAIN_FACTOR_MODE"); // Fit tolerance + if (!status) gainFactorMode = 0; + fitOptions->gainFactorMode = gainFactorMode; + + if (modelType == pmModelClassGetType("PS_MODEL_SERSIC")) { + fitOptions->mode = PM_SOURCE_FIT_NO_INDEX; + } + if (modelType == pmModelClassGetType("PS_MODEL_TRAIL")) { + fitOptions->mode = PM_SOURCE_FIT_TRAIL; + } + + if (TEST_FIT_CONVOLVED) { + pmPCMdata *pcm = pmPCMinit (source, fitOptions, model, maskVal, psfSize); + if (modelType == pmModelClassGetType("PS_MODEL_SERSIC")) { + psphotSersicModelClassGuessPCM (pcm, source); + } else { + pmSourceModelGuessPCM (pcm, source, maskVal, markVal); + } + + // if we provide a test guess value we want to use that value! + for (int i = 0; i < nParams; i++) { + if (i == PM_PAR_XPOS) continue; + if (i == PM_PAR_YPOS) continue; + + char name[32]; + sprintf (name, "TEST_FIT_PAR%d", i); + float value = psMetadataLookupF32 (&status, recipe, name); + if (status && isfinite (value)) { + params[i] = value; + } + } + + pmPCMupdate(pcm, source, fitOptions, model); + pmSourceFitPCM (pcm, source, fitOptions, maskVal, markVal, psfSize); + psFree (pcm); + } else { + status = pmSourceFitModel (source, model, fitOptions, maskVal); + } + + // measure the source mags + float fitMag = NAN; + float fitFlux = NAN; + float obsMag = NAN; + pmSourcePhotometryModel (&fitMag, &fitFlux, model); + pmSourcePhotometryAper (NULL, &obsMag, NULL, NULL, model, source->pixels, source->variance, source->maskObj, maskVal); + fprintf (stderr, "ap: %f, fit: %f, apmifit: %f, nIter: %d\n", obsMag, fitMag, obsMag - fitMag, model->nIter); + + // write out positive object + psphotSaveImage (NULL, source->pixels, "object.fits"); + + // subtract object, leave local sky + // pmModelSub (source->pixels, source->maskObj, model, PM_MODEL_OP_FULL, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + fprintf (stderr, "output parameters: \n"); + for (int i = 0; i < nParams; i++) { + fprintf (stderr, "%d : %f\n", i, params[i]); + } + + if (modelType == pmModelClassGetType("PS_MODEL_TRAIL")) { + fprintf (stderr, "result: %f @ (%f, %f)\n", params[6]*180/M_PI, params[4], params[5]); + } else { + bool useReff = pmModelUseReff (modelType); + pmModelParamsToAxes (&axes, params[PM_PAR_SXX], params[PM_PAR_SXY], params[PM_PAR_SYY], useReff); + fprintf (stderr, "result: %f @ (%f, %f) : %f\n", axes.theta*180/M_PI, axes.major, axes.minor, params[PM_PAR_SXY]); + } + + // write out + psphotSaveImage (NULL, source->pixels, "resid.fits"); + psphotSaveImage (NULL, source->maskObj, "mask.fits"); + + psLogMsg ("psphot", PS_LOG_INFO, "model test : %f sec\n", psTimerMark ("modelTest")); + + exit (0); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelWithPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelWithPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotModelWithPSF.c (revision 41083) @@ -0,0 +1,417 @@ +# include "psphotInternal.h" +# define SAVE_IMAGES 0 + +bool psphotModelWithPSF_LMM ( + psMinimization *min, + psImage *covar, + psVector *params, + psMinConstraint *constraint, + pmSource *source, + const psKernel *psf, + psMinimizeLMChi2Func func) +{ + psTrace("psphot", 3, "---- begin ----\n"); + PS_ASSERT_PTR_NON_NULL(min, false); + PS_ASSERT_VECTOR_NON_NULL(params, false); + PS_ASSERT_VECTOR_NON_EMPTY(params, false); + PS_ASSERT_VECTOR_TYPE(params, PS_TYPE_F32, false); + psVector *paramMask = NULL; + if (constraint != NULL) { + paramMask = constraint->paramMask; + if (paramMask != NULL) { + PS_ASSERT_VECTOR_TYPE(paramMask, PS_TYPE_VECTOR_MASK, false); + PS_ASSERT_VECTORS_SIZE_EQUAL(params, paramMask, false); + } + } + PS_ASSERT_PTR_NON_NULL(func, false); + PS_ASSERT_PTR_NON_NULL(source, false); + + psMinimizeLMLimitFunc checkLimits = NULL; + if (constraint) { + checkLimits = constraint->checkLimits; + } + + // this function has test values and current values for several things + // the current value is in lower case + // the test value is in upper case + + // allocate internal arrays (current vs Guess) + psImage *Alpha = NULL; + psVector *Beta = NULL; + + // Alpha & Beta only contain elements to represent the unmasked parameters + if (!psMinLM_AllocAB (&Alpha, &Beta, params, paramMask)) { + psAbort ("programming error: no unmasked parameters to be fit\n"); + } + + // allocate internal arrays (current vs Guess) + psImage *alpha = psImageAlloc(Alpha->numCols, Alpha->numRows, PS_TYPE_F32); + psVector *beta = psVectorAlloc(Beta->n, PS_TYPE_F32); + psVector *Params = psVectorAlloc(params->n, PS_TYPE_F32); + + psF32 Chisq = 0.0; + psF32 lambda = 0.001; + psF32 dLinear = 0.0; + + // generate PCM data storage structure + pmPCMData *pcm = pmPCMDataAlloc (params, paramMask, source); + + // calculate initial alpha and beta, set chisq (min->value) + min->value = psphotModelWithPSF_SetABX(alpha, beta, params, paramMask, pcm, source, psf, func); + if (isnan(min->value)) { + min->iter = min->maxIter; + return(false); + } + // dump some useful info if trace is defined + if (psTraceGetLevel("psphot") >= 6) { + p_psImagePrint(psTraceGetDestination(), alpha, "alpha guess (0)"); + p_psVectorPrint(psTraceGetDestination(), beta, "beta guess (0)"); + } + if (psTraceGetLevel("psphot") >= 5) { + p_psVectorPrint(psTraceGetDestination(), params, "params guess (0)"); + } + + // iterate until the tolerance is reached, or give up + while ((min->iter < min->maxIter) && ((min->lastDelta > min->minTol) || !isfinite(min->lastDelta))) { + psTrace("psphot", 5, "Iteration number %d. (max iterations is %d).\n", min->iter, min->maxIter); + psTrace("psphot", 5, "Last delta is %f. Min->minTol is %f.\n", min->lastDelta, min->minTol); + + + // set a new guess for Alpha, Beta, Params + if (!psMinLM_GuessABP(Alpha, Beta, Params, alpha, beta, params, paramMask, checkLimits, lambda, &dLinear)) { + min->iter ++; + lambda *= 10.0; + continue; + } + + // dump some useful info if trace is defined + if (psTraceGetLevel("psphot") >= 6) { + p_psImagePrint(psTraceGetDestination(), Alpha, "Alpha guess (1)"); + p_psVectorPrint(psTraceGetDestination(), Beta, "Beta guess (1)"); + p_psVectorPrint(psTraceGetDestination(), beta, "beta current (1)"); + } + if (psTraceGetLevel("psphot") >= 5) { + p_psVectorPrint(psTraceGetDestination(), Params, "params guess (1)"); + } + + // calculate Chisq for new guess, update Alpha & Beta + Chisq = psphotModelWithPSF_SetABX(Alpha, Beta, Params, paramMask, pcm, source, psf, func); + if (isnan(Chisq)) { + min->iter ++; + lambda *= 10.0; + continue; + } + + // convergence criterion: + // compare the delta (min->value - Chisq) with the + // expected delta from the linear model (dLinear) + // accept new guess if it is an improvement (rho > 0), or else increase lambda + psF32 rho = (min->value - Chisq) / dLinear; + + psTrace("psphot", 5, "last chisq: %f, new chisq %f, delta: %f, rho: %f\n", min->value, + Chisq, min->lastDelta, rho); + + // dump some useful info if trace is defined + if (psTraceGetLevel("psphot") >= 6) { + p_psImagePrint(psTraceGetDestination(), Alpha, "alpha guess (2)"); + p_psVectorPrint(psTraceGetDestination(), Beta, "beta guess (2)"); + } + + /* if (Chisq < min->value) { */ + if (rho > 0.0) { + min->lastDelta = (min->value - Chisq) / (source->pixels->numCols*source->pixels->numRows - params->n); + min->value = Chisq; + alpha = psImageCopy(alpha, Alpha, PS_TYPE_F32); + beta = psVectorCopy(beta, Beta, PS_TYPE_F32); + params = psVectorCopy(params, Params, PS_TYPE_F32); + lambda *= 0.25; + + // save the new convolved model image + psFree (source->modelFlux); + source->modelFlux = pmPCMDataSaveImage(pcm); + } else { + lambda *= 10.0; + } + min->iter++; + } + psTrace("psphot", 5, "chisq: %f, last delta: %f, Niter: %d\n", min->value, min->lastDelta, min->iter); + + // construct & return the covariance matrix (if requested) + if (covar != NULL) { + if (!psMinLM_GuessABP(Alpha, Beta, Params, alpha, beta, params, paramMask, NULL, 0.0, NULL)) { + psTrace ("psphot", 5, "failure to calculate covariance matrix\n"); + } + // set covar values which are not masked + psImageInit (covar, 0.0); + for (int j = 0, J = 0; j < params->n; j++) { + if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) { + covar->data.F32[j][j] = 1.0; + continue; + } + for (int k = 0, K = 0; k < params->n; k++) { + if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[k])) continue; + covar->data.F32[j][k] = Alpha->data.F32[J][K]; + K++; + } + J++; + } + } + + // free the internal temporary data + psFree(alpha); + psFree(Alpha); + psFree(beta); + psFree(Beta); + psFree(Params); + psFree(pcm); + + // if the last improvement was at least as good as maxTol, accept the fit: + if (min->lastDelta <= min->maxTol) { + psTrace("psphot", 6, "---- end (true) ----\n"); + return(true); + } + psTrace("psphot", 6, "---- end (false) ----\n"); + return(false); +} + +psF32 psphotModelWithPSF_SetABX( + psImage *alpha, + psVector *beta, + const psVector *params, + const psVector *paramMask, + pmPCMData *pcm, + const pmSource *source, + const psKernel *psf, + psMinimizeLMChi2Func func) +{ + // XXX: Check vector sizes. + PS_ASSERT_IMAGE_NON_NULL(alpha, NAN); + PS_ASSERT_VECTOR_NON_NULL(beta, NAN); + PS_ASSERT_VECTOR_NON_NULL(params, NAN); + + PS_ASSERT_PTR_NON_NULL(source, NAN); + PS_ASSERT_IMAGE_NON_NULL(source->pixels, NAN); + PS_ASSERT_IMAGE_NON_NULL(source->variance, NAN); + PS_ASSERT_IMAGE_NON_NULL(source->maskObj, NAN); + + PS_ASSERT_VECTOR_TYPE(params, PS_TYPE_F32, false); + if (paramMask) { + PS_ASSERT_VECTOR_TYPE(paramMask, PS_TYPE_VECTOR_MASK, false); + } + + // 1 *** generate the model and derivative images for this parameter set + + // storage for model derivatives + psVector *deriv = psVectorAlloc(params->n, PS_TYPE_F32); + + // working vector to store local coordinate + psVector *coord = psVectorAlloc(2, PS_TYPE_F32); + + psImageInit (pcm->model, 0.0); + for (int n = 0; n < params->n; n++) { + if (!pcm->dmodels->data[n]) continue; + psImageInit (pcm->dmodels->data[n], 0.0); + } + + // fill in the coordinate and value entries + for (psS32 i = 0; i < source->pixels->numRows; i++) { + for (psS32 j = 0; j < source->pixels->numCols; j++) { + + // XXX can we skip some of the data points where the model + // is not going to be fitted?? + + // skip masked points + // XXX probably should not skipped masked points: + // XXX skip if convolution of unmasked pixels will not see this pixel + // if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[i][j]) { + // continue; + // } + + // skip zero-variance points + // XXX why is this not masked? + // if (source->variance->data.F32[i][j] == 0) { + // continue; + // } + // skip nan value points + // XXX why is this not masked? + // if (!isfinite(source->pixels->data.F32[i][j])) { + // continue; + // } + + // Convert i/j to image space: + coord->data.F32[0] = (psF32) (j + source->pixels->col0); + coord->data.F32[1] = (psF32) (i + source->pixels->row0); + + pcm->model->data.F32[i][j] = func (deriv, params, coord); + + for (int n = 0; n < params->n; n++) { + if ((paramMask != NULL) && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[n])) { continue; } + psImage *dmodel = pcm->dmodels->data[n]; + dmodel->data.F32[i][j] = deriv->data.F32[n]; + } + } + } + psFree(coord); + psFree(deriv); + + // convolve model and dmodel arrays with PSF + psImageConvolveDirect (pcm->modelConv, pcm->model, psf); + for (int n = 0; n < pcm->dmodels->n; n++) { + if (pcm->dmodels->data[n] == NULL) continue; + psImage *dmodel = pcm->dmodels->data[n]; + psImage *dmodelConv = pcm->dmodelsConv->data[n]; + psImageConvolveDirect (dmodelConv, dmodel, psf); + } + + // XXX TEST : SAVE IMAGES +# if (SAVE_IMAGES) + psphotSaveImage (NULL, psf->image, "psf.fits"); + psphotSaveImage (NULL, pcm->model, "model.fits"); + psphotSaveImage (NULL, pcm->modelConv, "modelConv.fits"); + psphotSaveImage (NULL, source->pixels, "obj.fits"); + psphotSaveImage (NULL, source->maskObj, "mask.fits"); + psphotSaveImage (NULL, source->variance, "variance.fits"); +# endif + + // 2 *** accumulate alpha & beta + + // zero alpha and beta for summing below + psImageInit (alpha, 0.0); + psVectorInit (beta, 0.0); + float chisq = 0.0; + + for (psS32 i = 0; i < source->pixels->numRows; i++) { + for (psS32 j = 0; j < source->pixels->numCols; j++) { + // XXX are we doing the right thing with the mask? + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[i][j]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[i][j] == 0) { + continue; + } + // skip nan value points + if (!isfinite(source->pixels->data.F32[i][j])) { + continue; + } + + float ymodel = pcm->modelConv->data.F32[i][j]; + float yweight = 1.0 / source->variance->data.F32[i][j]; + float delta = ymodel - source->pixels->data.F32[i][j]; + + chisq += PS_SQR(delta) * yweight; + + if (isnan(delta)) psAbort("nan in delta"); + if (isnan(chisq)) psAbort("nan in chisq"); + + // alpha & beta only contain unmasked elements + for (int n1 = 0, N1 = 0; n1 < params->n; n1++) { + if ((paramMask != NULL) && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[n1])) continue; + psImage *dmodel = pcm->dmodelsConv->data[n1]; + float weight = dmodel->data.F32[i][j] * yweight; + for (int n2 = 0, N2 = 0; n2 <= n1; n2++) { + if ((paramMask != NULL) && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[n2])) continue; + dmodel = pcm->dmodelsConv->data[n2]; + alpha->data.F32[N1][N2] += weight * dmodel->data.F32[i][j]; + N2++; + } + beta->data.F32[N1] += weight * delta; + N1++; + } + } + } + + // calculate lower-left half of alpha + for (psS32 j = 1; j < alpha->numCols; j++) { + for (psS32 k = 0; k < j; k++) { + alpha->data.F32[k][j] = alpha->data.F32[j][k]; + } + } + + return(chisq); +} + +static void pmPCMDataFree (pmPCMData *pcm) { + + if (pcm == NULL) return; + + psFree (pcm->model); + psFree (pcm->modelConv); + psFree (pcm->dmodels); + psFree (pcm->dmodelsConv); + return; +} + +pmPCMData *pmPCMDataAlloc ( + const psVector *params, + const psVector *paramMask, + pmSource *source) { + + pmPCMData *pcm = (pmPCMData *) psAlloc(sizeof(pmPCMData)); + psMemSetDeallocator(pcm, (psFreeFunc) pmPCMDataFree); + + // Allocate storage images for raw model and derivative images + pcm->model = psImageCopy (NULL, source->pixels, PS_TYPE_F32); + pcm->dmodels = psArrayAlloc (params->n); + for (psS32 n = 0; n < params->n; n++) { + pcm->dmodels->data[n] = NULL; + if ((paramMask != NULL) && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[n])) { continue; } + pcm->dmodels->data[n] = psImageCopy (NULL, source->pixels, PS_TYPE_F32); + } + + // Allocate storage images for convolved model and derivative images + pcm->modelConv = psImageCopy (NULL, source->pixels, PS_TYPE_F32); + pcm->dmodelsConv = psArrayAlloc (params->n); + for (psS32 n = 0; n < params->n; n++) { + pcm->dmodelsConv->data[n] = NULL; + if ((paramMask != NULL) && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[n])) { continue; } + pcm->dmodelsConv->data[n] = psImageCopy (NULL, source->pixels, PS_TYPE_F32); + } + + return pcm; +} + +psImage *pmPCMDataSaveImage (pmPCMData *pcm) { + + psImage *model = psImageCopy (NULL, pcm->modelConv, PS_TYPE_F32); + + return model; +} + +/* + * + * we have a function func(param; value) + + * basic LMM: + + - fill in the data (x, y) + + chisq = SetABX (alpha, beta, params, paramMask, x, y, dy, func) + + while () { + GuessABP (Alpha, Beta, Params, alpha, beta, params, paramMask, checkLimits, lambda) + dLinear = dLinear(Beta, beta, lambda); + chisq = SetABX (alpha, beta, params, paramMask, x, y, dy, func) + convergence tests... + } + + + + ** GuessABP: + + f_c = sum_i (kern_i * func (x_i; p_o)) + + df_c/dp_o = d/dp_o [sum_i (kern_i * func (x_i; p_o))] + + df_c/dp_o = sum_i (d/dp_o [kern_i * func (x_i; p_o)]) + + df_c/dp_o = sum_i (kern_i * d/dp_o [func (x_i; p_o)]) + + - generate image arrays for func, dfunc/dp_j (not masked) + - convolve each with psf + - measure delta = f_conv - data + - etc +*/ + + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMomentsStudy.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMomentsStudy.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMomentsStudy.c (revision 41083) @@ -0,0 +1,171 @@ +# include "psphotInternal.h" + +int SetUpKapa (Graphdata *graphdata); +int PlotVectors (int kapa, Graphdata *graphdata, psVector *inVector, psVector *outVector); +int PlotModelSet (int kapa, Graphdata *graphdata, pmReadout *readout, char *name, float par7); + +FILE *output = NULL; + +int main (int argc, char **argv) { + + Graphdata graphdata; + int kapa; + + pmErrorRegister(); // register psModule's error codes/messages + pmModelClassInit(); + + // loop over sigma_window + // loop over sigma_input + // generate a fake source using the desired model + // add noise? + // measure the moments for the object + // accumulate Mxx, Myy + + // plot sigma_input vs sigma_output == sqrt(Mxx) + + // pmSourceMoments requires a source with a peak and pixels + + if (argc != 2) { + fprintf (stderr, "USAGE: psphotMomentsStudy (output)\n"); + exit (2); + } + + output = fopen (argv[1], "w"); + if (!output) exit (1); + fprintf (output, "# %4s %5s %6s %6s %6s\n", "type", "par7", "sigWin", "sigIn", "sigOut"); + + kapa = SetUpKapa(&graphdata); + + // create a containing image & associated readout + pmReadout *readout = pmReadoutAlloc(NULL); + readout->image = psImageAlloc(128, 128, PS_TYPE_F32); + + // create a dummy variance, but don't populate -- it is not used by pmSourceMoments if the sigma parameters is set to 0.0 + readout->variance = psImageAlloc(128, 128, PS_TYPE_F32); + + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_GAUSS", 0.0); + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_PGAUSS", 0.0); + + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_PS1_V1", 0.3); + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_PS1_V1", 1.0); + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_PS1_V1", 3.0); + + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_SERSIC", 0.125); + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_SERSIC", 0.250); + PlotModelSet (kapa, &graphdata, readout, "PS_MODEL_SERSIC", 0.500); + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[c]ontinue? "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + fclose (output); + + // psphotSaveImage(NULL, readout->image, "test.fits"); + exit (0); +} + +int SetUpKapa (Graphdata *graphdata) { + + int kapa = KapaOpenNamedSocket ("kapa", "psphot"); + + KapaClearPlots (kapa); + KapaInitGraph (graphdata); + KapaSetFont (kapa, "courier", 14); + + graphdata->color = KapaColorByName ("black"); + graphdata->xmin = -0.1; + graphdata->xmax = 1.6; + graphdata->ymin = -0.1; + graphdata->ymax = 1.6; + // graphdata->xmax = 10.1; + // graphdata->ymax = 10.1; + KapaSetLimits (kapa, graphdata); + + KapaBox (kapa, graphdata); + // KapaSendLabel (kapa, "&ss&h_in| (pixels)", KAPA_LABEL_XM); + // KapaSendLabel (kapa, "&ss&h_out| (pixels)", KAPA_LABEL_YM); + KapaSendLabel (kapa, "&ss&h_out| / &ss&h_window|", KAPA_LABEL_XM); + KapaSendLabel (kapa, "&ss&h_out| / &ss&h_in|", KAPA_LABEL_YM); + + return (kapa); +} + +int PlotVectors (int kapa, Graphdata *graphdata, psVector *inVector, psVector *outVector) { + + graphdata->color = KapaColorByName ("black"); + graphdata->ptype = 2; + graphdata->size = 1.0; + graphdata->style = 2; + KapaPrepPlot (kapa, inVector->n, graphdata); + KapaPlotVector (kapa, inVector->n, inVector->data.F32, "x"); + KapaPlotVector (kapa, inVector->n, outVector->data.F32, "y"); + + return 0; +} + + +int PlotModelSet (int kapa, Graphdata *graphdata, pmReadout *readout, char *name, float par7) { + + // create a model & associated source + // pmModelType type = pmModelClassGetType("PS_MODEL_GAUSS"); + + pmModelType type = pmModelClassGetType(name); + pmModel *model = pmModelAlloc(type); + + // set the model parameters + model->params->data.F32[PM_PAR_SKY] = 0.0; + model->params->data.F32[PM_PAR_I0] = 100.0; + model->params->data.F32[PM_PAR_XPOS] = 64.0; + model->params->data.F32[PM_PAR_YPOS] = 64.0; + + model->params->data.F32[PM_PAR_SXX] = 1.0 * M_SQRT2; + model->params->data.F32[PM_PAR_SYY] = 1.0 * M_SQRT2; + model->params->data.F32[PM_PAR_SXY] = 0.0; + + if (model->params->n > 7) { + model->params->data.F32[PM_PAR_7] = par7; + } + + pmSource *source = pmSourceFromModel(model, readout, 32.0, PM_SOURCE_TYPE_STAR); + + psVector *inVector = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *outVector = psVectorAllocEmpty(100, PS_TYPE_F32); + + for (float sigWindow = 1.0; sigWindow < 10.0; sigWindow += 1.0) { + + for (float sigIn = 0.5; sigIn < 10.0; sigIn += 0.1) { + + model->params->data.F32[PM_PAR_SXX] = sigIn * M_SQRT2; + model->params->data.F32[PM_PAR_SYY] = sigIn * M_SQRT2; + + // generate the modelFlux + pmSourceCacheModel(source, 0); + + // instantiate the source + pmSourceAdd(source, PM_MODEL_OP_FUNC, 0); + + pmSourceMoments (source, 32.0, sigWindow, 0.0, 0xffff); + // fprintf (stderr, "sigOut : %f\n", sqrt(source->moments->Mxx)); + + psVectorAppend(inVector, sqrt(source->moments->Mxx) / sigWindow); + psVectorAppend(outVector, sqrt(source->moments->Mxx) / sigIn); + + psImageInit (readout->image, 0.0); + + fprintf (output, " %4d %5.2f %6.3f %6.3f %6.3f\n", type, par7, sigWindow, sigIn, sqrt(source->moments->Mxx)); + } + } + + PlotVectors (kapa, graphdata, inVector, outVector); + inVector->n = 0; + outVector->n = 0; + + psFree (source); + psFree (inVector); + psFree (outVector); + + return 1; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMosaicChip.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMosaicChip.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMosaicChip.c (revision 41083) @@ -0,0 +1,38 @@ +# include "psphotInternal.h" + +bool psphotMosaicChip(pmConfig *config, const pmFPAview *view, char *outFile, char *inFile) +{ + bool status; // Status of MD lookup + + pmFPAfile *in = psMetadataLookupPtr(&status, config->files, inFile); // Input file + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + + pmFPAfile *out = psMetadataLookupPtr(&status, config->files, outFile); // Output file + if (!status) { + psErrorStackPrint(stderr, "Can't find required I/O file!\n"); + exit(EXIT_FAILURE); + } + + // XXXX we are failing to get the output hdu right + pmChip *outChip = pmFPAviewThisChip(view, out->fpa); + pmChip *inChip = pmFPAviewThisChip(view, in->fpa); + if (!outChip->hdu && !outChip->parent->hdu) { + pmFPAAddSourceFromView(out->fpa, view, out->format); + } + + psImageMaskType blankMask = pmConfigMaskGet("BLANK", config); + + // mosaic the chip, forcing a deep copy (resulting images are not subimages) + psTrace("pmChipMosaic", 5, "mosaic chip %s to %s (xbin,ybin: %d,%d to %d,%d)\n", + in->name, out->name, in->xBin, in->yBin, out->xBin, out->yBin); + status = pmChipMosaic(outChip, inChip, true, blankMask); + return status; +} + +// XXX does this do everything needed? +// * mask & weight +// * loaded PSF model (in readout->analysis) +// * loaded SRC sources (in readout->analysis) Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMosaicSubimage.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMosaicSubimage.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotMosaicSubimage.c (revision 41083) @@ -0,0 +1,55 @@ +# include "psphotInternal.h" + +// insert the source image into the outimage at Xo, Yo +bool psphotMosaicSubimage (psImage *outImage, pmSource *source, int Xo, int Yo, int DX, int DY, bool normalize) { + + psRegion inRegion, outRegion; + psImage *inImage = source->pixels; + + // identify the region in the output image + outRegion = psRegionSet (Xo, Xo + DX, Yo, Yo + DX); + outRegion = psRegionForImage (outImage, outRegion); + int DXo = outRegion.x1 - outRegion.x0; + int DYo = outRegion.y1 - outRegion.y0; + if (DXo <= 0) return false; + if (DYo <= 0) return false; + + // center the input source in the output box + int dX = (DXo - 1) / 2; + int dY = (DYo - 1) / 2; + + // int xo = inImage->col0 + inImage->numCols / 2; + // int yo = inImage->row0 + inImage->numRows / 2; + + int xo = source->peak->xf; + int yo = source->peak->yf; + + // adjust region to overlay input image pixels + inRegion = psRegionSet (xo - dX, xo + dX + 1, yo - dY, yo + dY + 1); + inRegion = psRegionForImage (inImage, inRegion); + + float peak = source->peak->rawFlux; + + psImage *subImage = psImageSubset (inImage, inRegion); + if (!subImage) { + psErrorClear(); // XXX I think there is an error in psphotVisual that is supplying bad images + return false; + } + + psImage *newImage = psImageAlloc (subImage->numCols, subImage->numRows, PS_TYPE_F32); + for (int iy = 0; iy < newImage->numRows; iy++) { + for (int ix = 0; ix < newImage->numCols; ix++) { + if (normalize) { + newImage->data.F32[iy][ix] = subImage->data.F32[iy][ix] / peak; + } else { + newImage->data.F32[iy][ix] = subImage->data.F32[iy][ix]; + } + } + } + + psImageOverlaySection (outImage, newImage, Xo, Yo, "="); + + psFree (subImage); + psFree (newImage); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotOldCode.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotOldCode.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotOldCode.c (revision 41083) @@ -0,0 +1,350 @@ + +/*************** + + In the process of developing psphot, I have written a bunch of code that has been used + for some tests or for a period, but which we no longer think is appropriate. I am + trying to clean up the kruft of old psphot bits and will but various such functions in + this file if they are worth keeping + +****************/ + +bool psphotMaskCosmicRayFootprintCheck (psArray *sources); + + +// maybe move this into psModules, pmFootprints? +bool psphotMaskCosmicRayFootprintCheck (psArray *sources) { +#ifdef CHECK_FOOTPRINTS + // This gets really expensive for complex images + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + pmPeak *peak = source->peak; + pmFootprint *footprint = peak->footprint; + if (!footprint) continue; + for (int j = 0; j < footprint->spans->n; j++) { + pmSpan *sp = footprint->spans->data[j]; + psAssert (sp, "missing span"); + } + } +#endif + return true; +} + +// mark the isophotal boundary +bool psphotMaskCosmicRayIsophot (pmSource *source, psImageMaskType maskVal, psImageMaskType crMask) { + + source->mode |= PM_SOURCE_MODE_CR_LIMIT; + pmPeak *peak = source->peak; + psAssert (peak, "NULL peak"); + + psImage *mask = source->maskView; + psImage *pixels = source->pixels; + psImage *variance = source->variance; + + // XXX This should be a recipe variable +# define SN_LIMIT 5.0 + + int xo = peak->x - pixels->col0; + int yo = peak->y - pixels->row0; + + // mark the pixels in this row to the left, then the right + for (int ix = xo; ix >= 0; ix--) { + float SN = pixels->data.F32[yo][ix] / sqrt(variance->data.F32[yo][ix]); + if (SN > SN_LIMIT) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[yo][ix] |= crMask; + } + } + for (int ix = xo + 1; ix < pixels->numCols; ix++) { + float SN = pixels->data.F32[yo][ix] / sqrt(variance->data.F32[yo][ix]); + if (SN > SN_LIMIT) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[yo][ix] |= crMask; + } + } + + // for each of the neighboring rows, mark the high pixels if they have a marked neighbor + // first go up: + for (int iy = PS_MIN(yo, mask->numRows-2); iy >= 0; iy--) { + // mark the pixels in this row to the left, then the right + for (int ix = 0; ix < pixels->numCols; ix++) { + float SN = pixels->data.F32[iy][ix] / sqrt(variance->data.F32[iy][ix]); + if (SN < SN_LIMIT) continue; + + bool valid = false; + valid |= (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix] & crMask); + valid |= (ix > 0) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix-1] & crMask) : 0; + valid |= (ix <= mask->numCols) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy+1][ix+1] & crMask) : 0; + + if (!valid) continue; + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; + } + } + // next go down: + for (int iy = PS_MIN(yo+1, mask->numRows-1); iy < pixels->numRows; iy++) { + // mark the pixels in this row to the left, then the right + for (int ix = 0; ix < pixels->numCols; ix++) { + float SN = pixels->data.F32[iy][ix] / sqrt(variance->data.F32[iy][ix]); + if (SN < SN_LIMIT) continue; + + bool valid = false; + valid |= (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix] & crMask); + valid |= (ix > 0) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix-1] & crMask) : 0; + valid |= (ix <= mask->numCols) ? (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy-1][ix+1] & crMask) : 0; + + if (!valid) continue; + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; + } + } + return true; +} + +// This attempt to mask the cosmic rays used the isophotal boundary +bool psphotMaskCosmicRay_V1 (psImage *mask, pmSource *source, psImageMaskType maskVal, psImageMaskType crMask) { + + // replace the source flux + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + // flag this as a CR + source->mode |= PM_SOURCE_MODE_CR_LIMIT; + pmPeak *peak = source->peak; + psAssert (peak, "NULL peak"); + + // grab the matching footprint + pmFootprint *footprint = peak->footprint; + if (!footprint) { + psTrace("psphot.czw",2,"Using isophot CR mask code."); + + // if we have not footprint, use the old code to mask by isophot + psphotMaskCosmicRayIsophot (source, maskVal, crMask); + return true; + } + + if (!footprint->spans) { + psTrace("psphot.czw",2,"Using isophot CR mask code."); + + // if we have no footprint, use the old code to mask by isophot + psphotMaskCosmicRayIsophot (source, maskVal, crMask); + return true; + } + psphotMaskCosmicRayIsophot (source, maskVal, crMask); + // mask all of the pixels covered by the spans of the footprint + for (int j = 1; j < footprint->spans->n; j++) { + pmSpan *span1 = footprint->spans->data[j]; + + int iy = span1->y; + int xs = span1->x0; + int xe = span1->x1; + + for (int ix = xs; ix < xe; ix++) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= crMask; + } + } + return true; +} + +// given the PSF ellipse parameters, navigate around the 1sigma contour, return the total +// deviation in sigmas. This is measured on the residual image - should we ignore negative +// deviations? NOTE: This function was an early attempt to classify extended objects, and is +// no longer used by psphot. +float psphotModelContour(const psImage *image, const psImage *variance, const psImage *mask, + psImageMaskType maskVal, const pmModel *model, float Ro) +{ + psF32 *PAR = model->params->data.F32; // Model parameters + float sxx = PAR[PM_PAR_SXX], sxy = PAR[PM_PAR_SXY], syy = PAR[PM_PAR_SYY]; // Ellipse parameters + + // We treat the contour as an ellipse: + // Ro = (x / SXX)^2 + (y / SYY)^2 + x y SXY + // y^2 (1/SYY^2) + y (x SXY) + (x / SXX)^2 - Ro = 0; + // This is a quadratic, Ay^2 + By + C with A = 1/SYY^2, B = x*SXY, C = (x / SXX)^2 - Ro + // The solution is y = [-B +/- sqrt (B^2 - 4 A C)] / [2 A], so: + // y = [-(x SXY) +/- sqrt ((x SXY)^2 - 4 (1/SYY^2) ((x/SXX)^2 - Ro))] * [SYY^2 / 2] + + // min/max value of x is where B^2 - 4AC = 0; solve this for x + float Q = Ro * PS_SQR(sxx) / (1.0 - PS_SQR(sxx * syy * sxy) / 4.0); + if (Q < 0.0) { + // ellipse is imaginary + return NAN; + } + + int radius = sqrtf(Q) + 0.5; // Radius of ellipse + int nPts = 0; // Number of points in ellipse + float nSigma = 0.0; // + + for (int x = -radius; x <= radius; x++) { + // Polynomial coefficients + // XXX Should we be using the centre of the pixel as x or x+0.5? + float A = PS_SQR (1.0 / syy); + float B = x * sxy; + float C = PS_SQR (x / sxx) - Ro; + float T = PS_SQR(B) - 4*A*C; + if (T < 0.0) { + continue; + } + + // y position in source frame + float yP = (-B + sqrt (T)) / (2.0 * A); + float yM = (-B - sqrt (T)) / (2.0 * A); + + // Get the closest pixel positions (image frame) + int xPix = x + PAR[PM_PAR_XPOS] - image->col0 + 0.5; + int yPixM = yM + PAR[PM_PAR_YPOS] - image->row0 + 0.5; + int yPixP = yP + PAR[PM_PAR_YPOS] - image->row0 + 0.5; + + if (xPix < 0 || xPix >= image->numCols) { + continue; + } + + if (yPixM >= 0 && yPixM < image->numRows && + !(mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[yPixM][xPix] & maskVal))) { + float dSigma = image->data.F32[yPixM][xPix] / sqrtf(variance->data.F32[yPixM][xPix]); + nSigma += dSigma; + nPts++; + } + + if (yPixM == yPixP) { + continue; + } + + if (yPixP >= 0 && yPixP < image->numRows && + !(mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[yPixP][xPix] & maskVal))) { + float dSigma = image->data.F32[yPixP][xPix] / sqrtf(variance->data.F32[yPixP][xPix]); + nSigma += dSigma; + nPts++; + } + } + nSigma /= nPts; + return nSigma; +} + +// this was an old attempt to identify cosmic rays based on the peak curvature +bool psphotSourcePeakCurvature (pmReadout *readout, psArray *sources, psphotSourceSizeOptions *options) { + + // classify the sources based on the CR test (place this in a function?) + // XXX use an internal flag to mark sources which have already been measured + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // skip source if it was already measured + if (source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) { + psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); + continue; + } + + // source must have been subtracted + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { + source->mode |= PM_SOURCE_MODE_SIZE_SKIPPED; + psTrace("psphot", 7, "Not calculating source size since source is not subtracted\n"); + continue; + } + + psF32 **resid = source->pixels->data.F32; + psF32 **variance = source->variance->data.F32; + psImageMaskType **mask = source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + // Integer position of peak + int xPeak = source->peak->xf - source->pixels->col0 + 0.5; + int yPeak = source->peak->yf - source->pixels->row0 + 0.5; + + // Skip sources which are too close to a boundary. These are mostly caught as DEFECT + if (xPeak < 1 || xPeak > source->pixels->numCols - 2 || + yPeak < 1 || yPeak > source->pixels->numRows - 2) { + psTrace("psphot", 7, "Not calculating crNsigma due to edge\n"); + continue; + } + + // Skip sources with masked pixels. These are mostly caught as DEFECT + bool keep = true; + for (int iy = -1; (iy <= +1) && keep; iy++) { + for (int ix = -1; (ix <= +1) && keep; ix++) { + if (mask[yPeak+iy][xPeak+ix] & options->maskVal) { + keep = false; + } + } + } + if (!keep) { + psTrace("psphot", 7, "Not calculating crNsigma due to masked pixels\n"); + continue; + } + + // Compare the central pixel with those on either side, for the four possible lines through it. + + // Soften variances (add systematic error) + float softening = options->soft * PS_SQR(source->peak->rawFlux); // Softening for variances + + // Across the middle: y = 0 + float cX = 2*resid[yPeak][xPeak] - resid[yPeak+0][xPeak-1] - resid[yPeak+0][xPeak+1]; + float dcX = 4*variance[yPeak][xPeak] + variance[yPeak+0][xPeak-1] + variance[yPeak+0][xPeak+1]; + float nX = cX / sqrtf(dcX + softening); + + // Up the centre: x = 0 + float cY = 2*resid[yPeak][xPeak] - resid[yPeak-1][xPeak+0] - resid[yPeak+1][xPeak+0]; + float dcY = 4*variance[yPeak][xPeak] + variance[yPeak-1][xPeak+0] + variance[yPeak+1][xPeak+0]; + float nY = cY / sqrtf(dcY + softening); + + // Diagonal: x = y + float cL = 2*resid[yPeak][xPeak] - resid[yPeak-1][xPeak-1] - resid[yPeak+1][xPeak+1]; + float dcL = 4*variance[yPeak][xPeak] + variance[yPeak-1][xPeak-1] + variance[yPeak+1][xPeak+1]; + float nL = cL / sqrtf(dcL + softening); + + // Diagonal: x = - y + float cR = 2*resid[yPeak][xPeak] - resid[yPeak+1][xPeak-1] - resid[yPeak-1][xPeak+1]; + float dcR = 4*variance[yPeak][xPeak] + variance[yPeak+1][xPeak-1] + variance[yPeak-1][xPeak+1]; + float nR = cR / sqrtf(dcR + softening); + + // P(chisq > chisq_obs; Ndof) = gamma_Q (Ndof/2, chisq/2) + // Ndof = 4 ? (four measurements, no free parameters) + // XXX this value is going to be biased low because of systematic errors. + // we need to calibrate it somehow + // source->psfProb = gsl_sf_gamma_inc_Q (2, 0.5*chisq); + + // not strictly accurate: overcounts the chisq contribution from the center pixel (by + // factor of 4); also biases a bit low if any pixels are masked + // XXX I am not sure I want to keep this value... + source->psfChisq = PS_SQR(nX) + PS_SQR(nY) + PS_SQR(nL) + PS_SQR(nR); + + float fCR = 0.0; + int nCR = 0; + if (nX > 0.0) { + fCR += nX; + nCR ++; + } + if (nY > 0.0) { + fCR += nY; + nCR ++; + } + if (nL > 0.0) { + fCR += nL; + nCR ++; + } + if (nR > 0.0) { + fCR += nR; + nCR ++; + } + source->crNsigma = (nCR > 0) ? fCR / nCR : 0.0; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + + if (!isfinite(source->crNsigma)) { + continue; + } + + // this source is thought to be a cosmic ray. flag the detection and mask the pixels + if (source->crNsigma > options->nSigmaCR) { + source->mode |= PM_SOURCE_MODE_CR_LIMIT; + // XXX still testing... : psphotMaskCosmicRay (readout->mask, source, maskVal, crMask); + // XXX acting strange... psphotMaskCosmicRay_Old (source, maskVal, crMask); + } + } + + // now that we have masked pixels associated with CRs, we can grow the mask + if (options->grow > 0) { + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolveMask + psImage *newMask = psImageConvolveMask(NULL, readout->mask, options->crMask, options->crMask, -options->grow, options->grow, -options->grow, options->grow); + psImageConvolveSetThreads(oldThreads); + if (!newMask) { + psError(PS_ERR_UNKNOWN, false, "Unable to grow CR mask"); + return false; + } + psFree(readout->mask); + readout->mask = newMask; + } + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotOutput.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotOutput.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotOutput.c (revision 41083) @@ -0,0 +1,522 @@ +# include "psphotInternal.h" + +// convert filerule to filerule.NUM and look up in the config->arguments metadata +int psphotFileruleCount(const pmConfig *config, const char *filerule) { + + bool status = false; + + psString name = NULL; + psStringAppend(&name, "%s.NUM", filerule); + int num = psMetadataLookupS32 (&status, config->arguments, name); + if (!status) { + // we only explicitly define (filerule.NUM) if we have more than 1 + num = 1; + } + psFree (name); + return num; +} + +// convert filerule to filerule.NUM and look up in the config->arguments metadata +bool psphotFileruleCountSet(const pmConfig *config, const char *filerule, int num) { + + psString name = NULL; + psStringAppend(&name, "%s.NUM", filerule); + + bool status = psMetadataAddS32(config->arguments, PS_LIST_TAIL, name, PS_META_REPLACE, "", num); + psFree (name); + + return status; +} + +pmReadout *psphotSelectBackground (pmConfig *config, const pmFPAview *view, int index) { + + pmFPAfile *file = pmFPAfileSelectSingle (config->files, psphotGetFilerule("PSPHOT.BACKMDL"), index); + if (!file) return NULL; + + pmReadout *background = READOUT_OR_INTERNAL(view, file); + return background; +} + +pmReadout *psphotSelectBackgroundStdev (pmConfig *config, const pmFPAview *view, int index) { + + pmFPAfile *file = pmFPAfileSelectSingle (config->files, psphotGetFilerule("PSPHOT.BACKMDL.STDEV"), index); + if (!file) return NULL; + + pmReadout *background = READOUT_OR_INTERNAL(view, file); + return background; +} + +// dump source stats for psf stars +bool psphotDumpStats (psArray *sources, char *stage) { + + char filename[64]; + snprintf (filename, 64, "psf.%s.dat", stage); + FILE *f = fopen (filename, "w"); + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + + pmModel *model = source->modelPSF; + if (!model) continue; + + // int xc = source->peak->x - source->pixels->col0; + // int yc = source->peak->y - source->pixels->row0; + // float mcore = source->modelFlux ? source->modelFlux->data.F32[yc][xc] : NAN; + // float mpeak = model ? model->params->data.F32[PM_PAR_I0] : NAN; + // bool subtracted = source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED; + // fprintf (stderr, "%d %d : %d : %f %f : %f %f\n", source->peak->x, source->peak->y, subtracted, source->peak->rawFlux, source->pixels->data.F32[yc][xc], mcore, mpeak); + + fprintf (f, "%6.1f %6.1f : %6.1f %6.1f : %8.3f %8.3f %8.3f : %f : %f %f %f : %f\n", + source->peak->xf, source->peak->yf, + model->params->data.F32[PM_PAR_XPOS], model->params->data.F32[PM_PAR_YPOS], + source->psfMag, source->apMag, source->psfMagErr, + model->params->data.F32[PM_PAR_I0], + model->params->data.F32[PM_PAR_SXX], model->params->data.F32[PM_PAR_SXY], model->params->data.F32[PM_PAR_SYY], + model->params->data.F32[PM_PAR_7]); + } + fclose (f); + return true; +} + +int psphotSaveImage (psMetadata *header, psImage *image, char *filename) { + + psFits *fits = psFitsOpen (filename, "w"); + psFitsWriteImage (fits, NULL, image, 0, NULL); + psFitsClose (fits); + return (TRUE); +} + +bool psphotDumpMoments (psMetadata *recipe, psArray *sources) { + + bool status; + + // optional dump of all rough source data + char *output = psMetadataLookupStr (&status, recipe, "MOMENTS_OUTPUT_FILE"); + if (!output) return false; + if (output[0] == 0) return false; + if (!strcasecmp (output, "NONE")) return false; + + pmMomentsWriteText (sources, output); + return true; +} + +bool psphotDumpSource (pmSource *source, char *name) { + + FILE *f = fopen (name, "w"); + if (f == NULL) psAbort("can't open file"); + + for (int i = 0; i < source->pixels->numRows; i++) { + for (int j = 0; j < source->pixels->numCols; j++) { + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[i][j]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[i][j] == 0) { + continue; + } + + fprintf (f, "%d %d %f %f %d\n", + (j + source->pixels->col0), + (i + source->pixels->row0), + source->pixels->data.F32[i][j], + 1.0 / source->variance->data.F32[i][j], + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[i][j]); + } + } + fclose (f); + return true; +} + +bool psphotCleanInputsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index); +bool psphotCleanInputs (pmConfig *config, const pmFPAview *view, const char *filerule) { + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotCleanInputsReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to clean inputs for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotCleanInputsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + PS_ASSERT (file, false); + + pmReadout *readout = pmFPAviewThisReadout (view, file->fpa); + + // XXX anything else to remove? + if (psMetadataLookup(readout->analysis, "PSPHOT.DETECTIONS")) { + psMetadataRemoveKey(readout->analysis, "PSPHOT.DETECTIONS"); + } + + return true; +} + +bool psphotAddPhotcode (pmConfig *config, const pmFPAview *view, const char *filerule) { + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotAddPhotcodeReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to add photcode to %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotAddPhotcodeReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + PS_ASSERT (file, false); + + pmReadout *readout = pmFPAviewThisReadout (view, file->fpa); + + // determine PHOTCODE from fpa & view, overwrite in readout->analysis + char *photcode = pmConceptsPhotcodeForView (file, view); + PS_ASSERT (photcode, false); + + psMetadataAddStr (readout->analysis, PS_LIST_TAIL, "PHOTCODE", PS_META_REPLACE, "photcode from FPA concepts", photcode); + psLogMsg ("psphot", 3, "PHOTCODE is %s", photcode); + + psFree (photcode); + return true; +} + +bool psphotSetHeaderNstars (psMetadata *header, psArray *sources) { + + int nSrc = 0; + int nCR = 0; + int nEXT = 0; + int nForced = 0; + int nDetections = sources != NULL ? sources->n : 0; + + // count the number of sources which will be written and other sub-types + for (int i = 0; (sources != NULL) && (i < sources->n); i++) { + pmSource *source = (pmSource *) sources->data[i]; + if (source->mode2 & PM_SOURCE_MODE2_MATCHED) { + nForced ++; + } + pmModel *model = pmSourceGetModel (NULL, source); + if (model == NULL) + continue; + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + nEXT ++; + } + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) { + nCR ++; + } + nSrc ++; + } + + // XXX: This loop cauess nSrc to be twice as large as it should be. This is kept + // for compatability + for (int i = 0; (sources != NULL) && (i < sources->n); i++) { + pmSource *source = (pmSource *) sources->data[i]; + pmModel *model = pmSourceGetModel (NULL, source); + if (model == NULL) + continue; + nSrc ++; + } + + psMetadataAddS32 (header, PS_LIST_TAIL, "NSTARS", PS_META_REPLACE, "Number of sources", nSrc); + psMetadataAddS32 (header, PS_LIST_TAIL, "NDET", PS_META_REPLACE, "Number of detections", nDetections); + psMetadataAddS32 (header, PS_LIST_TAIL, "NDET_EXT", PS_META_REPLACE, "Number of extended sources", nEXT); + psMetadataAddS32 (header, PS_LIST_TAIL, "NDET_CR", PS_META_REPLACE, "Number of cosmic rays", nCR); + psMetadataAddS32 (header, PS_LIST_TAIL, "NDET_FRC", PS_META_REPLACE, "Number of Forced detections", nForced); + return true; +} + +// these values are saved in an output header stub - they are added to either the PHU header +// (CMP) or the MEF table header (CMF). before the header is created, each readout has these +// values stored on readout->analysis +psMetadata *psphotDefineHeader (psMetadata *analysis) { + + bool status = true; + + psMetadata *header = psMetadataAlloc (); + + // write necessary information to output header + psMetadataItemSupplement (&status, header, analysis, "ZERO_PT"); + psMetadataItemSupplement (&status, header, analysis, "PHOTCODE"); + + psMetadataItemSupplement (&status, header, analysis, "APMIFIT"); + psMetadataItemSupplement (&status, header, analysis, "DAPMIFIT"); + psMetadataItemSupplement (&status, header, analysis, "NAPMIFIT"); + + // PSF model parameters (shape values for image center) + psMetadataItemSupplement (&status, header, analysis, "NPSFSTAR"); + psMetadataItemSupplement (&status, header, analysis, "APLOSS"); + psMetadataItemSupplement (&status, header, analysis, "APREFOFF"); + + psMetadataItemSupplement (&status, header, analysis, "FWHM_MAJ"); + psMetadataItemSupplement (&status, header, analysis, "FW_MJ_SG"); + psMetadataItemSupplement (&status, header, analysis, "FW_MJ_LQ"); + psMetadataItemSupplement (&status, header, analysis, "FW_MJ_UQ"); + + psMetadataItemSupplement (&status, header, analysis, "FWHM_MIN"); + psMetadataItemSupplement (&status, header, analysis, "FW_MN_SG"); + psMetadataItemSupplement (&status, header, analysis, "FW_MN_LQ"); + psMetadataItemSupplement (&status, header, analysis, "FW_MN_UQ"); + + psMetadataItemSupplement (&status, header, analysis, "ANGLE"); + + psMetadataItemSupplement (&status, header, analysis, "PSFMODEL"); + psMetadataItemSupplement (&status, header, analysis, "PSF_OK"); + + // Image Quality measurements + psMetadataItemSupplement (&status, header, analysis, "IQ_NSTAR"); + + psMetadataItemSupplement (&status, header, analysis, "IQ_FW1"); + psMetadataItemSupplement (&status, header, analysis, "IQ_FW1_E"); + psMetadataItemSupplement (&status, header, analysis, "IQ_FW2"); + psMetadataItemSupplement (&status, header, analysis, "IQ_FW2_E"); + + psMetadataItemSupplement (&status, header, analysis, "IQ_M2"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2_ER"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2_LQ"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2_UQ"); + + psMetadataItemSupplement (&status, header, analysis, "IQ_M2C"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2C_E"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2C_L"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2C_U"); + + psMetadataItemSupplement (&status, header, analysis, "IQ_M2S"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2S_E"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2S_L"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M2S_U"); + + psMetadataItemSupplement (&status, header, analysis, "IQ_M3"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M3_ER"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M3_LQ"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M3_UQ"); + + psMetadataItemSupplement (&status, header, analysis, "IQ_M4"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M4_ER"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M4_LQ"); + psMetadataItemSupplement (&status, header, analysis, "IQ_M4_UQ"); + + // XXX these need to be defined from elsewhere + psMetadataAdd (header, PS_LIST_TAIL, "FSATUR", PS_DATA_F32 | PS_META_REPLACE, "SATURATION MAG", 0.0); + psMetadataAdd (header, PS_LIST_TAIL, "FLIMIT", PS_DATA_F32 | PS_META_REPLACE, "COMPLETENESS MAG", 0.0); + psMetadataItemSupplement (&status, header, analysis, "NSTARS"); + + psMetadataItemSupplement (&status, header, analysis, "NDET"); + psMetadataItemSupplement (&status, header, analysis, "NDET_EXT"); + psMetadataItemSupplement (&status, header, analysis, "NDET_CR"); + psMetadataItemSupplement (&status, header, analysis, "NDET_FRC"); + + psMetadataItemSupplement (&status, header, analysis, "PSPHOT.PSF.APRESID"); + psMetadataItemSupplement (&status, header, analysis, "PSPHOT.PSF.APRESID.SYSERR"); + psMetadataItemSupplement (&status, header, analysis, "PSPHOT.CR.MAX.SIZE"); + psMetadataItemSupplement (&status, header, analysis, "PSPHOT.CR.MAX.MAG"); + + psMetadataItemSupplement (&status, header, analysis, "EFFECTIVE_AREA"); + psMetadataItemSupplement (&status, header, analysis, "SIGNIFICANCE_SCALE_FACTOR"); + + // sky background model statistics + psMetadataItemSupplement (&status, header, analysis, "MSKY_MN"); + psMetadataItemSupplement (&status, header, analysis, "MSKY_SIG"); + psMetadataItemSupplement (&status, header, analysis, "MSKY_MIN"); + psMetadataItemSupplement (&status, header, analysis, "MSKY_MAX"); + psMetadataItemSupplement (&status, header, analysis, "MSKY_NX"); + psMetadataItemSupplement (&status, header, analysis, "MSKY_NY"); + + psMetadataItemSupplement (&status, header, analysis, "MSKY_DEV"); + psMetadataItemSupplement (&status, header, analysis, "MSKY_DQ"); + + psMetadataItemSupplement (&status, header, analysis, "DETEFF.MAGREF"); + + psMetadataAddF32 (header, PS_LIST_TAIL, "DT_PHOT", PS_META_REPLACE, "elapsed psphot time", psTimerMark ("psphotReadout")); + + return header; +} + +// XXX add args as needed +bool psphotDumpPSFStars (pmReadout *readout, pmPSFtry *try, float radius, psImageMaskType maskVal, psImageMaskType markVal) { + + psphotSaveImage (NULL, readout->image, "rawstars.fits"); + + for (int i = 0; i < try->sources->n; i++) { + // masked for: bad model fit, outlier in parameters + if (try->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PSFTRY_MASK_ALL) + continue; + + pmSource *source = try->sources->data[i]; + float x = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + float y = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + + // set the mask and subtract the PSF model + // XXX should we be using maskObj? should we be unsetting the mask? + // use pmModelSub because modelFlux has not been generated + assert (source->maskObj); + psImageKeepCircle (source->maskObj, x, y, radius, "OR", markVal); + pmModelSub (source->pixels, source->maskObj, source->modelPSF, PM_MODEL_OP_FULL, maskVal); + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + } + + FILE *f = fopen ("shapes.dat", "w"); + for (int i = 0; i < try->sources->n; i++) { + psF32 inPar[10]; // must be psF32 to pmPSF_FitToModel + + // masked for: bad model fit, outlier in parameters + if (try->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PSFTRY_MASK_ALL) continue; + + pmSource *source = try->sources->data[i]; + psF32 *outPar = source->modelEXT->params->data.F32; + + bool useReff = pmModelUseReff (source->modelEXT->type); + + psEllipseAxes axes1; + pmModelParamsToAxes (&axes1, outPar[PM_PAR_SXX], outPar[PM_PAR_SXY], outPar[PM_PAR_SYY], useReff); + + psEllipsePol pol = pmPSF_ModelToFit (outPar, useReff); + inPar[PM_PAR_E0] = pol.e0; + inPar[PM_PAR_E1] = pol.e1; + inPar[PM_PAR_E2] = pol.e2; + pmPSF_FitToModel (inPar, 0.1, useReff); + + psEllipseAxes axes2 = psEllipsePolToAxes(pol, 0.1); + + psEllipsePol pol2 = psEllipseAxesToPol (axes1); + + fprintf (f, "%3d %7.2f %7.2f %7.4f %7.4f %7.4f -- %7.4f %7.4f %7.4f : %7.4f %7.4f %7.4f -- %7.4f %7.4f %7.4f : %7.4f %7.4f %6.1f : %7.4f %7.4f %6.1f\n", + i, outPar[PM_PAR_XPOS], outPar[PM_PAR_YPOS], + outPar[PM_PAR_SXX], outPar[PM_PAR_SXY], outPar[PM_PAR_SYY], + pol.e0, pol.e1, pol.e2, + pol2.e0, pol2.e1, pol2.e2, + inPar[PM_PAR_SXX], inPar[PM_PAR_SXY], inPar[PM_PAR_SYY], + axes1.major, axes1.minor, axes1.theta*PM_DEG_RAD, + axes2.major, axes2.minor, axes2.theta*PM_DEG_RAD + ); + } + fclose (f); + + psphotSaveImage (NULL, readout->image, "psfstars.fits"); + pmSourcesWritePSFs (try->sources, "psfstars.dat"); + pmSourcesWriteEXTs (try->sources, "extstars.dat", false); + // XXX need alternative output function + // psMetadata *psfData = pmPSFtoMetadata (NULL, try->psf); + // psMetadataConfigWrite (psfData, "psfmodel.dat", NULL); + psLogMsg ("psphot.choosePSF", PS_LOG_INFO, "wrote out psf-subtracted image, psf data, exiting\n"); + + return true; +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotDumpTest (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + static int npass = 0; + char filename[64]; + + // XXX dump tests are disabled unless this is commented out: + return true; + + bool status = true; + + int num = psphotFileruleCount(config, filerule); + + snprintf (filename, 64, "testdump.%02d.dat", npass); + FILE *f = fopen (filename, "w"); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + if (detections->newSources) { + fprintf (f, "## --- from new sources ---\n"); + } else { + fprintf (f, "## --- from all sources ---\n"); + } + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!source) continue; + + pmPeak *peak = source->peak; + if (!peak) continue; + + // XXX only dump a given region + // if (peak->xf < 20) continue; + // if (peak->yf < 20) continue; + // if (peak->xf > 40) continue; + // if (peak->yf > 70) continue; + + float Msum = source->moments ? source->moments->Sum : NAN; + float Mx = source->moments ? source->moments->Mx : NAN; + float My = source->moments ? source->moments->My : NAN; + // float Npix = source->moments ? source->moments->nPixels : NAN; + float Io = source->modelPSF ? source->modelPSF->params->data.F32[PM_PAR_I0] : NAN; + fprintf (f, "%d %f %f : %f %f : %f %f\n", source->imageID, peak->xf, peak->yf, Mx, My, Msum, Io); + } + } + fclose (f); + npass ++; + + return true; +} + +# if (0) +bool psphotDumpTest (pmConfig *config, const pmFPAview *view, const char *filerule, char *filename) { + + bool status; + + psTimerStart ("psphot.models"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + FILE *f = NULL; + + f = fopen (filename, "w"); + psAssert(f, "cannot open file"); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // skip non-astronomical objects (very likely defects) + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (!source->peak) continue; + if (!source->moments) continue; + + float Io = source->peak->rawFlux; + + fprintf (f, "%f %f : %f %f :: %f\n", + source->moments->Mx, source->moments->My, + source->peak->xf, source->peak->yf, Io); + } + fclose (f); + return true; +} +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPSFConvModel.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPSFConvModel.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPSFConvModel.c (revision 41083) @@ -0,0 +1,145 @@ +# include "psphotInternal.h" +# define USE_DELTA_PSF 0 + +// save as static values so they may be set externally +static psF32 PM_SOURCE_FIT_MODEL_NUM_ITERATIONS = 15; +static psF32 PM_SOURCE_FIT_MODEL_MIN_TOL = 0.1; +static psF32 PM_SOURCE_FIT_MODEL_MAX_TOL = 2.0; + +// input source has both modelPSF and modelEXT. on successful exit, we set the +// modelConv to contain the fitted parameters, and the modelFlux to contain the +// convolved model image. + +// XXX need to generalize this -- number of fitted parameters must be flexible based on the fitOptions + +pmModel *psphotPSFConvModel (pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, pmModelType modelType, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // make sure we save a cached copy of the psf flux + pmSourceCachePSF (source, maskVal); + + // convert the cached cached psf model for this source to a psKernel + psKernel *psf = psphotKernelFromPSF (source, psfSize); + if (!psf) return NULL; + +# if (USE_DELTA_PSF) + psImageInit (psf->image, 0.0); + psf->image->data.F32[(int)(0.5*psf->image->numRows)][(int)(0.5*psf->image->numCols)] = 1.0; +# endif + + // generate copy of the model + // XXX we could modify the parameter values or even the model + // here based on the observed seeing (some lookup table...) + + // use the source moments, etc to guess basic model parameters + pmModel *modelConv = pmSourceModelGuess (source, modelType); + if (!modelConv) { + psFree (psf); + return NULL; + } + + // adjust the pixels based on the footprint + float radius = psphotSetRadiusEXT (readout, source, markVal); + if (!pmSourceMoments (source, radius, 0.25*radius, 0.0, maskVal)) return false; + + // XXX test : modify the Io, SXX, SYY terms based on the psf SXX, SYY terms: + psEllipseShape psfShape; + psfShape.sx = source->modelPSF->params->data.F32[PM_PAR_SXX] / M_SQRT2; + psfShape.sxy = source->modelPSF->params->data.F32[PM_PAR_SXY]; + psfShape.sy = source->modelPSF->params->data.F32[PM_PAR_SYY] / M_SQRT2; + psEllipseAxes psfAxes = psEllipseShapeToAxes (psfShape, 20.0); + + // XXX test : modify the Io, SXX, SYY terms based on the psf SXX, SYY terms: + psEllipseShape extShape; + extShape.sx = modelConv->params->data.F32[PM_PAR_SXX] / M_SQRT2; + extShape.sxy = modelConv->params->data.F32[PM_PAR_SXY]; + extShape.sy = modelConv->params->data.F32[PM_PAR_SYY] / M_SQRT2; + psEllipseAxes extAxes = psEllipseShapeToAxes (extShape, 20.0); + + // decrease the initial guess ellipse by psf_minor axis: + psEllipseAxes extAxesMod; + extAxesMod.major = sqrt (PS_MAX (1.0, PS_SQR(extAxes.major) - PS_SQR(psfAxes.minor))); + extAxesMod.minor = sqrt (PS_MAX (1.0, PS_SQR(extAxes.minor) - PS_SQR(psfAxes.minor))); + extAxesMod.theta = extAxes.theta; + + psEllipseShape extShapeMod = psEllipseAxesToShape (extAxesMod); + modelConv->params->data.F32[PM_PAR_SXX] = extShapeMod.sx * M_SQRT2; + modelConv->params->data.F32[PM_PAR_SXY] = extShapeMod.sxy; + modelConv->params->data.F32[PM_PAR_SYY] = extShapeMod.sy * M_SQRT2; + + // increase the initial guess central intensity by 2pi r^2: + modelConv->params->data.F32[PM_PAR_I0] *= (1.0 + PS_SQR(psfAxes.minor) / PS_SQR(extAxesMod.minor)); + + psVector *params = modelConv->params; + psVector *dparams = modelConv->dparams; + + // create the minimization constraints + psMinConstraint *constraint = psMinConstraintAlloc(); + constraint->paramMask = psVectorAlloc (params->n, PS_TYPE_VECTOR_MASK); + constraint->checkLimits = modelConv->modelLimits; + + // set parameter mask based on fitting mode + // we fit a model without a floating sky term + int nParams = params->n - 1; + psVectorInit (constraint->paramMask, 0); + constraint->paramMask->data.PS_TYPE_VECTOR_MASK_DATA[PM_PAR_SKY] = 1; + + // force the floating parameters to fall within the contraint ranges + for (int i = 0; i < params->n; i++) { + modelConv->modelLimits (PS_MINIMIZE_PARAM_MIN, i, params->data.F32, NULL); + modelConv->modelLimits (PS_MINIMIZE_PARAM_MAX, i, params->data.F32, NULL); + } + + // set up the minimization process + psMinimization *myMin = psMinimizationAlloc (PM_SOURCE_FIT_MODEL_NUM_ITERATIONS, PM_SOURCE_FIT_MODEL_MIN_TOL, PM_SOURCE_FIT_MODEL_MAX_TOL); + + psImage *covar = psImageAlloc (params->n, params->n, PS_TYPE_F32); + + bool fitStatus = psphotModelWithPSF_LMM (myMin, covar, params, constraint, source, psf, modelConv->modelFunc); + for (int i = 0; i < dparams->n; i++) { + if (psTraceGetLevel("psphot") >= 4) { + fprintf (stderr, "%f ", params->data.F32[i]); + } + if ((constraint->paramMask != NULL) && constraint->paramMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) + continue; + dparams->data.F32[i] = sqrt(covar->data.F32[i][i]); + } + psTrace ("psphot", 4, "niter: %d, chisq: %f", myMin->iter, myMin->value); + + // renormalize output model image (generated by fitting process) + float Io = params->data.F32[PM_PAR_I0]; + for (int iy = 0; iy < source->modelFlux->numRows; iy++) { + for (int ix = 0; ix < source->modelFlux->numCols; ix++) { + source->modelFlux->data.F32[iy][ix] /= Io; + } + } + + // save the resulting chisq, nDOF, nIter + modelConv->chisq = myMin->value; + modelConv->nIter = myMin->iter; + + // XXX I actually need to count the number of unmasked pixels here + modelConv->nDOF = source->pixels->numCols*source->pixels->numRows - nParams; + + modelConv->flags |= PM_MODEL_STATUS_FITTED; + if (!fitStatus) modelConv->flags |= PM_MODEL_STATUS_NONCONVERGE; + + // models can go insane: reject these + bool onPic = true; + onPic &= (params->data.F32[PM_PAR_XPOS] >= source->pixels->col0); + onPic &= (params->data.F32[PM_PAR_XPOS] < source->pixels->col0 + source->pixels->numCols); + onPic &= (params->data.F32[PM_PAR_YPOS] >= source->pixels->row0); + onPic &= (params->data.F32[PM_PAR_YPOS] < source->pixels->row0 + source->pixels->numRows); + if (!onPic) modelConv->flags |= PM_MODEL_STATUS_OFFIMAGE; + + source->mode |= PM_SOURCE_MODE_FITTED; // XXX is this needed? + + psFree(psf); + psFree(myMin); + psFree(covar); + psFree(constraint); + + return modelConv; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotParseCamera.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotParseCamera.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotParseCamera.c (revision 41083) @@ -0,0 +1,76 @@ +# include "psphotStandAlone.h" + +// define the needed / desired I/O files + + +bool psphotParseCamera (pmConfig *config) { + + bool status = false; + + // the file to be loaded may have subdivisions at the cell and readout level + // we load into pmFPAfile *load, then reformat into pmFPAfile *input + pmFPAfile *load = pmFPAfileDefineFromArgs (&status, config, "PSPHOT.LOAD", "INPUT"); + if (!status) { + psError(PSPHOT_ERR_CONFIG, false, "Failed to build FPA from PSPHOT.LOAD"); + return status; + } + load->dataLevel = PM_FPA_LEVEL_CHIP; // force load at the CHIP level + + // if MASK or VARIANCE was supplied on command line, bind files to 'load' + // the mask and weight will be mosaicked with the image + pmFPAfileBindFromArgs (&status, load, config, "PSPHOT.MASK", "MASK"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "failed to load find definition"); + return NULL; + } + if (!psphotSetMaskBits (config)) { + psError (PS_ERR_UNKNOWN, false, "failed to set mask bit values"); + return NULL; + } + + pmFPAfileBindFromArgs (&status, load, config, "PSPHOT.VARIANCE", "VARIANCE"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "failed to load find definition"); + return NULL; + } + + // the psphot analysis is performed on chips + pmFPAfile *input = pmFPAfileDefineChipMosaic(config, load->fpa, "PSPHOT.INPUT"); + if (!input) { + psError(PSPHOT_ERR_CONFIG, false, _("Unable to generate new file from PSPHOT.INPUT")); + return NULL; + } + + pmFPAfileBindFromArgs (&status, input, config, "PSPHOT.EXPNUM", "EXPNUM"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "failed to load find definition"); + return NULL; + } + + + // define the additional input/output files associated with psphot + if (!psphotDefineFiles (config, input)) { + psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files"); + return false; + } + + // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg) + char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS"); + psArray *chips = psStringSplitArray (chipLine, ",", false); + if (chips->n > 0) { + // select on the basis of extname? + pmFPASelectChip (load->fpa, -1, true); // deselect all chips + for (int i = 0; i < chips->n; i++) { + int chipNum = atoi(chips->data[i]); + if (! pmFPASelectChip(load->fpa, chipNum, false)) { + psError(PSPHOT_ERR_CONFIG, false, "Chip number %d doesn't exist in camera.\n", chipNum); + return false; + } + } + } + psFree (chips); + psTrace("psphot", 1, "Done with psphotParseCamera...\n"); + + psErrorClear(); // some metadata lookup may have failed + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetroFlux.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetroFlux.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetroFlux.c (revision 41083) @@ -0,0 +1,259 @@ +# include "psphotInternal.h" + +# define PETROSIAN_RADII 2.0 + +bool psphotPetroFlux (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Petro Fluxes ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + psMetadataAddBool (recipe, PS_LIST_TAIL, "EXTENDED_SOURCE_ANALYSIS", PS_META_REPLACE, "we measured this, save to disk", true); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + if (!psphotPetroFluxReadout (config, recipe, view, filerule, readout, sources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotPetroFluxReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, const char * filerule, pmReadout *readout, psArray *sources) { + + bool status = false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping petrosian fluxes"); + return true; + } + + psTimerStart ("psphot.petro"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend"); + return true; + } + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_PETRO_FLUX"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } +# else + if (!psphotPetroFlux_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + // psFree(AnalysisRegion); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + psFree (cellGroups); + + psLogMsg ("psphot.petro", PS_LOG_WARN, "measure petro fluxes : %f sec for %ld objects\n", psTimerMark ("psphot.petro"), sources->n); + return true; +} + +bool psphotPetroFlux_Threaded (psThreadJob *job) { + + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[2],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[3],PS_TYPE_IMAGE_MASK_DATA); + + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + if (!source->extpars) continue; // if this is not set, we did not have a valid petRadius + + // check status of this source's moments + if (!source->moments) continue; + if (!(source->tmpFlags & PM_SOURCE_TMPF_MOMENTS_MEASURED)) continue; + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + + // XXX where are we storing the supplied petro radius? + if (!isfinite(source->moments->Mrf)) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // is the right?? XXX need the correct location for Rpet + float windowRadius = 2.0*source->moments->Mrf ; + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + pmSourceRedefinePixels (source, readout, source->peak->x, source->peak->y, windowRadius + 2); + psAssert (source->pixels, "WTF?"); + + // this function populates moments->Mrf,PetroFlux,PetroFluxErr + psphotPetroFluxSource (source, maskVal); + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + + return true; +} + +// measure just the flux in the given aperture (this is probably a single common function -- can I use one of the pmSourcePhotometry functions?) +bool psphotPetroFluxSource (pmSource *source, psImageMaskType maskVal) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + PS_ASSERT_PTR_NON_NULL(source->variance, false); + PS_ASSERT_PTR_NON_NULL(source->moments, false); + PS_ASSERT_PTR_NON_NULL(source->extpars, false); + + // the peak position is less accurate but less subject to extreme deviations + float dX = source->moments->Mx - source->peak->xf; + float dY = source->moments->My - source->peak->yf; + float dR = hypot(dX, dY); + float Xo = (dR < 2.0) ? source->moments->Mx : source->peak->xf; + float Yo = (dR < 2.0) ? source->moments->My : source->peak->yf; + + // center of mass in subimage. Note: the calculation below uses pixel index, so we correct + // xCM, yCM from pixel coords to pixel index here. + psF32 xCM = Xo - 0.5 - source->pixels->col0; // coord of peak in subimage + psF32 yCM = Yo - 0.5 - source->pixels->row0; // coord of peak in subimage + + // Calculate the Petro magnitude (make this block optional?) + // XXX set the aperture here + float radPetro = PETROSIAN_RADII*source->extpars->petrosianRadius; + float radPetro2 = radPetro*radPetro; + + int nPetroPix = 0; + float Sum = 0.0; + float Var = 0.0; + + // set vPix to the source pixels (it may have been set to the + // smoothed image above) + psF32 **vPix = source->pixels->data.F32; + psF32 **vWgt = source->variance->data.F32; + psImageMaskType **vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + for (psS32 row = 0; row < source->pixels->numRows ; row++) { + + psF32 yDiff = row - yCM; + if (fabs(yDiff) > radPetro) continue; + + for (psS32 col = 0; col < source->pixels->numCols ; col++) { + // check mask and value for this pixel + if (vMsk && (vMsk[row][col] & maskVal)) continue; + if (isnan(vPix[row][col])) continue; + + psF32 xDiff = col - xCM; + if (fabs(xDiff) > radPetro) continue; + + // radPetro is just a function of (xDiff, yDiff) + psF32 r2 = PS_SQR(xDiff) + PS_SQR(yDiff); + if (r2 > radPetro2) continue; + + float pDiff = vPix[row][col]; + psF32 wDiff = vWgt[row][col]; + + Sum += pDiff; + Var += wDiff; + nPetroPix ++; + } + } + + // return the flux and error to parameters? + source->extpars->petrosianFlux = Sum; + source->extpars->petrosianFluxErr = sqrt(Var); + source->extpars->petrosianFill = nPetroPix / (M_PI * radPetro2); + + // fprintf (stderr, "petro flux: %f +/- %f\n", Sum, sqrt(Var)); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosian.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosian.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosian.c (revision 41083) @@ -0,0 +1,32 @@ +# include "psphotInternal.h" + +bool psphotPetrosian (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal) { + + // XXX these need to go into recipe values + float Rmax = 200; + + psAssert (source->extpars, "need to run psphotRadialProfile first"); + psAssert (source->extpars->ellipticalFlux, "need to run psphotRadialProfile first"); + + // integrate the radial profile for radial bins defined for the petrosian measurement: + // SB_i (r_i) where \alpha r_i < r < \beta r_i + if (!psphotPetrosianRadialBins (source, Rmax, skynoise)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate elliptical profile"); + return false; + } + + // use the SB_i from above to calculate the petrosian radius and the flux within that radius + if (!psphotPetrosianStats (source)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate elliptical profile"); + return false; + } + + psTrace ("psphot", 3, "source at %f,%f: petrosian radius: %f, flux: %f, axis ratio: %f, angle: %f", + source->peak->xf, source->peak->yf, + source->extpars->petrosianRadius, + source->extpars->petrosianFlux, + source->extpars->axes.minor/source->extpars->axes.major, + source->extpars->axes.theta*PS_DEG_RAD); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianAnalysis.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianAnalysis.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianAnalysis.c (revision 41083) @@ -0,0 +1,61 @@ +# include "psphotInternal.h" + +// aperture-like measurements for extended sources +bool psphotPetrosianAnalysis (pmReadout *readout, psArray *sources, psMetadata *recipe) { + + bool status; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // XXX temporary user-supplied systematic sky noise measurement (derive from background model) + float skynoise = psMetadataLookupF32 (&status, recipe, "SKY.NOISE"); + + // S/N limit to perform full non-linear fits + float SN_LIM = psMetadataLookupF32 (&status, recipe, "EXTENDED_SOURCE_SN_LIM"); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion AnalysisRegion = psRegionForImage (readout->image, psRegionFromString (region)); + if (psRegionIsNaN (AnalysisRegion)) psAbort("analysis region mis-defined"); + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // choose the sources of interest + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_STAR) continue; + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + if (!(source->mode & PM_SOURCE_MODE_EXT_LIMIT)) continue; + + // limit selection to some SN limit + assert (source->peak); // how can a source not have a peak? + if (source->peak->SN < SN_LIM) continue; + + // limit selection by analysis region + if (source->peak->x < AnalysisRegion.x0) continue; + if (source->peak->y < AnalysisRegion.y0) continue; + if (source->peak->x > AnalysisRegion.x1) continue; + if (source->peak->y > AnalysisRegion.y1) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + psphotPetrosianProfile (readout, source, skynoise); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + + psphotVisualShowResidualImage (readout, false); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianProfile.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianProfile.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianProfile.c (revision 41083) @@ -0,0 +1,79 @@ +# include "psphotInternal.h" + +// generate the Petrosian radius and flux using elliptical contours + +// XXX much of this function is focused on generating the clean contours, which can be used by +// any number of aperture-like measurements. probably will want to rename the pmPetrosian +// structure to something the pmRadialProfile + +bool psphotPetrosianProfile (pmReadout *readout, pmSource *source, float skynoise) { + + // container to hold results from the radial profile analysis + pmPetrosian *petrosian = pmPetrosianAlloc(); + + // XXX these need to go into recipe values + int Nsec = 24; + float Rmax = 200; + float fluxMin = 0.0; + float fluxMax = source->peak->flux; + + // generate a series of radial profiles at Nsec evenly spaced angles. the profile flux + // is measured by interpolation for small radii; for large radii, the pixels in a box + // are averaged to increase the S/N (XXX not yet done) + if (!psphotRadialProfilesByAngles (source, petrosian, Nsec, Rmax)) { + psError (PS_ERR_UNKNOWN, false, "failed to measure radial profile for petrosian"); + psFree (petrosian); + return false; + } + + // use the radial profiles to determine the radius of a given isophote. this isophote + // is used to determine the elliptical shape of the object, so it has a relatively high + // value (nominally 50% of the peak) + if (!psphotRadiiFromProfiles (source, petrosian, fluxMin, fluxMax)) { + psError (PS_ERR_UNKNOWN, false, "failed to measure isophotal radii from profiles"); + psFree (petrosian); + return false; + } + + // convert the isophotal radius vs angle measurements to an elliptical contour + if (!psphotEllipticalContour (source, petrosian)) { + // psLogMsg ("psphot", 3, "failed to measure elliptical contour"); + psFree (petrosian); + return false; + } + + // generate a single, normalized radial profile following the elliptical contours. + // the radius is normalized by the axis ratio so that on the major axis, 1 pixel = 1 pixel + if (!psphotEllipticalProfile (source, petrosian)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate elliptical profile"); + psFree (petrosian); + return false; + } + + // integrate the radial profile for radial bins defined for the petrosian measurement: + // SB_i (r_i) where \alpha r_i < r < \beta r_i + if (!psphotPetrosianRadialBins (source, petrosian, Rmax, skynoise)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate elliptical profile"); + psFree (petrosian); + return false; + } + + // use the SB_i from above to calculate the petrosian radius and the flux within that radius + if (!psphotPetrosianStats (source, petrosian)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate elliptical profile"); + psFree (petrosian); + return false; + } + + // XXX this will only work in the psphot context, not the psphotPetrosianStudy... + // XXX add the petrosian to the pmSource structure... + psphotVisualShowPetrosian (source, petrosian); + + psphotPetrosianFreeVectors(petrosian); + + psTrace ("psphot", 3, "source at %f,%f: petrosian radius: %f, flux: %f, axis ratio: %f, angle: %f", + source->peak->xf, source->peak->yf, petrosian->petrosianRadius, petrosian->petrosianFlux, petrosian->axes.minor/petrosian->axes.major, PS_DEG_RAD*petrosian->axes.theta); + + psFree (petrosian); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianRadialBins.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianRadialBins.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianRadialBins.c (revision 41083) @@ -0,0 +1,236 @@ +# include "psphotInternal.h" +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X); + +// convert the flux vs elliptical radius to annular bins + +// we are guaranteed to be limited by either the seeing (1 - few pixels) or by the pixels +// themselves. this function does not attempt to measure the radial profiles accurately +// for radii that are smaller than ~2 pixels + +// for small radii, we are measuring the mean surface brightness in non-overlapping radial +// bins. for large radii (r > 2 pixels), we are measuring the surface brightness for a +// radius range \alpha r_i < i < \beta r_i, but performing this measurement for radii more +// finely spaced than r_{i+1} = r_i * \beta / \alpha. for the integration, we need to +// track the non-overlapping radius values. + +// Photo interpolates the image of interest to place the peak on the center of the central +// pixel, and then uses the exact fractions of the pixels in each of the first few annuli. +// Seems like a reasonable thing, but is there any significance to the difference? + +// XXX move the resulting elements from profile to extpars->petrosian? +bool psphotPetrosianRadialBins (pmSource *source, float radiusMax, float skynoise) { + + psAssert (source, "missing source"); + psAssert (source->extpars, "missing extpars"); + psAssert (source->extpars->ellipticalFlux, "missing ellipticalFlux"); + + psVector *radius = source->extpars->ellipticalFlux->radiusElliptical; + psVector *flux = source->extpars->ellipticalFlux->fluxElliptical; + + // sort incoming vectors by radius + pmSourceRadialProfileSortPair (radius, flux); + + if (!source->extpars->petProfile) { + source->extpars->petProfile = pmSourceRadialProfileAlloc(); + } + pmSourceRadialProfile *profile = source->extpars->petProfile; + + // float skyModelErrorSQ = PS_SQR(skynoise); + + int nMax = radiusMax; + + // radBin stores the centers of the radial bins, + // radMin, radMax store the bounds + psVector *radMin = psVectorAllocEmpty(nMax, PS_TYPE_F32); + psVector *radMax = psVectorAllocEmpty(nMax, PS_TYPE_F32); + psVector *radAlp = psVectorAllocEmpty(nMax, PS_TYPE_F32); + psVector *radBet = psVectorAllocEmpty(nMax, PS_TYPE_F32); + + psVector *binSB = psVectorAllocEmpty(nMax, PS_TYPE_F32); // surface brightness of radial bin + psVector *binSBstdev = psVectorAllocEmpty(nMax, PS_TYPE_F32); // surface brightness of radial bin + psVector *binRad = psVectorAllocEmpty(nMax, PS_TYPE_F32); // mean radius of radial bin + psVector *binArea = psVectorAllocEmpty(nMax, PS_TYPE_F32); // area of radial bin (contiguous, non-overlapping) + psVector *binFill = psVectorAllocEmpty(nMax, PS_TYPE_F32); // fraction of radial bin with valid pixels + + psVectorInit (binSB, 0.0); + psVectorInit (binSBstdev, 0.0); + psVectorInit (binRad, 0.0); + + // generate radial bin bounds + radMin->data.F32[0] = 0.0; + radMax->data.F32[0] = 1.0; + radAlp->data.F32[0] = 0.0; + radBet->data.F32[0] = 1.0; + + radMin->data.F32[1] = 1.0; + radMax->data.F32[1] = 1.5; + radAlp->data.F32[1] = 1.0; + radBet->data.F32[1] = 1.5; + + radMin->data.F32[2] = 1.5; + radMax->data.F32[2] = 2.0; + radAlp->data.F32[2] = 1.5; + radBet->data.F32[2] = 2.0; + +# define PETROSIAN_ALPHA 0.8 +# define PETROSIAN_BETA 1.25 +# define POWER_LAW_SPACING true + + // power-law spacing with overlapping boundaries at the geometric mid-points + float rBeta = sqrt(PETROSIAN_BETA); + for (int i = 3; radBet->data.F32[i-1] < radiusMax; i++) { + if (POWER_LAW_SPACING) { + radMin->data.F32[i] = radMax->data.F32[i-1]; + radMax->data.F32[i] = radMin->data.F32[i] * PETROSIAN_BETA; + radAlp->data.F32[i] = radMin->data.F32[i] / rBeta; + radBet->data.F32[i] = radMax->data.F32[i] * rBeta; + } else { + radMin->data.F32[i] = radMax->data.F32[i-1]; + radMax->data.F32[i] = radMin->data.F32[i] + 1; + float rMid = 0.5*(radMin->data.F32[i] + radMax->data.F32[i]); + radAlp->data.F32[i] = rMid * PETROSIAN_ALPHA; + radBet->data.F32[i] = rMid * PETROSIAN_BETA; + } + radMin->n = radMax->n = radAlp->n = radBet->n = i + 1; + } + + // generate radial area-weighted mean radius & non-overlapping areas + for (int i = 0; i < radMin->n; i++) { + float rMin = radMin->data.F32[i]; + float rMax = radMax->data.F32[i]; + + float rMin2 = rMin*rMin; + float rMin3 = rMin2*rMin; + + float rMax2 = rMax*rMax; + float rMax3 = rMax2*rMax; + + float rBin = 2.0 * (rMax3 - rMin3) / (rMax2 - rMin2) / 3.0; + + // XXX calculate area-weighted radius rather than asserting? + binRad->data.F32[i] = rBin; + binArea->data.F32[i] = M_PI * (rMax2 - rMin2); + + psTrace ("psphot", 6, "%3d %5.1f %5.1f : %5.1f : %5.1f %5.1f\n", + i, radAlp->data.F32[i], radMin->data.F32[i], binRad->data.F32[i], + radMax->data.F32[i], radBet->data.F32[i]); + } + + // storage vector for stats + psVector *values = psVectorAllocEmpty (flux->n, PS_TYPE_F32); + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + + bool done = false; + int nOut = 0; + float Rmin = radAlp->data.F32[nOut]; + float Rmax = radBet->data.F32[nOut]; + float Rnxt = radAlp->data.F32[nOut+1]; // minimum radius for next range + int iNext = 0; + for (int i = 0; !done && (i < radius->n); i++) { + if (radius->data.F32[i] < Rnxt) { + iNext = i; + } + if (radius->data.F32[i] > Rmax) { + // calculate the value for the nOut bin + float value; // , dvalue; + if (values->n > 0) { + psVectorStats (stats, values, NULL, NULL, 0); + value = stats->robustMedian; + // dvalue = stats->robustStdev; + } else { + value = NAN; + // dvalue = NAN; + } + + binSB->data.F32[nOut] = value; + // binSBstdev->data.F32[nOut] = sqrt(PS_SQR(dvalue) / values->n + skyModelErrorSQ); + binSBstdev->data.F32[nOut] = skynoise / sqrt(values->n); + binFill->data.F32[nOut] = values->n / binArea->data.F32[nOut]; + + // error in the SB is the stdev per bin / sqrt (number of pixels) + // added in quadrature to a fraction of the local sky (not the + // residual flux, but the sky from the sky model) + + psTrace ("psphot", 5, "%3d %5.1f %5.1f : %5.1f %5.2f\n", nOut, radAlp->data.F32[nOut], radBet->data.F32[nOut], binSB->data.F32[nOut], binSBstdev->data.F32[nOut]); + + nOut ++; + if (nOut >= radAlp->n) break; + Rmin = radAlp->data.F32[nOut]; + Rmax = radBet->data.F32[nOut]; + Rnxt = (nOut < nMax - 1) ? radAlp->data.F32[nOut+1] : Rmax; // minimum radius for next range + values->n = 0; + psStatsInit(stats); + i = iNext; + } + if (radius->data.F32[i] < Rmin) { + continue; + } + psVectorAppend (values, flux->data.F32[i]); + } + binSB->n = binSBstdev->n = binRad->n = binArea->n = nOut; + // XXX I think this misses the last radial bin -- do we care? + + // interpolate any bins that were empty (extrapolate to center if needed) + if (!isfinite(binSB->data.F32[0]) && !isfinite(binSB->data.F32[1])) { + psWarning ("center 2 bins of source at %f, %f are NAN, skipping this source", source->peak->xf, source->peak->yf); + // XXX raise a flag + psFree(binSB); + psFree(binSBstdev); + psFree(binRad); + psFree(binArea); + psFree(binFill); + psFree(radMin); + psFree(radMax); + psFree(radAlp); + psFree(radBet); + psFree(values); + psFree(stats); + source->mode2 |= PM_SOURCE_MODE2_RADBIN_NAN_CENTER; + return true; + } + + // if center bin is empty assume same SB as next radius (probably true due to PSF) + if (!isfinite(binSB->data.F32[0])) { + binSB->data.F32[0] = binSB->data.F32[1]; + binSBstdev->data.F32[0] = binSBstdev->data.F32[1]; + } + + // interpolate any bins that were empty (if center if needed) + for (int i = 1; i < binSB->n - 1; i++) { + if (isfinite(binSB->data.F32[i])) continue; + binSB->data.F32[i] = InterpolateValues (binRad->data.F32[i-1], binSB->data.F32[i-1], binRad->data.F32[i+1], binSB->data.F32[i+1], binRad->data.F32[i]); + binSBstdev->data.F32[i] = InterpolateValues (binRad->data.F32[i-1], binSBstdev->data.F32[i-1], binRad->data.F32[i+1], binSBstdev->data.F32[i+1], binRad->data.F32[i]); + } + + psFree(profile->binSB); + psFree(profile->binSBstdev); + psFree(profile->radialBins); + psFree(profile->area); + psFree(profile->binFill); + + // save the vectors + profile->radialBins = binRad; + profile->area = binArea; + profile->binFill = binFill; + profile->binSB = binSB; + profile->binSBstdev = binSBstdev; + + // psphotPetrosianVisualProfileRadii (radius, flux, binRad, binSB, source->peak->flux, 0.0); + + psFree(radMin); + psFree(radMax); + psFree(radAlp); + psFree(radBet); + psFree(values); + psFree(stats); + + return true; +} + +// the area-weighted mean radius is given by: + +// integral r * 2 pi r dr / integral 2 pi r dr + +// = 2/3 pi (r_max^3 - r_min^3) / pi (r_max^2 - r_min^2) +// = 2/3 (r_max^3 - r_min^3) / (r_max^2 - r_min^2) + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianStats.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianStats.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianStats.c (revision 41083) @@ -0,0 +1,294 @@ +# include "psphotInternal.h" + +# define PETROSIAN_RATIO 0.2 +# define PETROSIAN_RADII 2.0 + +// generate the Petrosian radius and flux from the mean surface brightness (r_i) + +float InterpolateValuesQuadratic (float *Xin, float *Yin, float X); +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X); +float InterpolateValuesErrX (float X0, float Y0, float X1, float Y1, float X, float dX0, float dX1); +float InterpolateValuesErrY (float X0, float Y0, float X1, float Y1, float X, float dY0, float dY1); + +bool psphotPetrosianStats (pmSource *source) { + + psAssert (source, "missing source"); + psAssert (source->extpars, "missing extpars"); + psAssert (source->extpars->petProfile, "missing petProfile"); + + pmSourceRadialProfile *profile = source->extpars->petProfile; + + if (!profile->binSB) { + psLogMsg ("psphot", PS_LOG_DETAIL, "no petrosian profile, skipping source %f, %f", source->peak->xf, source->peak->yf); + source->mode2 |= PM_SOURCE_MODE2_PETRO_NO_PROFILE; + return true; + } + + psVector *binSB = profile->binSB; + psVector *binSBstdev = profile->binSBstdev; + psVector *binRad = profile->radialBins; + psVector *area = profile->area; + psVector *binFill = profile->binFill; + + psVector *fluxSum = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *fluxSumErr2 = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *refRadius = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *petRatio = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *petRatioErr = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *meanSB = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *areaSum = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + psVector *apixSum = psVectorAllocEmpty(binSB->n, PS_TYPE_F32); + + float petRadius = NAN; + float petRadiusErr = NAN; + float petFlux = NAN; + float petFluxErr = NAN; + float petArea = NAN; + float petApix = NAN; + + bool anyPetro = false; + // bool manyPetro = false; XXX not used + bool above = true; + float Asum = 0.0; + float Psum = 0.0; + float Fsum = 0.0; + float dFsum2 = 0.0; + + float nSigma = 2.0; + int lowestSignificantRadius = 0; + float lowestSignificantRatio = 1.0; + + // find the Petrosian Radius and Petrosian Flux + + int nOut = 0; + for (int i = 0; i < binSB->n; i++) { + // skip nan bins (do not contribute to flux or area) + if (!isfinite(binSB->data.F32[i])) continue; + + float Area = area->data.F32[i]; + Asum += Area; // Asum is the cumulative area interior to this bin + Fsum += binSB->data.F32[i] * Area; + dFsum2 += PS_SQR(binSBstdev->data.F32[i] * Area); + Psum += Area*binFill->data.F32[i]; // Psum is the cumulative number of pixels interior to this bin + + float areaInner = 0.5 * Area; + float fluxInner = 0.5 * Area * binSB->data.F32[i]; + float fluxInnerErr2 = PS_SQR(binSBstdev->data.F32[i] * 0.5 * Area); + if (nOut > 0) { + areaInner += areaSum->data.F32[nOut-1]; + fluxInner += fluxSum->data.F32[nOut-1]; + fluxInnerErr2 += fluxSumErr2->data.F32[nOut-1]; + } + + // ratio = binSB / meanSB + // meanSB = flux / area + // flux = sum(binSB(i) * area(i) + // fluxErr^2 = sum(binSBerr(i)^2 area(i)^2) + // meanSBerr^2 = fluxErr^2 / area^2 + // (ratioErr/ratio)^2 = (binSBerr/binSB)^2 + (meanSBerr/meanSB)^2 + + psVectorAppend(meanSB, (fluxInner / areaInner)); + + float ratio = binSB->data.F32[i] / meanSB->data.F32[nOut]; + psVectorAppend(petRatio, ratio); + + float meanSBerr = sqrt(fluxInnerErr2) / areaInner; + float ratioErr = fabs(ratio) * sqrt(PS_SQR(binSBstdev->data.F32[i]/binSB->data.F32[i]) + PS_SQR(meanSBerr/meanSB->data.F32[nOut])); + + psVectorAppend(petRatioErr, ratioErr); + + psVectorAppend(areaSum, Asum); + psVectorAppend(apixSum, Psum); + psVectorAppend(fluxSum, Fsum); + psVectorAppend(fluxSumErr2, dFsum2); + psVectorAppend(refRadius, binRad->data.F32[i]); + + psTrace ("psphot", 4, "%3d : %5.2f : %5.3f %5.3f : %5.3f %5.3f : %5.3f %5.3f : %5.3f %5.3f : %5.1f %5.1f %5.1f\n", + i, refRadius->data.F32[nOut], + binSB->data.F32[i], binSBstdev->data.F32[i], + meanSB->data.F32[nOut], meanSBerr, + petRatio->data.F32[nOut], petRatioErr->data.F32[nOut], + fluxSum->data.F32[nOut], sqrt(fluxSumErr2->data.F32[nOut]), areaSum->data.F32[nOut], apixSum->data.F32[nOut], areaInner); + + // anytime we transition below the PETROSIAN_RATIO, calculate the radius and flux + // we will keep and report the last (largest radius) value + if (above && (petRatio->data.F32[nOut] < PETROSIAN_RATIO) && (petRatio->data.F32[nOut] > nSigma*petRatioErr->data.F32[nOut])) { + // interpolate Rvec between i-1 and i to PETROSIAN_RATIO to get flux (Fvec) and radius (rvec) + if (i == 0) { + // assume Fmax @ R = 0.0 + petRadius = InterpolateValues (1.0, 0.0, petRatio->data.F32[nOut], refRadius->data.F32[nOut], PETROSIAN_RATIO); + petRadiusErr = InterpolateValuesErrX (1.0, 0.0, petRatio->data.F32[nOut], refRadius->data.F32[nOut], PETROSIAN_RATIO, 0.0, petRatioErr->data.F32[nOut]); + source->mode2 |= PM_SOURCE_MODE2_PETRO_RATIO_ZEROBIN; + } else { + // petRadius = InterpolateValues (petRatio->data.F32[nOut-1], refRadius->data.F32[nOut-1], petRatio->data.F32[nOut], refRadius->data.F32[nOut], PETROSIAN_RATIO); + if (nOut > 1) { + petRadius = InterpolateValuesQuadratic (&petRatio->data.F32[nOut-2], &refRadius->data.F32[nOut-2], PETROSIAN_RATIO); + } else { + petRadius = InterpolateValuesQuadratic (&petRatio->data.F32[nOut-3], &refRadius->data.F32[nOut-3], PETROSIAN_RATIO); + } +# if (PS_TRACE_ON) + float petRadiusLinear = InterpolateValues (petRatio->data.F32[nOut-1], refRadius->data.F32[nOut-1], petRatio->data.F32[nOut], refRadius->data.F32[nOut], PETROSIAN_RATIO); + if (fabs(petRadius - petRadiusLinear) > fabs(refRadius->data.F32[nOut] - refRadius->data.F32[nOut-1])) { + fprintf (stderr, "big difference : %f vs %f\n", petRadius, petRadiusLinear); + } +#endif + petRadiusErr = InterpolateValuesErrX (petRatio->data.F32[nOut-1], refRadius->data.F32[nOut-1], petRatio->data.F32[nOut], refRadius->data.F32[nOut], PETROSIAN_RATIO, petRatioErr->data.F32[nOut-1], petRatioErr->data.F32[nOut]); + } + above = false; + // if (anyPetro) manyPetro = true; + anyPetro = true; + } + + // anytime we transition below the PETROSIAN_RATIO, calculate the radius and flux + // we will keep and report the last (largest radius) value + // find the last signficant measurement of the petrosian ratio + if (above && (petRatio->data.F32[nOut] < lowestSignificantRatio) && (petRatio->data.F32[nOut] > nSigma*petRatioErr->data.F32[nOut])) { + lowestSignificantRadius = nOut; + lowestSignificantRatio = petRatio->data.F32[nOut]; + } + + // reset on transitions up, but do not re-calculate rad_90, flux_90 + if (!above && (petRatio->data.F32[nOut] >= PETROSIAN_RATIO)) { + above = true; + } + nOut ++; + } + + // if we failed to reach the PETROSIAN_RATIO, use the lowest significant ratio instead (flag this!) + if (!anyPetro) { + petRadius = refRadius->data.F32[lowestSignificantRadius]; + petRadiusErr = NAN; + if (!isfinite(petRadius)) { + fprintf (stderr, "nan pet radius\n"); + } + source->mode2 |= PM_SOURCE_MODE2_PETRO_INSIG_RATIO; + } + + // now measure the flux within PETROSIAN_RADII * petRadius + float apRadius = PETROSIAN_RADII * petRadius; + for (int i = 0; i < refRadius->n; i++) { + // XXX use bisection to do this faster: + if (refRadius->data.F32[i] > apRadius) { + if (i == 0) { + psWarning ("does this case make any sense? (refRadius[0] %f > apRadius %f)", refRadius->data.F32[i], apRadius); + continue; + } else { + petFlux = InterpolateValues (refRadius->data.F32[i-1], fluxSum->data.F32[i-1], refRadius->data.F32[i], fluxSum->data.F32[i], apRadius); + petFluxErr = InterpolateValuesErrY (refRadius->data.F32[i-1], fluxSum->data.F32[i-1], refRadius->data.F32[i], fluxSum->data.F32[i], apRadius, sqrt(fluxSumErr2->data.F32[i-1]), sqrt(fluxSumErr2->data.F32[i])); + petArea = InterpolateValues (refRadius->data.F32[i-1], areaSum->data.F32[i-1], refRadius->data.F32[i], areaSum->data.F32[i], apRadius); + petApix = InterpolateValues (refRadius->data.F32[i-1], apixSum->data.F32[i-1], refRadius->data.F32[i], apixSum->data.F32[i], apRadius); + if (!isfinite(petFlux)) { + fprintf (stderr, "nan pet flux\n"); + } + break; + } + } + } + + // now measure the radii R90 and R50 where flux = 0.9 (or 0.5) * petFlux; + float flux90 = 0.9 * petFlux; + float flux50 = 0.5 * petFlux; + float R50 = NAN; + float R90 = NAN; + float R50err = NAN; + float R90err = NAN; + bool found50 = false; + bool found90 = false; + + // XXX use bisection to do this faster: + for (int i = 0; !(found50 && found90) && i < refRadius->n; i++) { + if (!found50 && (fluxSum->data.F32[i] > flux50)) { + if (i == 0) { + R50 = InterpolateValues (fluxSum->data.F32[i], refRadius->data.F32[i], fluxSum->data.F32[i+1], refRadius->data.F32[i+1], flux50); + R50err = InterpolateValuesErrX (fluxSum->data.F32[i], refRadius->data.F32[i], fluxSum->data.F32[i+1], refRadius->data.F32[i+1], flux50, sqrt(fluxSumErr2->data.F32[i]), sqrt(fluxSumErr2->data.F32[i+1])); + found50 = true; + } else { + R50 = InterpolateValues (fluxSum->data.F32[i-1], refRadius->data.F32[i-1], fluxSum->data.F32[i], refRadius->data.F32[i], flux50); + R50err = InterpolateValuesErrX (fluxSum->data.F32[i-1], refRadius->data.F32[i-1], fluxSum->data.F32[i], refRadius->data.F32[i], flux50, sqrt(fluxSumErr2->data.F32[i-1]), sqrt(fluxSumErr2->data.F32[i])); + found50 = true; + } + } + if (!found90 && (fluxSum->data.F32[i] > flux90)) { + if (i == 0) { + R90 = InterpolateValues (fluxSum->data.F32[i], refRadius->data.F32[i], fluxSum->data.F32[i+1], refRadius->data.F32[i+1], flux90); + R90err = InterpolateValuesErrX (fluxSum->data.F32[i], refRadius->data.F32[i], fluxSum->data.F32[i+1], refRadius->data.F32[i+1], flux90, sqrt(fluxSumErr2->data.F32[i]), sqrt(fluxSumErr2->data.F32[i+1])); + found90 = true; + } else { + R90 = InterpolateValues (fluxSum->data.F32[i-1], refRadius->data.F32[i-1], fluxSum->data.F32[i], refRadius->data.F32[i], flux90); + R90err = InterpolateValuesErrX (fluxSum->data.F32[i-1], refRadius->data.F32[i-1], fluxSum->data.F32[i], refRadius->data.F32[i], flux90, sqrt(fluxSumErr2->data.F32[i-1]), sqrt(fluxSumErr2->data.F32[i])); + found90 = true; + } + } + } + + + // XXX save flags (anyPetro, manyPetro) + source->extpars->petrosianRadius = petRadius; + source->extpars->petrosianFlux = petFlux; + source->extpars->petrosianR50 = R50; + source->extpars->petrosianR90 = R90; + source->extpars->petrosianFill = petApix / petArea; + + // XXX add the errors + source->extpars->petrosianRadiusErr = petRadiusErr; + source->extpars->petrosianFluxErr = petFluxErr; + source->extpars->petrosianR50Err = R50err; + source->extpars->petrosianR90Err = R90err; + + // fprintf (stderr, "source @ %f,%f\n", source->peak->xf, source->peak->yf); + psphotPetrosianVisualStats (binRad, binSB, refRadius, meanSB, petRatio, petRatioErr, fluxSum, petRadius, PETROSIAN_RATIO, petFlux, apRadius); + + psFree(fluxSum); + psFree(fluxSumErr2); + psFree(refRadius); + psFree(petRatio); + psFree(petRatioErr); + psFree(meanSB); + psFree(areaSum); + psFree(apixSum); + + return true; +} + +// Lagrange's form of the interpolating polynomial... +float InterpolateValuesQuadratic (float *Xin, float *Yin, float X) { + + float dx01 = Xin[0] - Xin[1]; + float dx02 = Xin[0] - Xin[2]; + float dx12 = Xin[1] - Xin[2]; + + float dx0 = X - Xin[0]; + float dx1 = X - Xin[1]; + float dx2 = X - Xin[2]; + + float y0 = Yin[0]*dx1*dx2/(dx01*dx02); + float y1 = Yin[1]*dx0*dx2/(dx01*dx12); // need - sign + float y2 = Yin[2]*dx0*dx1/(dx02*dx12); + + float Y = y0 - y1 + y2; + return Y; +} + +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X) { + float dydx = (Y1 - Y0) / (X1 - X0); + float Y = Y0 + dydx * (X - X0); + return Y; +} + +float InterpolateValuesErrX (float X0, float Y0, float X1, float Y1, float X, float dX0, float dX1) { + + float dydx = (Y1 - Y0) / (X1 - X0); + float dxdx = (X - X0) / (X1 - X0); + + float dY = sqrt(PS_SQR(dX1*dydx*dxdx) + PS_SQR(dX0*dydx*(dxdx - 1.0))); + return dY; +} + +float InterpolateValuesErrY (float X0, float Y0, float X1, float Y1, float X, float dY0, float dY1) { + + float dxdx = (X - X0) / (X1 - X0); + + float dY = sqrt(PS_SQR(dY1*dxdx) + PS_SQR(dY0*(1.0 - dxdx))); + return dY; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianStudy.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianStudy.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianStudy.c (revision 41083) @@ -0,0 +1,184 @@ +# include "psphotInternal.h" + +# define DX 512 +# define DY 512 + +// XXX add noise and seeing. +// XXX double check on sersic functional form +// XXX modify ratio if ratio > 1.0 (swap major and minor) + +pmPeak *psphotLocalPeak(pmReadout *readout, int Xo, int Yo); + +int main (int argc, char **argv) { + + pmErrorRegister(); // register psModule's error codes/messages + pmModelClassInit(); + + int N; + float Xo = 0.5*DX; + float Yo = 0.5*DY; + char *image = NULL; + pmSource *source = NULL; + + float peak = 1000.0; + float sigma = 2.0; // major axis size + float ARatio = 1.0; + float angle = 0.0; + float sersic = 0.5; + float skynoise = 0.0; + + if ((N = psArgumentGet (argc, argv, "-peak"))) { + psArgumentRemove (N, &argc, argv); + peak = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-sigma"))) { + psArgumentRemove (N, &argc, argv); + sigma = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-aratio"))) { + psArgumentRemove (N, &argc, argv); + ARatio = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-angle"))) { + psArgumentRemove (N, &argc, argv); + angle = PS_RAD_DEG*atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-sersic"))) { + psArgumentRemove (N, &argc, argv); + sersic = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-skynoise"))) { + psArgumentRemove (N, &argc, argv); + skynoise = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (N, &argc, argv); + pmVisualSetVisual(true); + } + if ((N = psArgumentGet (argc, argv, "-coords"))) { + psArgumentRemove (N, &argc, argv); + Xo = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + Yo = atof(argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-image"))) { + psArgumentRemove (N, &argc, argv); + image = psStringCopy (argv[N]); + psArgumentRemove (N, &argc, argv); + } + + if (argc != 1) { + fprintf (stderr, "USAGE: psphotPetrosianStudy\n"); + exit (2); + } + + // create a containing image & associated readout + pmReadout *readout = pmReadoutAlloc(NULL); + if (!image) { + readout->image = psImageAlloc(DX, DY, PS_TYPE_F32); + + // create a dummy variance, but don't populate -- it is not used by pmSourceMoments if the sigma parameters is set to 0.0 + readout->variance = psImageAlloc(DX, DY, PS_TYPE_F32); + + // create a model & associated source + pmModelType type = pmModelClassGetType("PS_MODEL_SERSIC"); + pmModel *model = pmModelAlloc(type); + + // set the model parameters + model->params->data.F32[PM_PAR_SKY] = 0.0; + model->params->data.F32[PM_PAR_I0] = peak; + model->params->data.F32[PM_PAR_XPOS] = Xo; + model->params->data.F32[PM_PAR_YPOS] = Yo; + + psEllipseAxes axes; + axes.major = sigma; + axes.minor = sigma*ARatio; + axes.theta = angle; + + psEllipseShape shape = psEllipseAxesToShape (axes); + + // XXX set the sigma with user input + model->params->data.F32[PM_PAR_SXX] = shape.sx * M_SQRT2; + model->params->data.F32[PM_PAR_SYY] = shape.sy * M_SQRT2; + model->params->data.F32[PM_PAR_SXY] = shape.sxy; + + if (model->params->n > 7) { + model->params->data.F32[PM_PAR_7] = sersic; + } + + // generate source container & populate image + source = pmSourceFromModel(model, readout, Xo, PM_SOURCE_TYPE_STAR); + + // generate the modelFlux + pmSourceCacheModel(source, 0); + + // instantiate the source + pmSourceAdd(source, PM_MODEL_OP_FUNC, 0); + + // XXX add noise here... + psphotSaveImage(NULL, readout->image, "sersic.fits"); + + } else { + psRegion full = psRegionSet(0,0,0,0); + psFits *fits = psFitsOpen(image, "r"); + readout->image = psFitsReadImage(fits, full, 0); + + source = pmSourceAlloc(); + source->peak = psphotLocalPeak(readout, Xo, Yo); + pmSourceDefinePixels (source, readout, Xo, Yo, 128); + } + + psphotPetrosianProfile (readout, source, skynoise); + + psFree (source); + + exit (0); +} + +// Xo, Yo are in parent coords +pmPeak *psphotLocalPeak(pmReadout *readout, int Xo, int Yo) { + + int Xp = Xo; + int Yp = Yo; + float peakFlux = readout->image->data.F32[Yp][Xp]; + + // find local peak within +/- 3 pix of the given coordinate + for (int iy = Yo - 3; iy <= Yo + 3; iy++) { + for (int ix = Xo - 3; ix <= Xo + 3; ix++) { + if (peakFlux < readout->image->data.F32[iy][ix]) { + Xp = ix; + Yp = iy; + peakFlux = readout->image->data.F32[Yp][Xp]; + } + } + } + + pmPeak *peak = pmPeakAlloc(Xp, Yp, peakFlux, PM_PEAK_LONE); + + // calculate fractional peak position relative to Xp,Yp + psPolynomial2D *bicube = psImageBicubeFit (readout->image, Xp, Yp); + psPlane min = psImageBicubeMin (bicube); + psFree (bicube); + + // if min point is too deviant, use the peak value + if ((fabs(min.x) < 1.5) && (fabs(min.y) < 1.5)) { + peak->xf = min.x + Xp; + peak->yf = min.y + Yp; + + // xf,yf must land on image with 0 pixel border + peak->xf = PS_MAX (PS_MIN (peak->xf, readout->image->numCols - 1), readout->image->col0); + peak->yf = PS_MAX (PS_MIN (peak->yf, readout->image->numRows - 1), readout->image->row0); + } else { + peak->xf = Xp; + peak->yf = Yp; + } + + return peak; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianVisual.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianVisual.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotPetrosianVisual.c (revision 41083) @@ -0,0 +1,403 @@ +# include "psphotInternal.h" + +// this function displays representative images as the psphot analysis progresses: +// 0 : image, 1 : variance +// 0 : backsub, 1 : variance, 2 : backgnd +// 0 : backsub, 1 : variance, 2 : signif +// (overlay peaks on images) +// (overlay footprints on images) +// (overlay moments on images) +// (overlay rough class on images) +// 0 : backsub, 1 : psfpos, 2: psfsub +// 0 : backsub, 1 : lin_resid, 2: psfsub + +# if (HAVE_KAPA) +# include + +// functions used to visualize the analysis as it goes +// these are invoked by the -visual options + +static int kapa = -1; +static int kapa2 = -1; + +// if no valid data is supplied (NULL or n <- 0), leave limits as they were +bool pmVisualLimitsFromVectors (Graphdata *graphdata, psVector *xVec, psVector *yVec) { + + if (xVec && xVec->n > 0) { + graphdata->xmin = graphdata->xmax = xVec->data.F32[0]; + for (int i = 1; i < xVec->n; i++) { + if (!isfinite(xVec->data.F32[i])) continue; + graphdata->xmin = PS_MIN (graphdata->xmin, xVec->data.F32[i]); + graphdata->xmax = PS_MAX (graphdata->xmax, xVec->data.F32[i]); + } + float range = graphdata->xmax - graphdata->xmin; + graphdata->xmax += 0.05*range; + graphdata->xmin -= 0.05*range; + } + if (yVec && yVec->n > 0) { + graphdata->ymin = graphdata->ymax = yVec->data.F32[0]; + for (int i = 1; i < yVec->n; i++) { + if (!isfinite(yVec->data.F32[i])) continue; + graphdata->ymin = PS_MIN (graphdata->ymin, yVec->data.F32[i]); + graphdata->ymax = PS_MAX (graphdata->ymax, yVec->data.F32[i]); + } + float range = graphdata->ymax - graphdata->ymin; + graphdata->ymax += 0.05*range; + graphdata->ymin -= 0.05*range; + } + return true; +} + +bool psphotPetrosianVisualProfileByAngle (psVector *radius, psVector *flux) { + + Graphdata graphdata; + + if (!pmVisualTestLevel("psphot.petro.byangle", 2)) return true; + + if (kapa2 == -1) { + kapa2 = KapaOpenNamedSocket ("kapa", "psphot:plots"); + if (kapa2 == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + return false; + } + } + + KapaClearPlots (kapa2); + KapaInitGraph (&graphdata); + KapaSetFont (kapa2, "courier", 14); + + pmVisualLimitsFromVectors (&graphdata, radius, flux); + KapaSetLimits (kapa2, &graphdata); + + KapaBox (kapa2, &graphdata); + KapaSendLabel (kapa2, "radius", KAPA_LABEL_XM); + KapaSendLabel (kapa2, "flux", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.style = 2; + graphdata.ptype = 0; + graphdata.size = 1.0; + KapaPrepPlot (kapa2, radius->n, &graphdata); + KapaPlotVector (kapa2, radius->n, radius->data.F32, "x"); + KapaPlotVector (kapa2, radius->n, flux->data.F32, "y"); + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[c]ontinue? "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + return true; +} + +bool psphotPetrosianVisualProfileRadii (psVector *radius, psVector *flux, psVector *radiusBin, psVector *fluxBin, float peakFlux, float RadiusRef) { + + float FluxRef = 500.0; + + Graphdata graphdata; + + if (!pmVisualTestLevel("psphot.petro.radii", 2)) return true; + + if (kapa == -1) { + kapa = KapaOpenNamedSocket ("kapa", "psphot:plots"); + if (kapa == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + return false; + } + } + + KapaClearPlots (kapa); + KapaInitGraph (&graphdata); + KapaSetFont (kapa, "courier", 14); + + graphdata.ymax = +1.05*peakFlux; + graphdata.ymin = -0.05*peakFlux; + pmVisualLimitsFromVectors (&graphdata, radius, NULL); + KapaSetLimits (kapa, &graphdata); + + KapaBox (kapa, &graphdata); + KapaSendLabel (kapa, "radius", KAPA_LABEL_XM); + KapaSendLabel (kapa, "flux", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.style = 2; + graphdata.ptype = 0; + graphdata.size = 1.0; + KapaPrepPlot (kapa, radius->n, &graphdata); + KapaPlotVector (kapa, radius->n, radius->data.F32, "x"); + KapaPlotVector (kapa, radius->n, flux->data.F32, "y"); + + // do this with log-r, log-flux? + graphdata.color = KapaColorByName ("blue"); + graphdata.style = 2; + graphdata.ptype = 2; + graphdata.size = 2.0; + KapaPrepPlot (kapa, radiusBin->n, &graphdata); + KapaPlotVector (kapa, radiusBin->n, radiusBin->data.F32, "x"); + KapaPlotVector (kapa, radiusBin->n, fluxBin->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.style = 2; + graphdata.ptype = 7; + graphdata.size = 3.0; + KapaPrepPlot (kapa, 1, &graphdata); + KapaPlotVector (kapa, 1, &RadiusRef, "x"); + KapaPlotVector (kapa, 1, &FluxRef, "y"); + + fprintf (stderr, "radius: %f\n", RadiusRef); + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[c]ontinue? "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + return true; +} + +bool psphotPetrosianVisualStats (psVector *radBin, psVector *fluxBin, + psVector *refRadius, psVector *meanSB, + psVector *petRatio, psVector *petRatioErr, + psVector *fluxSum, + float petRadius, float ratioForRadius, + float petFlux, float radiusForFlux) +{ + Graphdata graphdata; + KapaSection section; + + if (!pmVisualTestLevel("psphot.petro.stats", 2)) return true; + + if (kapa2 == -1) { + kapa2 = KapaOpenNamedSocket ("kapa", "psphot:stats"); + if (kapa2 == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + return false; + } + } + + KapaClearPlots (kapa2); + KapaInitGraph (&graphdata); + KapaSetFont (kapa2, "courier", 14); + + // radius vs flux + // radius vs mean SB + // radius vs petRatio + + // *** section 1: radius vs mean SB + section.dx = 1.00; + section.dy = 0.33; + section.x = 0.00; + section.y = 0.00; + section.name = psStringCopy ("meanSB"); + KapaSetSection (kapa2, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + pmVisualLimitsFromVectors (&graphdata, radBin, fluxBin); + KapaSetLimits (kapa2, &graphdata); + + KapaBox (kapa2, &graphdata); + KapaSendLabel (kapa2, "radius", KAPA_LABEL_XM); + KapaSendLabel (kapa2, "mean SB", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.style = 2; + graphdata.ptype = 0; + graphdata.size = 1.0; + KapaPrepPlot (kapa2, radBin->n, &graphdata); + KapaPlotVector (kapa2, radBin->n, radBin->data.F32, "x"); + KapaPlotVector (kapa2, radBin->n, fluxBin->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.style = 2; + graphdata.ptype = 1; + graphdata.size = 2.0; + KapaPrepPlot (kapa2, refRadius->n, &graphdata); + KapaPlotVector (kapa2, refRadius->n, refRadius->data.F32, "x"); + KapaPlotVector (kapa2, refRadius->n, meanSB->data.F32, "y"); + + // *** section 2: radius vs petrosian ratio + section.dx = 1.00; + section.dy = 0.33; + section.x = 0.00; + section.y = 0.33; + section.name = psStringCopy ("ratio"); + KapaSetSection (kapa2, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.ymax = +1.05; + graphdata.ymin = -0.05; + pmVisualLimitsFromVectors (&graphdata, radBin, NULL); + KapaSetLimits (kapa2, &graphdata); + + KapaBox (kapa2, &graphdata); + KapaSendLabel (kapa2, "radius", KAPA_LABEL_XM); + KapaSendLabel (kapa2, "ratio", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.style = 2; + graphdata.ptype = 0; + graphdata.size = 1.0; + graphdata.etype = 0x01; + KapaPrepPlot (kapa2, refRadius->n, &graphdata); + KapaPlotVector (kapa2, refRadius->n, refRadius->data.F32, "x"); + KapaPlotVector (kapa2, refRadius->n, petRatio->data.F32, "y"); + KapaPlotVector (kapa2, refRadius->n, petRatioErr->data.F32, "dym"); + KapaPlotVector (kapa2, refRadius->n, petRatioErr->data.F32, "dyp"); + graphdata.etype = 0; + + graphdata.color = KapaColorByName ("red"); + graphdata.style = 2; + graphdata.ptype = 2; + graphdata.size = 2.0; + KapaPrepPlot (kapa2, 1, &graphdata); + KapaPlotVector (kapa2, 1, &petRadius, "x"); + KapaPlotVector (kapa2, 1, &ratioForRadius, "y"); + + // *** section 3: radius vs integrated flux + section.dx = 1.00; + section.dy = 0.33; + section.x = 0.00; + section.y = 0.66; + section.name = psStringCopy ("flux"); + KapaSetSection (kapa2, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + pmVisualLimitsFromVectors (&graphdata, radBin, fluxSum); + KapaSetLimits (kapa2, &graphdata); + + KapaBox (kapa2, &graphdata); + KapaSendLabel (kapa2, "radius", KAPA_LABEL_XM); + KapaSendLabel (kapa2, "integrated flux", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.style = 2; + graphdata.ptype = 0; + graphdata.size = 1.0; + KapaPrepPlot (kapa2, refRadius->n, &graphdata); + KapaPlotVector (kapa2, refRadius->n, refRadius->data.F32, "x"); + KapaPlotVector (kapa2, refRadius->n, fluxSum->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 2; + graphdata.style = 2; + graphdata.size = 2.0; + KapaPrepPlot (kapa2, 1, &graphdata); + KapaPlotVector (kapa2, 1, &radiusForFlux, "x"); + KapaPlotVector (kapa2, 1, &petFlux, "y"); + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[c]ontinue? "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + return true; +} + +bool psphotPetrosianVisualEllipticalContour (pmSourceRadialFlux *radFlux, pmSourceExtendedPars *extpars) { + + Graphdata graphdata; + + if (!pmVisualTestLevel("psphot.petro.ellipse", 2)) return true; + + if (kapa == -1) { + kapa = KapaOpenNamedSocket ("kapa", "psphot:plots"); + if (kapa == -1) { + fprintf (stderr, "failure to open kapa; visual mode disabled\n"); + pmVisualSetVisual(false); + return false; + } + } + + KapaClearPlots (kapa); + KapaInitGraph (&graphdata); + KapaSetFont (kapa, "courier", 14); + + psVector *theta = radFlux->theta; + psVector *radius = radFlux->isophotalRadii; + + // find Rmin and Rmax for the initial guess + float Rmin = radius->data.F32[0]; + float Rmax = radius->data.F32[0]; + + psVector *Rx = psVectorAlloc(radius->n, PS_TYPE_F32); + psVector *Ry = psVectorAlloc(radius->n, PS_TYPE_F32); + + for (int i = 0; i < theta->n; i++) { + Rx->data.F32[i] = radius->data.F32[i]*cos(theta->data.F32[i]); + Ry->data.F32[i] = radius->data.F32[i]*sin(theta->data.F32[i]); + + // check the radius range + Rmin = MIN (Rmin, radius->data.F32[i]); + Rmax = MAX (Rmax, radius->data.F32[i]); + } + + psVector *rx = psVectorAlloc(361, PS_TYPE_F32); + psVector *ry = psVectorAlloc(361, PS_TYPE_F32); + + float epsilon = extpars->axes.minor / extpars->axes.major; + + for (int i = 0; i < 361; i++) { + + float alpha = PS_RAD_DEG * i; + + float cs_alpha = cos(alpha); + float sn_alpha = sin(alpha); + + float cs_phi = cos(alpha - extpars->axes.theta); + float sn_phi = sin(alpha - extpars->axes.theta); + + float r = 1.0 / sqrt(SQ(sn_phi) + SQ(epsilon*cs_phi)); + + // generate the model fit here + rx->data.F32[i] = extpars->axes.minor * cs_alpha * r; + ry->data.F32[i] = extpars->axes.minor * sn_alpha * r; + } + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = -1.1*Rmax; + graphdata.ymin = -1.1*Rmax; + graphdata.xmax = +1.1*Rmax; + graphdata.ymax = +1.1*Rmax; + KapaSetLimits (kapa, &graphdata); + + KapaBox (kapa, &graphdata); + KapaSendLabel (kapa, "R_x", KAPA_LABEL_XM); + KapaSendLabel (kapa, "R_y", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("red"); + graphdata.style = 2; + graphdata.ptype = 2; + graphdata.size = 1.0; + KapaPrepPlot (kapa, Rx->n, &graphdata); + KapaPlotVector (kapa, Rx->n, Rx->data.F32, "x"); + KapaPlotVector (kapa, Rx->n, Ry->data.F32, "y"); + + graphdata.color = KapaColorByName ("black"); + graphdata.style = 0; + graphdata.ptype = 0; + graphdata.size = 1.0; + KapaPrepPlot (kapa, rx->n, &graphdata); + KapaPlotVector (kapa, rx->n, rx->data.F32, "x"); + KapaPlotVector (kapa, rx->n, ry->data.F32, "y"); + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[c]ontinue? "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + return true; +} + +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialApertures.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialApertures.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialApertures.c (revision 41083) @@ -0,0 +1,572 @@ +# include "psphotInternal.h" + +bool psphotRadialAperturesSortFlux (psVector *radius, psVector *pixFlux, psVector *pixVar); + +// this function measures the radial aperture fluxes for the set of readouts. this function +// may be called multiple times, presumably for different versions of PSF-matched or unmatched images. + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotRadialApertures (pmConfig *config, const pmFPAview *view, const char *filerule, int entry) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Radial Apertures ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full non-linear fits / extended source analysis? + if (!psMetadataLookupBool (&status, recipe, "RADIAL_APERTURES")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping radial apertures\n"); + return true; + } + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + if (!psphotRadialAperturesReadout (config, view, filerule, i, recipe, entry)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on measure extended source aperture-like parameters for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// these values are used by all threads repeatedly (and are not modified) +static psVector *aperRadii = NULL; +static psVector *aperRadii2 = NULL; +static float outerRadius = NAN; +static float SN_LIM = NAN; +static int RADIAL_AP_MIN = 5; +static psImageMaskType maskVal = 0; + +// aperture-like measurements for extended sources +// flux in simple, circular apertures +// 'entry' tells us which of the matched-PSF images we are working on (0 == unmatched image, also non-stack psphot) +bool psphotRadialAperturesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, int entry) { + + bool status; + int Nradial = 0; + + // perform full non-linear fits / extended source analysis? + if (!psMetadataLookupBool (&status, recipe, "RADIAL_APERTURES")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping radial apertures\n"); + return true; + } + + psTimerStart ("psphot.radial"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psLogMsg ("psphot", PS_LOG_DETAIL, "skipping radial aptertures for input file %d", index); + return true; + } + + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source size"); + return true; + } + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // aperRadii stores the upper bounds of the annuli + // XXX keep the same name here as for the petrosian / elliptical apertures? + aperRadii = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + psAssert (aperRadii, "annular bins (RADIAL.ANNULAR.BINS.UPPER) are not defined in the recipe"); + psAssert (aperRadii->n, "no valid annular bins (RADIAL.ANNULAR.BINS.UPPER) are define"); + + outerRadius = aperRadii->data.F32[aperRadii->n - 1]; + + // save the R^2 values as well for quicker comparison + aperRadii2 = psVectorAlloc(aperRadii->n, PS_TYPE_F32); + for (int i = 0; i < aperRadii->n; i++) { + aperRadii2->data.F32[i] = PS_SQR(aperRadii->data.F32[i]); + } + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // S/N limit to perform full non-linear fits + SN_LIM = psMetadataLookupF32 (&status, recipe, "RADIAL_APERTURES_SN_LIM"); + + // S/N limit to perform full non-linear fits + RADIAL_AP_MIN = psMetadataLookupS32 (&status, recipe, "RADIAL.ANNULAR.BINS.MIN"); + if (!status) { + RADIAL_AP_MIN = 5; + } + + // source analysis is done in S/N order (brightest first) + // XXX are we getting the objects out of order? does it matter? + sources = psArraySort (sources, pmSourceSortByFlux); + + // XXX make this consistent with entry 0 == unmatched + int nEntry = 1; + psVector *fwhmValues = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + if (fwhmValues) { + psAssert (entry < fwhmValues->n, "inconsistent matched-PSF entry"); + nEntry = fwhmValues->n; + } + if (entry > 0) { + psLogMsg ("psphot", PS_LOG_DETAIL, "Radial Apertures for matched image %s : PSF FWHM = %f pixels\n", file->name, fwhmValues->data.F32[entry]); + } else { + psLogMsg ("psphot", PS_LOG_DETAIL, "Radial Apertures for unmatched image %s\n", file->name); + } + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion *AnalysisRegion = psRegionAlloc(0,0,0,0); + *AnalysisRegion = psRegionForImage (readout->image, psRegionFromString (region)); + if (psRegionIsNaN (*AnalysisRegion)) psAbort("analysis region mis-defined"); + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_RADIAL_APERTURES"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, AnalysisRegion); + PS_ARRAY_ADD_SCALAR(job->args, entry, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, nEntry, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nradial + + // set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } +# else + if (!psphotRadialApertures_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } + psScalar *scalar = NULL; + scalar = job->args->data[5]; + Nradial += scalar->data.S32; + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + psFree(AnalysisRegion); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[5]; + Nradial += scalar->data.S32; + } + psFree(job); + } + } + psFree (cellGroups); + psFree(AnalysisRegion); + psFree (aperRadii2); + + psLogMsg ("psphot", PS_LOG_WARN, "radial source apertures: %f sec for %d objects\n", psTimerMark ("psphot.radial"), Nradial); + return true; +} + +bool psphotRadialApertures_Threaded (psThreadJob *job) { + + int Nradial = 0; + + // arguments: readout, sources, models, region, psfSize, maskVal, markVal + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psRegion *region = job->args->data[2]; + int entry = PS_SCALAR_VALUE(job->args->data[3],S32); // which psf-matched image are we working on? (0 == unmatched) + int nEntry = PS_SCALAR_VALUE(job->args->data[4],S32); // total number of psf-matched images + 1 unmatched + + // storage for the derived pixel values (these are passed into psphotRadialApertureSource) + psVector *pixRadius2 = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *pixFlux = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *pixVar = psVectorAllocEmpty(100, PS_TYPE_F32); + + // choose the sources of interest + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // if we have checked the source validity on the basis of the object set, then + // we either skip these tests below or we skip the source completely + if (source->tmpFlags & PM_SOURCE_TMPF_RADIAL_SKIP) continue; + if (source->tmpFlags & PM_SOURCE_TMPF_RADIAL_KEEP) goto keepSource; + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // XXX measure radial apertures even for saturated stars + // if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + // limit selection to some SN limit + assert (source->peak); // how can a source not have a peak? + if (sqrt(source->peak->detValue) < SN_LIM) continue; + + // limit selection by analysis region + if (source->peak->x < region->x0) continue; + if (source->peak->y < region->y0) continue; + if (source->peak->x > region->x1) continue; + if (source->peak->y > region->y1) continue; + + keepSource: + + // allocate pmSourceExtendedParameters, if not already defined + // XXX check that nPSFsizes is consistent with targets + if (source->parent) { + if (!source->parent->radialAper) { + source->parent->radialAper = psArrayAlloc(nEntry); + } + } else { + if (!source->radialAper) { + source->radialAper = psArrayAlloc(nEntry); + } + } + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + Nradial ++; + + if (!psphotRadialApertureSource (source, readout, entry, pixRadius2, pixFlux, pixVar)) { + psTrace ("psphot", 5, "failed to extract radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + } else { + source->mode |= PM_SOURCE_MODE_RADIAL_FLUX; + } + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + psScalar *scalar = job->args->data[5]; + scalar->data.S32 = Nradial; + + psFree (pixRadius2); + psFree (pixFlux); + psFree (pixVar); + + return true; +} + +bool psphotRadialApertureSource (pmSource *source, pmReadout *readout, int entry, psVector *pixRadius2, psVector *pixFlux, psVector *pixVar) { + + // if we are a child source, save the results to the parent source radial aperture array + psArray *radialAperSet = source->radialAper; + if (source->parent) { + radialAperSet = source->parent->radialAper; + } + psAssert(radialAperSet, "this should be defined before calling"); + psAssert(radialAperSet->data[entry] == NULL, "why is this already defined?"); + + pmSourceRadialApertures *radialAper = pmSourceRadialAperturesAlloc (); + radialAperSet->data[entry] = radialAper; + + // find the largest aperture of interest (use only apertures with inner radii <= + // source->skyRadius, as long as i >= RADIAL_AP_MIN + int lastAp = aperRadii->n; + for (int i = RADIAL_AP_MIN; i < aperRadii->n; i++) { + if (aperRadii->data.F32[i] < source->skyRadius) continue; // measure out to this radius + lastAp = i + 1; + break; + } + + // outer-most radius for initial truncation + float Rmax = aperRadii->data.F32[lastAp - 1]; + float Rmax2 = PS_SQR(Rmax); + + // in this function, the operatins are relative to the full image (readout->image, etc) + + float xCM = NAN, yCM = NAN; + if (pmSourcePositionUseMoments(source)) { + xCM = source->moments->Mx; // index coord of peak in readout + yCM = source->moments->My; // index coord of peak in readout + } else { + xCM = source->peak->xf; // index coord of peak in readout + yCM = source->peak->yf; // index coord of peak in readout + } + + int Nx = readout->image->numCols; + int Ny = readout->image->numRows; + + pixRadius2->n = 0; + pixFlux->n = 0; + pixVar->n = 0; + + // one pass through the pixels to select the valid pixels and calculate R^2 + for (int iy = -Rmax; iy < Rmax + 1; iy++) { + + float yDiff = iy + 0.5 + yCM; // y-coordinate at this offse + int yPix = (int) yDiff; + + if (yPix < 0) continue; + if (yPix > Ny - 1) continue; + if (fabs(iy) > Rmax) continue; + + float *vPix = readout->image->data.F32[yPix]; + float *vWgt = readout->variance->data.F32[yPix]; + psImageMaskType *vMsk = readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[yPix]; + + for (int ix = -Rmax; ix < Rmax + 1; ix++) { + + float xDiff = ix + 0.5 + xCM; // x-coordinate at this offse + int xPix = (int) xDiff; + + if (xPix < 0) continue; + if (xPix > Nx - 1) continue; + if (fabs(ix) > Rmax) continue; + + if (vMsk[xPix] & maskVal) continue; + if (isnan(vPix[xPix])) continue; + + // radius is just a function of (xDiff, yDiff) + float r2 = PS_SQR(ix) + PS_SQR(iy); + if (r2 > Rmax2) continue; + + psVectorAppend(pixRadius2, r2); + psVectorAppend(pixFlux, vPix[xPix]); + psVectorAppend(pixVar, vWgt[xPix]); + } + } + + psVector *flux = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fluxErr = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fluxStd = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fill = psVectorAlloc(aperRadii->n, PS_TYPE_F32); // surface brightness of radial bin + + // init the apertures of interest to 0.0, the rest go to NAN + psVectorInit (flux, 0.0); + psVectorInit (fluxStd, 0.0); + psVectorInit (fluxErr, 0.0); + psVectorInit (fill, 0.0); + for (int i = lastAp; i < flux->n; i++) { + flux->data.F32[i] = NAN; + fluxStd->data.F32[i] = NAN; + fluxErr->data.F32[i] = NAN; + fill->data.F32[i] = NAN; + } + + float *rPix2 = pixRadius2->data.F32; + for (int i = 0; i < pixRadius2->n; i++, rPix2++) { + + int j = 0; + float *aRad2 = aperRadii2->data.F32; + for (; (*aRad2 < *rPix2) && (j < lastAp); j++, aRad2++); + + // XXX I can speed this up by only saving this single aperture + for (; j < lastAp; j++, aRad2++) { + flux->data.F32[j] += pixFlux->data.F32[i]; + fluxStd->data.F32[j] += PS_SQR(pixFlux->data.F32[i]); + fluxErr->data.F32[j] += pixVar->data.F32[i]; + fill->data.F32[j] += 1.0; + } + } + + /* for each radial bin, R(i), we measure: + 1) the flux within that aperture: F(i) = \sum_{r_jdata.F32[i]; + + int nPix = fill->data.F32[i]; + float SBmean = flux->data.F32[i] / nPix; + float SBstdv = sqrt((fluxStd->data.F32[i] / nPix) - PS_SQR(SBmean)); + + // XXX report the total flux or the mask-corrected flux? + // flux->data.F32[i] = SBmean * Area; + // fluxErr->data.F32[i] = sqrt(fluxErr->data.F32[i]) * Area / Pinx; + + fluxErr->data.F32[i] = sqrt(fluxErr->data.F32[i]); + fluxStd->data.F32[i] = SBstdv * Area; + fill->data.F32[i] /= Area; + + psTrace ("psphot", 5, "radial bins: %3d %5.1f : %8.1f +/- %7.2f : %8.1f +/- %8.1f : %4.2f %6.1f\n", + i, aperRadii->data.F32[i], flux->data.F32[i], fluxErr->data.F32[i], SBmean, SBstdv, fill->data.F32[i], Area); + } + +# if (1) + radialAper->flux = flux; + radialAper->fluxStdev = fluxStd; + radialAper->fluxErr = fluxErr; + radialAper->fill = fill; +# else + // XXX TEST + psFree(flux); + psFree(fluxStd); + psFree(fluxErr); + psFree(fill); +# endif + + return true; +} + +/*** below is a test to use a sort to speed this up, not very successfully ***/ + +static int nCalls = 0; +static int nPass = 0; +static int nPix = 0; + +bool psphotRadialApertureSource_With_Sort (pmSource *source, psMetadata *recipe, psImageMaskType maskVal, const psVector *radMax, int entry) { + + psAssert(source->radialAper->data[entry] == NULL, "why is this already defined?"); + + pmSourceRadialApertures *radialAper = pmSourceRadialAperturesAlloc (); + source->radialAper->data[entry] = radialAper; + + psVector *pixRadius = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *pixFlux = psVectorAllocEmpty(100, PS_TYPE_F32); + psVector *pixVar = psVectorAllocEmpty(100, PS_TYPE_F32); + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + + // 0.5 PIX: get pixRadius as a function of pixel coord + float x = ix + 0.5 - source->peak->xf + source->pixels->col0; + float y = iy + 0.5 - source->peak->yf + source->pixels->row0; + + float r = hypot(x, y); + + psVectorAppend(pixRadius, r); + psVectorAppend(pixFlux, source->pixels->data.F32[iy][ix]); + psVectorAppend(pixVar, source->variance->data.F32[iy][ix]); + nPix ++; + // if (nPix % 10000 == 0) {fprintf (stderr, "?");} + } + } + psphotRadialAperturesSortFlux(pixRadius, pixFlux, pixVar); + + psVector *flux = psVectorAllocEmpty(radMax->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fluxErr = psVectorAllocEmpty(radMax->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *fill = psVectorAllocEmpty(radMax->n, PS_TYPE_F32); // surface brightness of radial bin + + psVectorInit (flux, 0.0); + psVectorInit (fluxErr, 0.0); + psVectorInit (fill, 0.0); + + float fluxSum = 0.0; + float varSum = 0.0; + int nPixSum = 0; + + bool done = false; + int nOut = 0; + float Rmax = radMax->data.F32[nOut]; + + // XXX assume (or enforce) that the bins are contiguous and non-overlapping (Rmax[i] = Rmin[i+1]) + for (int i = 0; !done && (i < pixRadius->n); i++) { + if (pixRadius->data.F32[i] > Rmax) { + // calculate the total flux for bin 'nOut' + float Area = M_PI*PS_SQR(Rmax); + flux->data.F32[nOut] = fluxSum; + fluxErr->data.F32[nOut] = sqrt(varSum); + fill->data.F32[nOut] = nPixSum / Area; + + psTrace ("psphot", 5, "radial bins: %3d %5.1f : %8.1f +/- %7.2f : %4.2f %6.1f\n", + nOut, radMax->data.F32[nOut], flux->data.F32[nOut], fluxErr->data.F32[nOut], fill->data.F32[nOut], Area); + + nPass ++; + // if (nPass % 1000 == 0) {fprintf (stderr, "!");} + + nOut ++; + if (nOut >= radMax->n) break; + Rmax = radMax->data.F32[nOut]; + } + fluxSum += pixFlux->data.F32[i]; + varSum += pixVar->data.F32[i]; + nPixSum ++; + } + flux->n = fluxErr->n = fill->n = nOut; + + radialAper->flux = flux; + radialAper->fluxErr = fluxErr; + radialAper->fill = fill; + + psFree (pixRadius); + psFree (pixFlux); + psFree (pixVar); + + nCalls ++; + // if (nCalls % 100 == 0) {fprintf (stderr, "*");} + return true; +} + +// *** pmSourceRadialProfileSortPair is a utility function for sorting a pair of vectors +# define COMPARE_VECT(A,B) (radius->data.F32[A] < radius->data.F32[B]) +# define SWAP_VECT(TYPE,A,B) { \ + float tmp; \ + if (A != B) { \ + tmp = radius->data.F32[A]; \ + radius->data.F32[A] = radius->data.F32[B]; \ + radius->data.F32[B] = tmp; \ + tmp = pixFlux->data.F32[A]; \ + pixFlux->data.F32[A] = pixFlux->data.F32[B]; \ + pixFlux->data.F32[B] = tmp; \ + tmp = pixVar->data.F32[A]; \ + pixVar->data.F32[A] = pixVar->data.F32[B]; \ + pixVar->data.F32[B] = tmp; \ + } \ + } + +bool psphotRadialAperturesSortFlux (psVector *radius, psVector *pixFlux, psVector *pixVar) { + + // sort the vector set by the radius + PSSORT (radius->n, COMPARE_VECT, SWAP_VECT, NONE); + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialAperturesByObject.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialAperturesByObject.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialAperturesByObject.c (revision 41083) @@ -0,0 +1,171 @@ +# include "psphotInternal.h" + +// aperture-like measurements for extended sources +// flux in simple, circular apertures + +// **** it looks like this function will re-point the source pixels at the specified FILERULE +// **** I need to distinguish PSF-matched images from raw +// **** save (somewhere) the PSF-matched PSF values + +// this function measures the radial aperture fluxes for the set of readouts. this function +// may be called multiple times (presumably with different matched PSF sizes). we must have +// already added an entry to the readout->analysis identifying the FWHM of this version. + +bool psphotRadialAperturesByObject (pmConfig *config, psArray *objects, const pmFPAview *view, const char *filerule, int nMatchedPSF) { + + bool status; + int Nradial = 0; + + psTimerStart ("psphot.radial"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // perform full non-linear fits / extended source analysis? + if (!psMetadataLookupBool (&status, recipe, "RADIAL_APERTURES")) { + psLogMsg ("psphot", PS_LOG_INFO, "skipping radial apertures\n"); + return true; + } + + // number of images used to define sources + int nImages = psphotFileruleCount(config, filerule); + + // radMax stores the upper bounds of the annuli + // XXX keep the same name here as for the petrosian / elliptical apertures? + psVector *radMax = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + psAssert (radMax, "annular bins (RADIAL.ANNULAR.BINS.UPPER) are not defined in the recipe"); + psAssert (radMax->n, "no valid annular bins (RADIAL.ANNULAR.BINS.UPPER) are define"); + float outerRadius = radMax->data.F32[radMax->n - 1]; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // S/N limit to perform full non-linear fits + float SN_LIM = psMetadataLookupF32 (&status, recipe, "RADIAL_APERTURES_SN_LIM"); + + // how many target PSFs do we want? + int nPSFsizes = 0; + { + psMetadataLookupF32 (&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); + if (status) { + nPSFsizes = 1; + } else { + psVector *fwhmValues = psMetadataLookupVector(&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); // Magnitude offsets + psAssert (status, "missing psphot recipe value PSPHOT.STACK.TARGET.PSF.FWHM"); + nPSFsizes = fwhmValues->n; + } + } + + // source analysis is done in S/N order (brightest first) + objects = psArraySort (objects, pmPhotObjSortByFlux); + + // generate look-up arrays for readouts + psArray *readouts = psArrayAlloc(nImages); + for (int i = 0; i < nImages; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + psVector *fwhmValues = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + if (!fwhmValues) { + psError (PSPHOT_ERR_CONFIG, true, "convolved or measured FWHM is not defined for this readout"); + return false; + } + if (fwhmValues->n != nMatchedPSF + 1) { + psError (PSPHOT_ERR_CONFIG, true, "convolved or measured FWHM sequence is inconsistent this readout"); + return false; + } + psLogMsg ("psphot", PS_LOG_DETAIL, "PSF FWHM of %s : %f pixels\n", file->name, fwhmValues->data.F32[nMatchedPSF]); + + readouts->data[i] = psMemIncrRefCounter(readout); + } + + // process the objects in order. + for (int i = 0; i < objects->n; i++) { + pmPhotObj *object = objects->data[i]; + if (!object) continue; + if (!object->sources) continue; + + // choose the sources of interest + for (int j = 0; j < object->sources->n; j++) { + + pmSource *source = object->sources->data[j]; + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + // limit selection to some SN limit + assert (source->peak); // how can a source not have a peak? + if (sqrt(source->peak->detValue) < SN_LIM) continue; + + int index = source->imageID; + if (index >= readouts->n) continue; // skip the sources generated by the chisq image + pmReadout *readout = readouts->data[index]; + + // psLogMsg("psphot", PS_LOG_INFO, "radial apertures for %d", index); + // psphotVisualShowImage(readout); + + // allocate pmSourceExtendedParameters, if not already defined + if (source->parent) { + if (!source->parent->radialAper) { + source->parent->radialAper = psArrayAlloc(nPSFsizes); + } + } else { + if (!source->radialAper) { + source->radialAper = psArrayAlloc(nPSFsizes); + } + } + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + + // we need to change the view for the radial aperture analysis, but we want to recover exactly + // the original view; the following elements get destroyed by pmSourceRedefinePixels so save them: + psImage *oldMaskObj = psMemIncrRefCounter(source->maskObj); + psImage *oldModelFlux = psMemIncrRefCounter(source->modelFlux); + psImage *oldPSFimage = psMemIncrRefCounter(source->psfImage); + psRegion oldRegion = source->region; + + Nradial ++; + + // psLogMsg("psphot", PS_LOG_INFO, "radial apertures for %d", index); + // psphotVisualShowImage(readout); + + // force source image to be a bit larger... + pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, outerRadius + 2); + + if (!psphotRadialApertureSource (source, recipe, maskVal, radMax, nMatchedPSF)) { + psTrace ("psphot", 5, "failed to extract radial profile for source at %7.1f, %7.1f", source->moments->Mx, source->moments->My); + } else { + source->mode |= PM_SOURCE_MODE_RADIAL_FLUX; + if (source->parent) { + source->parent->mode |= PM_SOURCE_MODE_RADIAL_FLUX; + } + } + + pmSourceRedefinePixelsByRegion (source, readout, oldRegion); + psFree(source->maskObj); source->maskObj = oldMaskObj; + psFree(source->modelFlux); source->modelFlux = oldModelFlux; + psFree(source->psfImage); source->psfImage = oldPSFimage; + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + } + + psLogMsg ("psphot", PS_LOG_INFO, "radial source apertures: %f sec for %d objects\n", psTimerMark ("psphot.radial"), Nradial); + + psFree(readouts); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialBins.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialBins.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialBins.c (revision 41083) @@ -0,0 +1,272 @@ +# include "psphotInternal.h" +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X); + +// convert the flux vs elliptical radius to annular bins + +// we are guaranteed to be limited by either the seeing (1 - few pixels) or by the pixels +// themselves. this function does not attempt to measure the radial profiles accurately +// for radii that are smaller than ~2 pixels + +// for small radii, we are measuring the mean surface brightness in non-overlapping radial +// bins. for large radii (r > 2 pixels), we are measuring the surface brightness for a +// radius range \alpha r_i < i < \beta r_i, but performing this measurement for radii more +// finely spaced than r_{i+1} = r_i * \beta / \alpha. for the integration, we need to +// track the non-overlapping radius values. + +// Photo interpolates the image of interest to place the peak on the center of the central +// pixel, and then uses the exact fractions of the pixels in each of the first few annuli. +// Seems like a reasonable thing, but is there any significance to the difference? + +// XXX move the resulting elements from profile to extpars->petrosian? +bool psphotRadialBins (psMetadata *recipe, pmSource *source, float radiusMax, float skynoise) { + + psAssert (source, "missing source"); + psAssert (source->extpars, "missing extpars"); + psAssert (source->extpars->ellipticalFlux, "missing ellipticalFlux"); + + psVector *radius = source->extpars->ellipticalFlux->radiusElliptical; + psVector *flux = source->extpars->ellipticalFlux->fluxElliptical; + + // sort incoming vectors by radius + pmSourceRadialProfileSortPair (radius, flux); + + if (!source->extpars->radProfile) { + source->extpars->radProfile = pmSourceRadialProfileAlloc(); + } + pmSourceRadialProfile *profile = source->extpars->radProfile; + + float skyModelErrorSQ = PS_SQR(skynoise); + psEllipseAxes axes = source->extpars->axes; + float AxialRatio = axes.minor / axes.major; + + // radMin, radMax store the bounds of the annuli + bool status = false; + psVector *radMin = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.LOWER"); + psVector *radMax = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + psAssert (radMin, "RADIAL.ANNULAR.BINS.LOWER is missing from recipe"); + psAssert (radMin->n, "RADIAL.ANNULAR.BINS.LOWER is empty in recipe"); + psAssert (radMax, "RADIAL.ANNULAR.BINS.UPPER is missing from recipe"); + psAssert (radMax->n, "RADIAL.ANNULAR.BINS.UPPER is empty in recipe"); + + psVector *binSB = psVectorAllocEmpty(radMin->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *binSBstdev = psVectorAllocEmpty(radMin->n, PS_TYPE_F32); // surface brightness error of radial bin + psVector *binSum = psVectorAllocEmpty(radMin->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *binFill = psVectorAllocEmpty(radMin->n, PS_TYPE_F32); // surface brightness of radial bin + psVector *binRad = psVectorAllocEmpty(radMin->n, PS_TYPE_F32); // mean radius of radial bin + psVector *binArea = psVectorAllocEmpty(radMin->n, PS_TYPE_F32); // area of radial bin (contiguous, non-overlapping) + + psVectorInit (binSB, 0.0); + psVectorInit (binSBstdev, 0.0); + psVectorInit (binSum, 0.0); + psVectorInit (binFill, 0.0); + + psVectorInit (binRad, 0.0); + + // generate radial area-weighted mean radius & non-overlapping areas + for (int i = 0; i < radMin->n; i++) { + float rMin2 = PS_SQR(radMin->data.F32[i]); + float rMax2 = PS_SQR(radMax->data.F32[i]); + + float rMin3 = rMin2*radMin->data.F32[i]; + float rMax3 = rMax2*radMax->data.F32[i]; + + float rBin = 2.0 * (rMax3 - rMin3) / (rMax2 - rMin2) / 3.0; + + // XXX calculate area-weighted radius rather than asserting? + binRad->data.F32[i] = rBin; + binArea->data.F32[i] = M_PI * (rMax2 - rMin2); + } + + // storage vector for stats + psVector *values = psVectorAllocEmpty (flux->n, PS_TYPE_F32); + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + + float fluxSum = 0.0; + int nPixSum = 0; + + bool done = false; + int nOut = 0; + float Rmin = radMin->data.F32[nOut]; + float Rmax = radMax->data.F32[nOut]; + float Rnxt = radMin->data.F32[nOut+1]; // minimum radius for next range + int iNext = 0; + int iLast = -1; + for (int i = 0; !done && (i < radius->n); i++) { + if (radius->data.F32[i] < Rnxt) { + iNext = i; // on the next pass, we will start back here to ensure we hit all pixels in the next bin + } + if (radius->data.F32[i] > Rmax) { + // calculate the surface brightness for bin 'nOut' + float value, dvalue; + if (values->n > 0) { + psVectorStats (stats, values, NULL, NULL, 0); + value = stats->robustMedian; + dvalue = stats->robustStdev; + } else { + value = NAN; + dvalue = NAN; + } + + binSB->data.F32[nOut] = value; + binSBstdev->data.F32[nOut] = sqrt(PS_SQR(dvalue) / values->n + skyModelErrorSQ); + + // calculate the total flux for bin 'nOut' + float Area = M_PI*AxialRatio*PS_SQR(Rmax); + binSum->data.F32[nOut] = fluxSum; + binFill->data.F32[nOut] = nPixSum / Area; + + psTrace ("psphot", 5, "radial bins: %3d %5.1f %5.1f : %7.1f %6.2f : %8.1f %4.2f %6.1f\n", + nOut, radMin->data.F32[nOut], radMax->data.F32[nOut], + binSB->data.F32[nOut], binSBstdev->data.F32[nOut], + binSum->data.F32[nOut], binFill->data.F32[nOut], Area); + + nOut ++; + if (nOut >= radMin->n) break; + Rmin = radMin->data.F32[nOut]; + Rmax = radMax->data.F32[nOut]; + Rnxt = (nOut < radMin->n - 1) ? radMin->data.F32[nOut+1] : Rmax; // minimum radius for next range + values->n = 0; + psStatsInit(stats); + iLast = i; + i = iNext; + } + if (radius->data.F32[i] < Rmin) { + continue; + } + psVectorAppend (values, flux->data.F32[i]); + + if (i > iLast) { + fluxSum += flux->data.F32[i]; + nPixSum ++; + } + } + binSB->n = binSBstdev->n = binSum->n = binRad->n = binArea->n = nOut; + + // interpolate any bins that were empty (extrapolate to center if needed) + if (!isfinite(binSB->data.F32[0]) && !isfinite(binSB->data.F32[1])) { + psWarning ("center 2 bins of source at %f, %f are NAN, skipping this source", source->peak->xf, source->peak->yf); + // XXX raise a flag + psFree(binSB); + psFree(binSBstdev); + + psFree(binSum); + psFree(binFill); + + psFree(binRad); + psFree(binArea); + psFree(values); + psFree(stats); + source->mode2 |= PM_SOURCE_MODE2_RADBIN_NAN_CENTER; + return true; + } + + // if center bin is empty assume same SB as next radius (probably true due to PSF) + if (!isfinite(binSB->data.F32[0])) { + binSB->data.F32[0] = binSB->data.F32[1]; + binSBstdev->data.F32[0] = binSBstdev->data.F32[1]; + } + + // interpolate any bins that were empty (if center if needed) + for (int i = 1; i < binSB->n - 1; i++) { + if (isfinite(binSB->data.F32[i])) continue; + binSB->data.F32[i] = InterpolateValues (binRad->data.F32[i-1], binSB->data.F32[i-1], binRad->data.F32[i+1], binSB->data.F32[i+1], binRad->data.F32[i]); + binSBstdev->data.F32[i] = InterpolateValues (binRad->data.F32[i-1], binSBstdev->data.F32[i-1], binRad->data.F32[i+1], binSBstdev->data.F32[i+1], binRad->data.F32[i]); + } + + psFree(profile->binSB); + psFree(profile->binSBstdev); + + psFree(profile->binSum); + psFree(profile->binFill); + + psFree(profile->radialBins); + psFree(profile->area); + + // save the vectors + profile->binSB = binSB; + profile->binSBstdev = binSBstdev; + + profile->binSum = binSum; + profile->binFill = binFill; + + profile->radialBins = binRad; + profile->area = binArea; + + psFree(values); + psFree(stats); + + return true; +} + +// If the number of sources is large, edit the recipe to remove radial bins beyond +// a value specified in the recipe +bool psphotLimitRadialApertures(psMetadata *recipe, long nSources) { + + bool status = false; + long sourceLimit = psMetadataLookupS32 (&status, recipe, "RADIAL.NUM.SOURCES.LIMIT"); + if (!status) { + sourceLimit = 50000; + } + if (nSources < sourceLimit) { + return true; + } + psF32 maxRadius = psMetadataLookupF32 (&status, recipe, "RADIAL.SOURCES.OVER.LIMIT.RADIUS"); + if (!status) { + maxRadius = 20.0; + } + psLogMsg ("psphot", PS_LOG_INFO, "Number of objects: %ld is greater than limit %ld. Limiting radial annular bins to %.3f pixels\n", + nSources, sourceLimit, maxRadius); + + psVector *radMin = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.LOWER"); + psVector *radMax = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + if (!radMin || !radMin->n) { + psError (PSPHOT_ERR_CONFIG, true, "error in definition of annular bins (radMin missing or empty)"); + return false; + } + if (!radMax || !radMax->n) { + psError (PSPHOT_ERR_CONFIG, true, "error in definition of annular bins (radMax missing or empty)"); + return false; + } + if (radMax->n != radMin->n) { + psError (PSPHOT_ERR_CONFIG, true, "length of radMin %ld and radMax %ld not equal)", radMin->n, radMax->n); + return false; + } + int i = 0; + psF32 lastRadius = 0; + for (; i < radMax->n; i++) { + if (radMax->data.F32[i] > maxRadius) { + break; + } + lastRadius = radMax->data.F32[i]; + } + if (i == radMax->n) { + psLogMsg ("psphot", PS_LOG_INFO, "Radius of all bins is within the limit. lastRadius: %.3f\n", lastRadius); + return true; + } + psLogMsg ("psphot", PS_LOG_INFO, " radius limit exceeded at bin %d of %ld. New max radius is %.3f\n", + i, radMax->n, lastRadius); + + psVector *radMinNew = psVectorRealloc(radMin, i); + if (radMinNew != radMin) { + psMetadataAddVector(recipe, PS_LIST_TAIL, "RADIAL.ANNULAR.BINS.LOWER", PS_META_REPLACE, "", radMinNew); + // XXX: I don't need this so I? + // psFree(radMinNew); + } + + psVector *radMaxNew = psVectorRealloc(radMax, i); + if (radMaxNew != radMax) { + psMetadataAddVector(recipe, PS_LIST_TAIL, "RADIAL.ANNULAR.BINS.UPPER", PS_META_REPLACE, "", radMaxNew); + // XXX: I don't need to free this do I? + // psFree(radMaxNew); + } + + return true; +} + +// the area-weighted mean radius is given by: + +// integral r * 2 pi r dr / integral 2 pi r dr + +// = 2/3 pi (r_max^3 - r_min^3) / pi (r_max^2 - r_min^2) +// = 2/3 (r_max^3 - r_min^3) / (r_max^2 - r_min^2) + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialPlot.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialPlot.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialPlot.c (revision 41083) @@ -0,0 +1,118 @@ +# include "psphotInternal.h" + +// this variable is defined in psmodules.h if ohana-config is found +# if (HAVE_KAPA) + +# include + +static int nCount = 0; + +bool psphotRadialPlot (int *kapa, const char *filename, pmSource *source) { + + Graphdata graphdata; + + // only plot 50 stars for now... + if (nCount > 00) { + if (*kapa != 0) { + KapaClose (*kapa); + *kapa = 0; + } + return true; + } + + // XXX get the 'showWindow' option from the recipes somewhere + // XXX 'showWindow = false' is broken + if (*kapa == 0) { + *kapa = pmKapaOpen (false); + KapaResize (*kapa, 500, 500); + unlink (filename); + } + if (*kapa == -1) { + psError(PSPHOT_ERR_UNKNOWN, true, "failure to open kapa"); + return false; + } + + KapaInitGraph (&graphdata); + KapaClearPlots (*kapa); + + // examine sources to set data range + graphdata.xmin = -0.05; + graphdata.xmax = +30.05; + graphdata.ymin = -0.05; + graphdata.ymax = +5.05; + KapaSetLimits (*kapa, &graphdata); + + KapaSetFont (*kapa, "helvetica", 14); + KapaBox (*kapa, &graphdata); + KapaSendLabel (*kapa, "radius (pixels)", KAPA_LABEL_XM); + KapaSendLabel (*kapa, "log flux (counts)", KAPA_LABEL_YM); + + int nPts = source->pixels->numRows * source->pixels->numCols; + psVector *rg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *fg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *rb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *fb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + + int ng = 0; + int nb = 0; + float Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS] - source->pixels->col0; + float Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS] - source->pixels->row0; + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]) { + rb->data.F32[nb] = hypot (ix - Xo, iy - Yo) ; + fb->data.F32[nb] = log10(source->pixels->data.F32[iy][ix]); + nb++; + } else { + rg->data.F32[ng] = hypot (ix - Xo, iy - Yo) ; + fg->data.F32[ng] = log10(source->pixels->data.F32[iy][ix]); + ng++; + } + } + } + + // set the plot range here based on lflux, radius + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (*kapa, ng, &graphdata); + KapaPlotVector (*kapa, ng, rg->data.F32, "x"); + KapaPlotVector (*kapa, ng, fg->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (*kapa, nb, &graphdata); + KapaPlotVector (*kapa, nb, rb->data.F32, "x"); + KapaPlotVector (*kapa, nb, fb->data.F32, "y"); + + psLogMsg ("psphot", 3, "saving plot to %s", filename); + + char pagename[16]; + sprintf (pagename, "%02d", nCount); + if (nCount == 0) { + KiiPS (*kapa, filename, false, KAPA_PS_NEWPLOT, pagename); + } else { + KiiPS (*kapa, filename, false, KAPA_PS_NEWPAGE, pagename); + } + + psFree (rg); + psFree (fg); + psFree (rb); + psFree (fb); + + nCount ++; + return true; +} + +# else + +bool psphotRadialPlot (int *kapa, const char *filename, pmSource *source) { + psLogMsg ("psphot", 3, "skipping source radial plots"); + return true; +} + +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfile.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfile.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfile.c (revision 41083) @@ -0,0 +1,81 @@ +# include "psphotInternal.h" + +static int Nskip1 = 0; +static int Nskip2 = 0; +static int Nskip3 = 0; +static int Nskip4 = 0; +static int Nskip5 = 0; + +# define SKIP(VALUE) { VALUE++; return false; } + +bool psphotRadialProfile (pmSource *source, psMetadata *recipe, float skynoise, psImageMaskType maskVal) { + + bool status; + + // allocate pmSourceExtendedParameters, if not already defined + if (!source->extpars) { + source->extpars = pmSourceExtendedParsAlloc (); + } + + // XXX these need to go into recipe values + int Nsec = 24; + float Rmax = 200; + float fluxMin = 0.0; + float fluxMax = source->peak->rawFlux; + + bool RAW_RADIUS = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_RAW_RADIUS"); + + // generate a series of radial profiles at Nsec evenly spaced angles. the profile flux + // is measured by interpolation for small radii; for large radii, the pixels in a box + // are averaged to increase the S/N + if (!psphotRadialProfilesByAngles (source, Nsec, Rmax)) { + psError (PS_ERR_UNKNOWN, false, "failed to measure radial profile for petrosian"); + SKIP (Nskip1); + } + // allocate: extpars->radFlux->radii,fluxes,theta + + // use the radial profiles to determine the radius of a given isophote. this isophote + // is used to determine the elliptical shape of the object, so it has a relatively high + // value (nominally 25% of the peak) + if (!psphotRadiiFromProfiles (source, fluxMin, fluxMax)) { + psError (PS_ERR_UNKNOWN, false, "failed to measure isophotal radii from profiles"); + SKIP (Nskip2); + } + // allocate : extpars->radFlux->isophotalRadii (use profile->radii,fluxes) + + + // convert the isophotal radius vs angle measurements to an elliptical contour + if (!psphotEllipticalContour (source)) { + // psLogMsg ("psphot", 3, "failed to measure elliptical contour"); + SKIP (Nskip3); + } + // use extpars->radFlux->isophotalRadii,theta (result in extpars->axes) + + // generate a single, normalized radial profile following the elliptical contours. + // the radius is normalized by the axis ratio so that on the major axis, 1 pixel = 1 pixel + if (!psphotEllipticalProfile (source, RAW_RADIUS)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate elliptical profile"); + SKIP (Nskip4); + } + // allocate extpars->ellipticalFlux->radiusElliptical,fluxElliptical (use axes to scale raw pixels) + + // generated profile in averaged bins + if (!psphotRadialBins (recipe, source, Rmax, skynoise)) { + psError (PS_ERR_UNKNOWN, false, "failed to generate radial bins"); + SKIP (Nskip5); + } + // allocate extpars->radProfile->binSB, binSBstdv, binSum, binFill, radialBins, area (small lengths) + // use radiusElliptical, fluxElliptical, + + return true; +} + +void psphotRadialProfileShowSkips () { +# if (PS_TRACE_ON) + fprintf (stderr, "radial profile skipped @ 1 : %d\n", Nskip1); + fprintf (stderr, "radial profile skipped @ 2 : %d\n", Nskip2); + fprintf (stderr, "radial profile skipped @ 3 : %d\n", Nskip3); + fprintf (stderr, "radial profile skipped @ 4 : %d\n", Nskip4); + fprintf (stderr, "radial profile skipped @ 5 : %d\n", Nskip5); +#endif +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfileByAngles.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfileByAngles.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfileByAngles.c (revision 41083) @@ -0,0 +1,243 @@ +# include "psphotInternal.h" + +// Given a source at (x,y), generate a collection of radial profiles at even angular separations + +// These functions are used to calculate the stats in a rectangle at arbitrary orientation. +// XXX Move these elsewhere (psLib?) +float psphotMeanSectorValue (psImage *image, float x, float y, float dL, float dW, float theta); +psVector *psphotBoxValues (psImage *image, float x0, float y0, float dL, float dW, float theta); +psVector *psphotLineValues (psImage *image, double x1, double y1, double x2, double y2, int dW); +psVector *psphotLineValuesBresen (psImage *image, int X1, int Y1, int X2, int Y2, int dW, int swapcoords); + +bool psphotRadialProfilesByAngles (pmSource *source, int Nsec, float Rmax) { + + psAssert (source->extpars, "define extpars"); + + // we want to have an even number of sectors so we can do 180 deg symmetrizing + Nsec = (Nsec % 2) ? Nsec + 1 : Nsec; + float dtheta = 2.0*M_PI / Nsec; + + if (!source->extpars->radFlux) { + source->extpars->radFlux = pmSourceRadialFluxAlloc(); + } + pmSourceRadialFlux *profile = source->extpars->radFlux; + psFree(profile->radii); + psFree(profile->fluxes); + psFree(profile->theta); + + profile->radii = psArrayAllocEmpty(Nsec); + profile->fluxes = psArrayAllocEmpty(Nsec); + profile->theta = psVectorAllocEmpty(Nsec, PS_TYPE_F32); + + for (int i = 0; i < Nsec; i++) { + + float theta = i*dtheta; + + psVector *radius = psVectorAllocEmpty(Rmax, PS_TYPE_F32); + psVector *flux = psVectorAllocEmpty(Rmax, PS_TYPE_F32); + + // Start at Xo,Yo and find the x,y locations for r_i, theta where r_i initially + // increments by 1 pixel. At large radii (r*dtheta > 2) use stats in a box rather than + // sub-pixel interpolation + + int dR = 1.0; + for (float r = 0; r < Rmax; r += dR) { + + float Xo = source->peak->xf; + float Yo = source->peak->yf; + + // Xo,Yo are referenced to pixels with bounds i+0.0, i+1.0 + float x = r * cos (theta) + Xo; + float y = r * sin (theta) + Yo; + dR = 2*(int)(0.5*r*sin(dtheta)) + 1; + + if (x < 0) goto badvalue; + if (y < 0) goto badvalue; + if (x >= source->pixels->parent->numCols) goto badvalue; + if (y >= source->pixels->parent->numRows) goto badvalue; + + float value = NAN; + if (dR < 2) { + // value is NAN if we run off the image + // 0.5 PIX: this function takes pixel coords; source peak is in pixel coords + value = psImageInterpolatePixelBilinear(x, y, source->pixels); + } else { + // 0.5 PIX: this function takes pixel coords; source peak is in pixel coords + value = psphotMeanSectorValue(source->pixels, x, y, dR, dR, theta); + } + + // keep the all values (even NAN) so all vectors are matched in length + psVectorAppend (radius, r); + psVectorAppend (flux, value); + continue; + + badvalue: + psVectorAppend (radius, r); + psVectorAppend (flux, NAN); + } + + psArrayAdd (profile->radii, 100, radius); + psArrayAdd (profile->fluxes, 100, flux); + psVectorAppend (profile->theta, theta); + + // psphotPetrosianVisualProfileByAngle (radius, flux); + + psFree(radius); + psFree(flux); + } + + for (int i = 0; i < Nsec / 2; i++) { + + psVector *r1 = profile->radii->data[i]; + psVector *r2 = profile->radii->data[i+Nsec/2]; + + psVector *f1 = profile->fluxes->data[i]; + psVector *f2 = profile->fluxes->data[i+Nsec/2]; + + psAssert (r1->n == r2->n, "mis-matched vectors"); + psAssert (f1->n == f2->n, "mis-matched vectors"); + + // we have a pair of vectors i, i+Nsec/2; replace them with the finite minimum of the pair + for (int j = 0; j < r1->n; j++) { + + float flux; + + if (!isfinite(f1->data.F32[j]) && !isfinite(f2->data.F32[j])) { + flux = NAN; + goto setflux; + } + + if (!isfinite(f1->data.F32[j])) { + flux = f2->data.F32[j]; + goto setflux; + } + if (!isfinite(f2->data.F32[j])) { + flux = f1->data.F32[j]; + goto setflux; + } + + flux = PS_MIN(f1->data.F32[j], f2->data.F32[j]); + + setflux: + f1->data.F32[j] = flux; + f2->data.F32[j] = flux; + } + } + return true; +} + +float psphotMeanSectorValue (psImage *image, float x, float y, float dL, float dW, float theta) { + + psVector *values = psphotBoxValues (image, x, y, dL, dW, theta); + if (!values) goto escape; + if (!values->n) goto escape; + + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); + psVectorStats (stats, values, NULL, NULL, 0); + + float value = stats->sampleMedian; + + psFree (stats); + psFree (values); + + return value; + +escape: + psFree(values); + return NAN; +} + +psVector *psphotBoxValues (psImage *image, float x0, float y0, float dL, float dW, float theta) { + + // extract pixels from a series of lines (from -0.5*dW to +0.5*dW) of length dL, + // centered on x0, y0 in parent pixel coordinates (not pixel indicies) + + float xs = x0 - image->col0 - 0.5*dL*cos(theta); + float ys = y0 - image->row0 - 0.5*dL*sin(theta); + + float xe = xs + 0.5*dL*cos(theta); + float ye = ys + 0.5*dL*sin(theta); + + psVector *values = psphotLineValues (image, xs, ys, xe, ye, (int) dW); + return values; +} + +/** + * identify the quadrant and draw the correct line + */ +psVector *psphotLineValues (psImage *image, double x1, double y1, double x2, double y2, int dW) { + + int FlipDirect, FlipCoords; + int X1, Y1, X2, Y2, dX, dY; + + /* rather than draw the line from float positions, we find the closest + integer end-points and draw the line between those pixels */ + + X1 = ROUND(x1); + Y1 = ROUND(y1); + X2 = ROUND(x2); + Y2 = ROUND(y2); + + dX = X2 - X1; + dY = Y2 - Y1; + + FlipCoords = (abs(dX) < abs(dY)); + FlipDirect = FlipCoords ? (y1 > y2) : (x1 > x2); + + psVector *values = NULL; + if (!FlipDirect && !FlipCoords) values = psphotLineValuesBresen (image, X1, Y1, X2, Y2, dW, FALSE); + if ( FlipDirect && !FlipCoords) values = psphotLineValuesBresen (image, X2, Y2, X1, Y1, dW, FALSE); + if (!FlipDirect && FlipCoords) values = psphotLineValuesBresen (image, Y1, X1, Y2, X2, dW, TRUE); + if ( FlipDirect && FlipCoords) values = psphotLineValuesBresen (image, Y2, X2, Y1, X1, dW, TRUE); + + return values; +} + +/** + * use the Bresenham line drawing technique + * integer-only Bresenham line-draw version which is fast + */ +psVector *psphotLineValuesBresen (psImage *image, int X1, int Y1, int X2, int Y2, int dW, int swapcoords) { + + int X, Y, dX, dY; + int e, e2; + + psVector *values = psVectorAllocEmpty(100, PS_TYPE_F32); + + dX = X2 - X1; + dY = Y2 - Y1; + + Y = Y1; + e = 0; + for (X = X1; X <= X2; X++) { + if (X > 0) { + if (swapcoords) { + if (X >= image->numRows) continue; + for (int y = Y - dW; y <= Y + dW; y++) { + if (y < 0) continue; + if (y >= image->numCols) continue; + psVectorAppend(values, image->data.F32[X][y]); + } + } else { + if (X >= image->numCols) continue; + for (int y = Y - dW; y <= Y + dW; y++) { + if (y < 0) continue; + if (y >= image->numRows) continue; + psVectorAppend(values, image->data.F32[y][X]); + } + } + } + e += dY; + e2 = 2 * e; + if (e2 > dX) { + Y++; + e -= dX; + } + if (e2 < -dX) { + Y--; + e += dX; + } + } + return values; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfileWings.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfileWings.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadialProfileWings.c (revision 41083) @@ -0,0 +1,431 @@ +# include "psphotInternal.h" + +// measure the radial profile outside of the core. the goal is to find the point at which we +// reach sky + X sigma + +bool psphotRadialProfileWingsSource (pmSource *source, pmReadout *readout, psImageMaskType maskVal); +bool psphotRadialProfileFluxAtRadius (psVector *flux, psVector *fluxVar, pmSource *source, pmReadout *readout, float Radius, float dRadius, psImageMaskType maskVal); + +bool psphotRadialProfileWings (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // return true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Radial Profile Wings ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources ? detections->newSources : detections->allSources; + psAssert (sources, "missing sources?"); + + if (!psphotRadialProfileWingsReadout (config, recipe, view, readout, sources)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to measure magnitudes for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// these are set before we fork off to threads and used by all threads as constant values +static float MAX_RADIUS = NAN; +static float MIN_RADIUS = NAN; +static float SKY_STDEV = NAN; +static float SKY_SLOPE_MIN = NAN; +// static FILE *file = NULL; + +bool psphotRadialProfileWingsReadout(pmConfig *config, psMetadata *recipe, const pmFPAview *view, pmReadout *readout, psArray *sources) { + + bool status = false; + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping radial profile wings"); + return true; + } + + psTimerStart ("psphot.wings"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // XXX is this a good recipe value to use for MAX RADIUS?? + MAX_RADIUS = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_RADIUS"); + if (!status) { + MAX_RADIUS = 50.0; + } + + MIN_RADIUS = psMetadataLookupF32 (&status, readout->analysis, "PSF_MOMENTS_RADIUS"); + if (!status || (MIN_RADIUS > MAX_RADIUS)) { + MIN_RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_MOMENTS_RADIUS"); + } + + // SKY_STDEV is the sigma of the sky model (ie, smoothed on large scales) + SKY_STDEV = psMetadataLookupF32 (&status, readout->analysis, "MSKY_SIG"); + if (!status) { + SKY_STDEV = 1.0; // a crude default value (why would this not exist?) + } + + // SKY_SLOPE_MIN is the sigma of the sky model (ie, smoothed on large scales) + SKY_SLOPE_MIN = psMetadataLookupF32 (&status, recipe, "SKY_SLOPE_MIN"); + if (!status) { + SKY_SLOPE_MIN = 3.0; + } + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // source analysis is done in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping blend"); + return true; + } + + // file = fopen ("radii.dat", "w"); + + // threaded measurement of the source magnitudes + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_RADIAL_PROFILE_WINGS"); + + psArrayAdd(job->args, 1, readout); + psArrayAdd(job->args, 1, cells->data[j]); // sources + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); + +// set this to 0 to run without threading +# if (1) + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } +# else + if (!psphotRadialProfileWings_Threaded(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + psFree(job); +# endif + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } + psFree(job); + } + } + psFree (cellGroups); + + // fclose (file); + + psLogMsg ("psphot.wings", PS_LOG_WARN, "measure radial profile wings : %f sec for %ld objects\n", psTimerMark ("psphot.wings"), sources->n); + return true; +} + +bool psphotRadialProfileWings_Threaded (psThreadJob *job) { + + pmReadout *readout = job->args->data[0]; + psArray *sources = job->args->data[1]; + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[2],PS_TYPE_IMAGE_MASK_DATA); + + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + + // allocate space for moments + if (!source->moments) continue; + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // skip non-detected sources matched from other images + if (source->mode2 & PM_SOURCE_MODE2_MATCHED) return true; // skip matched sources (no signal) + + // XXX unclear if I should run this analysis on both 1st and 2nd pass for 1st pass objects, + // or just use the 1st pass value. I think I should just use the 1st pass value, so skip + // any that have already been assigned + if (isfinite(source->skyRadius)) return true; + + // replace object in image + bool reSubtract = false; + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + reSubtract = true; + } + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + // XXX don't measure on subraster images: + // pmSourceRedefinePixels (source, readout, source->peak->x, source->peak->y, windowRadius + 2); + // psAssert (source->pixels, "WTF?"); + + // this function populates moments->Mrf,KronFlux,KronFluxErr + psphotRadialProfileWingsSource (source, readout, maskVal); + + // if we subtracted it above, re-subtract the object, leave local sky + if (reSubtract) { + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + } + return true; +} + +# define TEST_X 3158 +# define TEST_Y 3096 + +float InterpolateValues (float X0, float Y0, float X1, float Y1, float X); + +// XXX use integer radius values? the rings assume integer values, right? or do they? +bool psphotRadialProfileWingsSource (pmSource *source, pmReadout *readout, psImageMaskType maskVal) { + + PS_ASSERT_PTR_NON_NULL(source, false); + PS_ASSERT_PTR_NON_NULL(source->peak, false); + PS_ASSERT_PTR_NON_NULL(source->pixels, false); + + // psImageMaskType **vMsk = (source->maskObj == NULL) ? NULL : source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA; + + // radii will be MIN_RADIUS to MAX_RADIUS in NN log steps: + float NSTEP = 25.0; + float MIN_DR = 2; + float NSIGMA = 1.0; + float THRESHOLD = 2.0*SKY_STDEV; + float alpha = pow ((MAX_RADIUS / MIN_RADIUS), 1.0/NSTEP) - 1.0; + float dRmax = MAX_RADIUS * alpha / (1.0 + alpha); // approximate size of last annulus, to get a rough size for vector allocation + + int iter = 0; + + psVector *flux = psVectorAllocEmpty(7*MAX_RADIUS*dRmax, PS_TYPE_F32); + psVector *fluxVar = psVectorAllocEmpty(7*MAX_RADIUS*dRmax, PS_TYPE_F32); + + // should I just use sample median here? + psStats *fluxStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); + psStats *varStats = psStatsAlloc(PS_STAT_SAMPLE_MEAN); + + float lastFlux = NAN; + float lastRadius = NAN; + float lastSlope = NAN; + bool limit = false; + float limitRadius = NAN; + float limitFlux = NAN; + float limitSlope = NAN; + + // note: radius is the inner radius of the annulus; outer radius = inner radius + dR + for (float radius = MIN_RADIUS; !limit && (radius < MAX_RADIUS); iter ++) { + + float dR = (int)(radius * alpha); + if (dR < MIN_DR) dR = MIN_DR; + float outerRadius = radius + dR; + float meanRadius = (2.0/3.0) * (outerRadius*outerRadius*outerRadius - radius*radius*radius) / (PS_SQR(outerRadius) - PS_SQR(radius)); + + // extract a vector of the pixel values (signal, variance) at this radius + dR + psphotRadialProfileFluxAtRadius (flux, fluxVar, source, readout, radius, dR, maskVal); + + psStatsInit (fluxStats); + psStatsInit (varStats); + + if ((flux->n == 0) || (fluxVar->n == 0)) { + flux->n = 0; + fluxVar->n = 0; + radius += dR; + continue; + } + + psVectorStats (fluxStats, flux, NULL, NULL, 0); + psVectorStats (varStats, fluxVar, NULL, NULL, 0); + + // get the mean / median flux statistic and significance here + float meanFlux = fluxStats->robustMedian; + float meanFluxError = sqrt(varStats->sampleMean / fluxVar->n); + // this is a bit crude on the flux error: the error. technically, it should be + // sqrt(\sum(variance)) / Npts. I am using the identity \sum(variance) = + // \mean(variance) * Npts and cancelling the Npts term inside and out of the sqrt() + + float slope = NAN; + if (isfinite(lastFlux)) { + slope = (meanFlux - lastFlux) / (meanRadius - lastRadius); + } + + // fprintf (stderr, "%f %f : %f : %f %f : %f\n", source->peak->xf, source->peak->yf, radius, meanFlux, meanFluxError, slope); + + if (!limit && (meanFlux - NSIGMA * meanFluxError < THRESHOLD)) { + // dropped to sky level + limit = true; + // linearly interpolate to the radius at which we hit the sky + if (isfinite(lastFlux)) { + limitRadius = InterpolateValues(lastFlux, lastRadius, meanFlux, meanRadius, 0.0); + } else { + limitRadius = meanRadius; + } + limitFlux = meanFlux; + limitSlope = slope; + } + if (!limit && isfinite(slope) && (fabs(slope) < SKY_SLOPE_MIN)) { + // SB no longer changing. + limit = true; + // linearly interpolate to the radius at which we hit the sky, using the last flux and the limiting slope + if (isfinite(lastFlux)) { + float interpolatedRadius = lastRadius + lastFlux / SKY_SLOPE_MIN; + if (interpolatedRadius < MAX_RADIUS) { + limitRadius = interpolatedRadius; + } else { + // XXX should we keep going in this case? + limitRadius = meanRadius; + } + } else { + limitRadius = meanRadius; + } + limitFlux = meanFlux; + limitSlope = slope; + } + + // completion criteria: + // 1) flux - NSIGMA * dflux <= sky + // 2) flux rising? + // 3) flux flat? + + lastFlux = meanFlux; + lastRadius = meanRadius; + lastSlope = slope; + + // reset the flux & fluxVar vector length to zero for re-use above: + flux->n = 0; + fluxVar->n = 0; + radius += dR; + } + + if (!limit) { + limitRadius = lastRadius; + limitFlux = lastFlux; + limitSlope = lastSlope; + } + // fprintf (file, "%f %f : %f %f : %f\n", source->peak->xf, source->peak->yf, limitRadius, limitFlux, limitSlope); + + psFree (flux); + psFree (fluxVar); + psFree (fluxStats); + psFree (varStats); + + source->skyRadius = limitRadius; + source->skyFlux = limitFlux; + source->skySlope = limitSlope; + + // save the max radius (and anything else?) + // source->moments->Mrf = Mrf; + + return true; +} + +// inline this? macro this? +bool psphotRadialProfileGetFlux (psVector *flux, psVector *fluxVar, pmReadout *readout, int xc, int yc, psImageMaskType maskVal) { + + int Nx = readout->image->numCols; + int Ny = readout->image->numRows; + + if (xc < 0) return false; + if (xc >= Nx) return false; + if (yc < 0) return false; + if (yc >= Ny) return false; + + if (readout->mask && (readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[yc][xc] & maskVal)) return false; + psVectorAppend (flux, readout->image->data.F32[yc][xc]); + psVectorAppend (fluxVar, readout->variance->data.F32[yc][xc]); + return true; +} + +// select the pixels at the given radius and extract the flux and variance into the given vectors +// XXX should radius be 'int'? +bool psphotRadialProfileFluxAtRadius (psVector *flux, psVector *fluxVar, pmSource *source, pmReadout *readout, float Radius, float dRadius, psImageMaskType maskVal) { + + psAssert (flux, "must allocate output flux vector"); + psAssert (fluxVar, "must allocate output fluxVar vector"); + + // the peak position is less accurate but less subject to extreme deviations + float dX = source->moments->Mx - source->peak->xf; + float dY = source->moments->My - source->peak->yf; + float dR = hypot(dX, dY); + float Xo = (dR < 2.0) ? source->moments->Mx : source->peak->xf; + float Yo = (dR < 2.0) ? source->moments->My : source->peak->yf; + + for (int radius = Radius; radius < Radius + dRadius; radius ++) { + + int x = 0; + int y = radius; + int d = 5 - 4*radius; + + while (x <= y) { + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo + x), (Yo + y), maskVal); + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo + x), (Yo - y), maskVal); + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo + y), (Yo + x), maskVal); + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo - y), (Yo + x), maskVal); + + if (x > 0) { + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo - x), (Yo + y), maskVal); + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo - x), (Yo - y), maskVal); + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo - y), (Yo - x), maskVal); + psphotRadialProfileGetFlux(flux, fluxVar, readout, (Xo + y), (Yo - x), maskVal); + } + + if (d < 0) { + d = d + 8*x + 4; + } else { + d = d + 8*(x-y) + 8; + y--; + } + x++; + } + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadiiFromProfiles.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadiiFromProfiles.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadiiFromProfiles.c (revision 41083) @@ -0,0 +1,160 @@ +# include "psphotInternal.h" + +// Given the Radial Profiles (radii, fluxes) determine the radius for each profile at the desired isophote + +bool psphotRadiiFromProfiles (pmSource *source, float fluxMin, float fluxMax) { + + psAssert (source, "missing source"); + psAssert (source->extpars, "missing extpars"); + psAssert (source->extpars->radFlux, "missing radFlux"); + + pmSourceRadialFlux *profile = source->extpars->radFlux; + + psFree(profile->isophotalRadii); + profile->isophotalRadii = psVectorAlloc(profile->theta->n, PS_TYPE_F32); + + for (int i = 0; i < profile->theta->n; i++) { + psVector *radii = profile->radii->data[i]; + psVector *fluxes = profile->fluxes->data[i]; + float radius = psphotRadiusFromProfile (source, radii, fluxes, fluxMin, fluxMax); + + // psphotPetrosianVisualProfileByAngle (radii, fluxes, radius); + + // warn on NAN? + profile->isophotalRadii->data.F32[i] = radius; + } + return true; +} + +float psphotRadiusFromProfile (pmSource *source, psVector *radius, psVector *flux, float fluxMin, float fluxMax) { + + // 'flux' is a noisy sample of the galaxy radial profile at points 'radius' + // rebin flux into samples defined by the isophote Fo = 0.5*(fluxMax + fluxMin). the noisy + // sample is cleaned by rebinning to a well-matched radial binning + + // base selections on fluxes defined by the flux range dF + float fluxRange = fluxMax - fluxMin; + + // examine data in the two ranges Fm - Fo and Fo - Fp to define the bin size + // XXX reconsider the fractional isophote value + float Fm = fluxMin + 0.10*fluxRange; + float Fo = fluxMin + 0.25*fluxRange; + float Fp = fluxMin + 0.50*fluxRange; + int Rbin = 1; + + // find the median radius of the points in the flux range Fm - Fp: + { + // storage vector for stats + psVector *values = psVectorAllocEmpty (flux->n, PS_TYPE_F32); + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); + + for (int i = 0; i < flux->n; i++) { + if (!isfinite(flux->data.F32[i])) continue; + if (flux->data.F32[i] < Fm) continue; + if (flux->data.F32[i] > Fp) continue; + + psVectorAppend (values, radius->data.F32[i]); + } + if (values->n > 1) { + psVectorStats (stats, values, NULL, NULL, 0); + + // if we have a valid range, rebin with bin size 1/2 of median radius + if (isfinite(stats->sampleMedian)) { + Rbin = MAX(1, 0.5*stats->sampleMedian); + } + } + psFree (values); + psFree (stats); + } + Rbin = 3; + + psVector *fluxBinned = NULL; + psVector *radiusBinned = NULL; + + // do not bother rebinning if the bin size is only 2 or less + if (Rbin <= 2) { + fluxBinned = psMemIncrRefCounter (flux); + radiusBinned = psMemIncrRefCounter (radius); + } else { + // storage vector for stats + psVector *values = psVectorAllocEmpty (flux->n, PS_TYPE_F32); + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); + + // rebinned vectors + fluxBinned = psVectorAllocEmpty (flux->n, PS_TYPE_F32); + radiusBinned = psVectorAllocEmpty (flux->n, PS_TYPE_F32); + + // sort the flux by the radius + pmSourceRadialProfileSortPair (radius, flux); + + int nOut = 0; + radiusBinned->data.F32[nOut] = (nOut + 0.5)*Rbin; + float Rmin = radiusBinned->data.F32[nOut] - 0.5*Rbin; + float Rmax = radiusBinned->data.F32[nOut] + 0.5*Rbin; + + for (int i = 0; i < flux->n; i++) { + if (radius->data.F32[i] < Rmin) { + // XXX not sure how we can hit this, if there is full coverage of radiusBinned + continue; + } + if (radius->data.F32[i] > Rmax) { + // calculate the value for the nOut bin + // XXX need to fix this as well psStats (stats, values); + float value; + if (values->n > 0) { + psVectorStats (stats, values, NULL, NULL, 0); + value = stats->sampleMedian; + } else { + value = NAN; + } + fluxBinned->data.F32[nOut] = value; + nOut ++; + radiusBinned->data.F32[nOut] = (nOut + 0.5)*Rbin; + Rmin = radiusBinned->data.F32[nOut] - 0.5*Rbin; + Rmax = radiusBinned->data.F32[nOut] + 0.5*Rbin; + values->n = 0; + psStatsInit(stats); + } + if (!isfinite(flux->data.F32[i])) continue; + psVectorAppend (values, flux->data.F32[i]); + } + fluxBinned->n = nOut; + radiusBinned->n = nOut; + psFree (values); + psFree(stats); + } + + float Ro = NAN; + bool above = true; + for (int i = 0; i < fluxBinned->n; i++) { + + if (!isfinite(fluxBinned->data.F32[i])) continue; + + // find the largest radius that matches the flux transition + if (above && (fluxBinned->data.F32[i] < Fo)) { + // XXX is there a macro in psLib that does this interpolation? + if (i == 0) { + psTrace ("psphot", 4, "bogus radial profile for source at %f, %f, skipping", source->peak->xf, source->peak->yf); + psFree (fluxBinned); + psFree (radiusBinned); + return NAN; + } + Ro = radiusBinned->data.F32[i-1] + (radiusBinned->data.F32[i] - radiusBinned->data.F32[i-1]) * (Fo - fluxBinned->data.F32[i-1]) / (fluxBinned->data.F32[i] - fluxBinned->data.F32[i-1]); + above = FALSE; + } + + if (!above && (fluxBinned->data.F32[i] >= Fo)) { + above = TRUE; + } + } + + // show the results + // psphotPetrosianVisualProfileRadii (radius, flux, radiusBinned, fluxBinned, fluxMax, Ro); + + psFree(fluxBinned); + psFree(radiusBinned); + return Ro; +} + + + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadiusChecks.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadiusChecks.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRadiusChecks.c (revision 41083) @@ -0,0 +1,308 @@ +# include "psphotInternal.h" +# define RADIUS_TYPE int + +static float PSF_FIT_NSIGMA; +static float PSF_FIT_PADDING; +static float PSF_APERTURE = 0; // radius to use in PSF aperture mags +static float PSF_FIT_RADIUS = 0; // radius to use in fitting (ignored if <= 0, + // and a per-object radius is calculated) + +bool psphotInitRadiusPSF(psMetadata *recipe, pmReadout *readout) { + + bool status = true; + + PSF_FIT_NSIGMA = psMetadataLookupF32(&status, recipe, "PSF_FIT_NSIGMA"); + PSF_FIT_PADDING = psMetadataLookupF32(&status, recipe, "PSF_FIT_PADDING"); + + PSF_FIT_RADIUS = psMetadataLookupF32(&status, readout->analysis, "PSF_FIT_RADIUS"); + if (!status) { + PSF_FIT_RADIUS = psMetadataLookupF32(&status, recipe, "PSF_FIT_RADIUS"); + } + + PSF_APERTURE = psMetadataLookupF32(&status, readout->analysis, "PSF_APERTURE"); + if (!status) { + PSF_APERTURE = psMetadataLookupF32(&status, recipe, "PSF_APERTURE"); + } + + // The PSF_FIT_RADIUS and PSF_APERTURE may not be set if the PSF was loaded and not chosen + + if (PSF_FIT_RADIUS == 0.0) { + float gaussSigma = psMetadataLookupF32(&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + gaussSigma = psMetadataLookupF32(&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float fitScale = psMetadataLookupF32(&status, recipe, "PSF_FIT_RADIUS_SCALE"); + PSF_FIT_RADIUS = (int)(fitScale*gaussSigma); + } + + if (PSF_APERTURE == 0.0) { + float gaussSigma = psMetadataLookupF32(&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + gaussSigma = psMetadataLookupF32(&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float apScale = psMetadataLookupF32(&status, recipe, "PSF_APERTURE_SCALE"); + PSF_APERTURE = (int)(apScale*gaussSigma); + } + + return true; +} + +// call this function whenever you (re)-define the PSF model +bool psphotCheckRadiusPSF (pmReadout *readout, pmSource *source, pmModel *model, psImageMaskType markVal) +{ + psF32 *PAR = model->params->data.F32; + + // XXX do we have a better value for the sky noise level? not really... + pmMoments *moments = source->moments; + + // set the fit radius based on the object flux limit and the model + float radiusFit = PSF_FIT_RADIUS; + if (radiusFit <= 0) { // use fixed radius + if (moments == NULL) { + radiusFit = model->class->modelRadius(model->params, PSF_FIT_NSIGMA*moments->dSky); + } else { + radiusFit = model->class->modelRadius(model->params, 1.0); + } + model->fitRadius = (RADIUS_TYPE)(radiusFit + PSF_FIT_PADDING); + } else { + model->fitRadius = radiusFit; + } + if (isnan(model->fitRadius)) psAbort("error in radius"); + + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + model->fitRadius *= 2; + } + + // radius used to measure aperture photometry + source->apRadius = PSF_APERTURE; + + bool status = pmSourceRedefinePixels (source, readout, PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], model->fitRadius); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], model->fitRadius, "OR", markVal); + return status; +} + +bool psphotCheckRadiusPSFBlend (pmReadout *readout, pmSource *source, pmModel *model, psImageMaskType markVal, float dR) { + + psF32 *PAR = model->params->data.F32; + + pmMoments *moments = source->moments; + if (moments == NULL) return false; + + // set the fit radius based on the object flux limit and the model + float radiusFit = PSF_FIT_RADIUS; + if (radiusFit <= 0) { // use fixed radius + if (moments == NULL) { + radiusFit = model->class->modelRadius(model->params, PSF_FIT_NSIGMA*moments->dSky); + } else { + radiusFit = model->class->modelRadius(model->params, 1.0); + } + model->fitRadius = (RADIUS_TYPE)(radiusFit + PSF_FIT_PADDING); + } else { + model->fitRadius = radiusFit; + } + if (isnan(model->fitRadius)) psAbort("error in radius"); + + // above sets a radius for a single star, bump by blend separation + model->fitRadius += dR; + + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + model->fitRadius *= 2; + } + + // radius used to measure aperture photometry + source->apRadius = PSF_APERTURE; + + bool status = pmSourceRedefinePixels (source, readout, PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], model->fitRadius); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], model->fitRadius, "OR", markVal); + return status; +} + +static float EXT_FIT_SKY_SIG; +static float EXT_FIT_NSIGMA; +static float EXT_FIT_PADDING; +static float EXT_FIT_MAX_RADIUS; + +bool psphotInitRadiusEXT (psMetadata *recipe, pmReadout *readout) { + + bool status; + + EXT_FIT_NSIGMA = psMetadataLookupF32 (&status, recipe, "EXT_FIT_NSIGMA"); + EXT_FIT_PADDING = psMetadataLookupF32 (&status, recipe, "EXT_FIT_PADDING"); + EXT_FIT_MAX_RADIUS = psMetadataLookupF32 (&status, recipe, "EXT_FIT_MAX_RADIUS"); + + float skyStdev = psMetadataLookupF32 (&status, readout->analysis, "SKY_STDEV"); + + EXT_FIT_SKY_SIG = skyStdev; + + return true; +} + +# define MIN_WINDOW 5.0 +# define SCALE1 5.0 +# define SCALE2 12.0 + +// call this function whenever you (re)-define the EXT model +// XXX this function does not shrink the window +bool psphotSetRadiusMoments (float *fitRadius, float *windowRadius, pmReadout *readout, pmSource *source, psImageMaskType markVal) { + + psAssert (source, "source not defined??"); + psAssert (source->moments, "moments not defined??"); + + *fitRadius = SCALE1 * source->moments->Mrf; + *fitRadius = PS_MIN (PS_MAX(*fitRadius, MIN_WINDOW), EXT_FIT_MAX_RADIUS); + + *windowRadius = SCALE2 * source->moments->Mrf; + *windowRadius = PS_MIN (PS_MAX(*windowRadius, 2.5*MIN_WINDOW), 2.5*EXT_FIT_MAX_RADIUS); + + // redefine the pixels if needed + pmSourceRedefinePixels (source, readout, source->peak->xf, source->peak->yf, *windowRadius); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, source->peak->xf, source->peak->yf, *fitRadius, "OR", markVal); + + return true; +} +# undef SCALE1 +# undef SCALE2 +# undef MIN_WINDOW + +// XXX EAM : 20130724 : for a test, double the window size parameters +// # define MIN_WINDOW 10.0 +// # define SCALE1 7.0 +# define MIN_WINDOW 5.0 +# define SCALE1 3.0 +# define PAD_WINDOW 3.0 + +// call this function whenever you (re)-define the EXT model +// XXX alternate function to set exactly the desired window size +bool psphotSetRadiusMomentsExact (float *fitRadius, float *windowRadius, pmReadout *readout, pmSource *source, psImageMaskType markVal) { + + psRegion newRegion; + + psAssert (source, "source not defined??"); + psAssert (source->moments, "moments not defined??"); + + *fitRadius = SCALE1 * source->moments->Mrf; + *fitRadius = PS_MIN (PS_MAX(*fitRadius, MIN_WINDOW), EXT_FIT_MAX_RADIUS); + + *windowRadius = *fitRadius + PAD_WINDOW; + + // check to see if new region is completely contained within old region + newRegion = psRegionForSquare (source->peak->xf, source->peak->yf, *windowRadius); + newRegion = psRegionForImage (readout->image, newRegion); + + // redefine the pixels to match + pmSourceRedefinePixelsByRegion (source, readout, newRegion); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, source->peak->xf, source->peak->yf, *fitRadius, "OR", markVal); + + return true; +} + +// call this function whenever you (re)-define the EXT model +bool psphotSetRadiusFootprint (float *radius, pmReadout *readout, pmSource *source, psImageMaskType markVal, float factor) { + + psAssert (source, "source not defined??"); + psAssert (source->peak, "peak not defined??"); + + pmPeak *peak = source->peak; + + // set the radius based on the footprint: + if (!peak->footprint) return false; + pmFootprint *footprint = peak->footprint; + if (!footprint->spans) return false; + if (footprint->spans->n < 1) return false; + + // find the max radius + float rawRadius = 0.0; + for (int j = 0; j < footprint->spans->n; j++) { + pmSpan *span = footprint->spans->data[j]; + + float dY = span->y - peak->yf; + float dX0 = span->x0 - peak->xf; + float dX1 = span->x1 - peak->xf; + + rawRadius = PS_MAX (rawRadius, hypot(dY, dX0)); + rawRadius = PS_MAX (rawRadius, hypot(dY, dX1)); + } + if (isnan(rawRadius)) return false; + rawRadius = PS_MIN (factor*rawRadius + EXT_FIT_PADDING, EXT_FIT_MAX_RADIUS); + + // redefine the pixels if needed + pmSourceRedefinePixels (source, readout, peak->xf, peak->yf, rawRadius); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, peak->xf, peak->yf, rawRadius, "OR", markVal); + + *radius = rawRadius; + return true; +} + +// call this function whenever you (re)-define the EXT model +bool psphotMaskFootprint (pmReadout *readout, pmSource *source, psImageMaskType markVal) { + + psAssert (source, "source not defined??"); + psAssert (source->peak, "peak not defined??"); + + pmPeak *peak = source->peak; + + // set the radius based on the footprint: + if (!peak->footprint) return false; + pmFootprint *footprint = peak->footprint; + if (!footprint->spans) return false; + if (footprint->spans->n < 1) return false; + + int Xo = source->maskObj->col0; + int Yo = source->maskObj->row0; + + // mark all pixels + for (int j = 0; j < source->maskObj->numRows; j++) { + for (int i = 0; i < source->maskObj->numCols; i++) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= markVal; + } + } + + psImageMaskType clearVal = PS_NOT_IMAGE_MASK(markVal); + + for (int j = 0; j < footprint->spans->n; j++) { + pmSpan *span = footprint->spans->data[j]; + + // mask the rows before and after each span + int minX = span->x0 - Xo - 2; + int maxX = span->x1 - Xo + 2; + int myY = span->y - Yo; + + // unmark pixels inside the footprint + for (int i = minX; i <= maxX; i++) { + source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[myY][i] &= clearVal; + } + } + return true; +} + +// alternative EXT radius based on model guess (for use without footprints) +bool psphotSetRadiusModel (pmModel *model, pmReadout *readout, pmSource *source, psImageMaskType markVal, bool deep) { + + pmPeak *peak = source->peak; + + // set the fit radius based on the object flux limit and the model + float flux = deep ? EXT_FIT_NSIGMA*EXT_FIT_SKY_SIG : 0.1 * model->params->data.F32[PM_PAR_I0]; + + float rawRadius = model->class->modelRadius (model->params, flux); + if (isnan(rawRadius)) return false; + + rawRadius = PS_MIN (rawRadius + EXT_FIT_PADDING, EXT_FIT_MAX_RADIUS); + model->fitRadius = rawRadius; + + // redefine the pixels if needed + pmSourceRedefinePixels (source, readout, peak->xf, peak->yf, model->fitRadius); + + // set the mask to flag the excluded pixels + psImageKeepCircle (source->maskObj, peak->xf, peak->yf, model->fitRadius, "OR", markVal); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadout.c (revision 41083) @@ -0,0 +1,379 @@ +# include "psphotInternal.h" + +// this should be called by every program that links against libpsphot +bool psphotInit (void) { + psphotErrorRegister(); // register our error codes/messages + psphotModelClassInit (); // load implementation-specific models + psphotSetThreads (); + return true; +} + +# if (0) +// TEST CODE, can be removed +bool psphotDumpFlux (pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status = false; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + static int npass; + char filename[64]; + snprintf (filename, 64, "mags.%d.dat", npass); + FILE *ftest = fopen (filename, "w"); + for (int j = 0; j < sources->n; j++) { + pmSource *source = sources->data[j]; + + float psfMag; + status = pmSourcePhotometryModel (&psfMag, NULL, source->modelPSF); + + float psfMagNorm; + float Io = source->modelPSF->params->data.F32[PM_PAR_I0]; + source->modelPSF->params->data.F32[PM_PAR_I0] = 1.0; + status = pmSourcePhotometryModel (&psfMagNorm, NULL, source->modelPSF); + source->modelPSF->params->data.F32[PM_PAR_I0] = Io; + + // double apTrend = pmTrend2DEval (psf->ApTrend, (float)source->peak->x, (float)source->peak->y); + fprintf (ftest, "%d %d %d %f %f %f %f %f %f\n", j, source->peak->x, source->peak->y, source->modelPSF->params->data.F32[PM_PAR_I0], source->modelPSF->params->data.F32[PM_PAR_SXX], source->modelPSF->params->data.F32[PM_PAR_SYY], source->modelPSF->params->data.F32[PM_PAR_SXY], psfMag, psfMagNorm); + } + fclose (ftest); + npass++; + + return true; +} +# endif + +bool psphotReadout(pmConfig *config, const pmFPAview *view, const char *filerule) { + + // measure the total elapsed time in psphotReadout. dtime is the elapsed time used jointly + // by the multiple threads, not the total time used by all threads. + psTimerStart ("psphotReadout"); + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + // optional break-point for processing + char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); + psAssert (breakPt, "configuration error: set BREAK_POINT"); + + // remove cruft from the input analysis structure + if (!psphotCleanInputs (config, view, filerule)) { + psError (PSPHOT_ERR_PROG, false, "trouble setting up the inputs"); + return false; + } + + // set the photcode for this image + if (!psphotAddPhotcode (config, view, filerule)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images, including the user-defined analysis region of interest + if (!psphotSetMaskAndVariance (config, view, filerule)) { + return psphotReadoutCleanup(config, view, filerule); + } + if (!strcasecmp (breakPt, "NOTHING")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, filerule)) { + // XXX this should result in a bad quality flag + return psphotReadoutCleanup (config, view, filerule); + } + if (!psphotSubtractBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "BACKMDL")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // load the psf model, if suppled. FWHM_MAJ,FWHM_MIN,etc are determined and saved on + // readout->analysis. NOTE: this function currently only loads from PSPHOT.PSF.LOAD + if (!psphotLoadPSF (config, view, filerule)) { // ??? need to supply 2 ? + psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // find the detections (by peak and/or footprint) in the image. + if (!psphotFindDetections (config, view, filerule, true)) { // pass 1 + // this only happens if we had an error in psphotFindDetections + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, filerule); + } + + // construct sources and measure moments and other basic stats (saved on detections->newSources) + // all sources use the auto-scaled window appropriate to a PSF, except for the saturated + // stars : these use a larger window (3x the basic window) + if (!psphotSourceStats (config, view, filerule, true)) { // pass 1 + psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "PEAKS")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // mark blended peaks PS_SOURCE_BLEND (detections->newSources) + // XXX I've deactivated this because it was preventing galaxies close to stars from being + // XXX fitted as an extended source. + if (false && !psphotBasicDeblend (config, view, filerule)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed on deblend analysis"); + return psphotReadoutCleanup (config, view, filerule); + } + + // classify sources based on moments, brightness. if a PSF model has been loaded, the PSF + // clump defined for it is used not measured (detections->newSources) + if (!psphotRoughClass (config, view, filerule)) { // pass 1 + psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough classifications"); + return psphotReadoutCleanup (config, view, filerule); + } + + // find and subtract radial profile models for saturated stars (XXX change name eventually) + if (!psphotDeblendSatstars (config, view, filerule)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed on satstar deblend analysis"); + return psphotReadoutCleanup (config, view, filerule); + } + + // if we were not supplied a PSF model, determine the IQ stats here (detections->newSources) + if (!psphotImageQuality (config, view, filerule)) { // pass 1 + psError (PSPHOT_ERR_UNKNOWN, false, "failed to measure image quality"); + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "MOMENTS")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // use bright stellar objects to measure PSF if we were supplied a PSF for any input file, + // this step is skipped + if (!psphotChoosePSF (config, view, filerule, true)) { // pass 1 + psLogMsg ("psphot", 3, "failure to construct a psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + if (!strcasecmp (breakPt, "PSFMODEL")) { + return psphotReadoutCleanup (config, view, filerule); + } + + // include externally-supplied sources + // XXX fix this in the new multi-input context + // psphotLoadExtSources (config, view, filerule); // pass 1 + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + // pass 1 + if (!psphotGuessModels (config, view, filerule)) { + psLogMsg ("psphot", 3, "failure to Guess Model - pass 1"); + return psphotReadoutCleanup (config, view, filerule); + } + + // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) + psphotFitSourcesLinear (config, view, filerule, false, true); // pass 1 (detections->allSources) + + // measure the radial profiles to the sky + psphotRadialProfileWings (config, view, filerule); + + // re-measure the kron mags with models subtracted. this pass uses a circular window of size PSF_MOMENTS_RADIUS (same window used to measure the psf-scale moments) + + // but this is chosen above to be appropriate for the PSF objects (not galaxies) + // psphotKronMasked(config, view, filerule); + psphotKronIterate(config, view, filerule, 1); + + // identify CRs and extended sources (only unmeasured sources are measured) + psphotSourceSize (config, view, filerule, true); // pass 1 (detections->allSources) + if (!strcasecmp (breakPt, "ENSEMBLE")) { + goto finish; + } + + // non-linear PSF and EXT fit to brighter sources + // replace model flux, adjust mask as needed, fit, subtract the models (full stamp) + // XXX: can leave faulted job in done queue + psphotBlendFit (config, view, filerule); // pass 1 (detections->allSources) + + // replace all sources + psphotReplaceAllSources (config, view, filerule, false); // pass 1 (detections->allSources) + + // linear fit to include all sources (subtract again) + // NOTE : apply to ALL sources (extended + psf) + psphotFitSourcesLinear (config, view, filerule, true, true); // pass 2 (detections->allSources) + + // if we only do one pass, skip to extended source analysis + if (!strcasecmp (breakPt, "PASS1")) goto pass1finish; + + // NOTE: possibly re-measure background model here with objects subtracted / or masked + + // NOTE: this block performs the 2nd pass low-significance PSF detection stage + { + // add noise for subtracted objects & subtracted saturated stars + psphotAddNoise (config, view, filerule); // pass 1 (detections->allSources) + + // find fainter sources + // NOTE: finds new peaks and new footprints, OLD and FULL set are saved on detections + psphotFindDetections (config, view, filerule, false); // pass 2 (detections->peaks, detections->footprints) + + // remove noise for subtracted objects (ie, return to normal noise level) + // NOTE: this needs to operate only on the OLD sources + bool footprintUseUnsubtracted = psMetadataLookupBool(NULL, recipe, "FOOTPRINT_USE_UNSUBTRACTED"); + // Note: if footprintUseUnsubtracted is true the noise was already subtracted in psphotFindDetections() + if (!footprintUseUnsubtracted) { + psphotSubNoise (config, view, filerule); // pass 1 (detections->allSources) + } + + // define new sources based on only the new peaks & measure moments + // NOTE: new sources are saved on detections->newSources + psphotSourceStats (config, view, filerule, false); // pass 2 (detections->newSources) + + // set source type + // NOTE: apply only to detections->newSources + if (!psphotRoughClass (config, view, filerule)) { // pass 2 (detections->newSources) + psLogMsg ("psphot", 3, "failed to find a valid PSF clump for image"); + return psphotReadoutCleanup (config, view, filerule); + } + + // replace all sources so fit below applies to all at once + // NOTE: apply only to OLD sources (which have been subtracted) + psphotReplaceAllSources (config, view, filerule, false); // pass 2 + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); // (detections->newSources + detections->allSources -> detections->allSources) + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, filerule); // pass 1 + + // NOTE: apply to ALL sources + psphotFitSourcesLinear (config, view, filerule, true, true); // pass 3 (detections->allSources) + } + + // NOTE: this block performs the 2nd pass low-significance EXT detection stage (smooth or rebin by NxN times PSF size) + if (0) { + // add noise for subtracted objects + psphotAddNoise (config, view, filerule); // pass 1 (detections->allSources) + + // find fainter sources + // NOTE: finds new peaks and new footprints, OLD and FULL set are saved on detections + psphotFindDetections (config, view, filerule, false); // pass 2 (detections->peaks, detections->footprints) + + // remove noise for subtracted objects (ie, return to normal noise level) + // NOTE: this needs to operate only on the OLD sources + psphotSubNoise (config, view, filerule); // pass 1 (detections->allSources) + + // define new sources based on only the new peaks + // NOTE: new sources are saved on detections->newSources + psphotSourceStats (config, view, filerule, false); // pass 2 (detections->newSources) + + // set source type + // NOTE: apply only to detections->newSources + if (!psphotRoughClass (config, view, filerule)) { // pass 2 (detections->newSources) + psLogMsg ("psphot", 3, "failed to find a valid PSF clump for image"); + return psphotReadoutCleanup (config, view, filerule); + } + + // replace all sources so fit below applies to all at once + // NOTE: apply only to OLD sources (which have been subtracted) + psphotReplaceAllSources (config, view, filerule, false); // pass 2 + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); // (detections->newSources + detections->allSources -> detections->allSources) + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, filerule); // pass 1 + + // NOTE: apply to ALL sources + psphotFitSourcesLinear (config, view, filerule, true, true); // pass 3 (detections->allSources) + } + +pass1finish: + + // measure the radial profiles to the sky (only measures new objects) + psphotRadialProfileWings (config, view, filerule); + + // re-measure the kron mags with models subtracted + // psphotKronMasked(config, view, filerule); + psphotKronIterate(config, view, filerule, 2); + + // measure source size for the remaining sources + // NOTE: applies only to NEW (unmeasured) sources + psphotSourceSize (config, view, filerule, false); // pass 2 (detections->allSources) + + // XXX currently we are doing both the analysis of the size and the assessment of "fit ext" + // in source size. this overloads the bit MODE_EXT_LIMIT to mean "fit ext" not just + // "bigger than a PSF" + + // decide which source(s) are to be fitted with the extended source analysis code. + psphotChooseAnalysisOptions (config, view, filerule); + + psphotExtendedSourceAnalysis (config, view, filerule); // pass 1 (detections->allSources) + psphotExtendedSourceFits (config, view, filerule); // pass 1 (detections->allSources) + // measure some parameters for galaxy science + psphotGalaxyParams (config, view, filerule); + psphotRadialApertures(config, view, filerule, 0); + +finish: + + // plot positive sources + // psphotSourcePlots (readout, sources, recipe); + + // measure aperture photometry corrections + if (!psphotApResid (config, view, filerule)) { // pass 1 (detections->allSources) + psLogMsg ("psphot", 3, "failed on psphotApResid"); + return psphotReadoutCleanup (config, view, filerule); + } + + // calculate source magnitudes + if (!psphotMagnitudes(config, view, filerule)) { // pass 1 (detections->allSources) + psErrorStackPrint(stderr, "Unable to do magnitudes."); + psErrorClear(); + } + + // calculate lensing parameters + if (!psphotLensing(config, view, filerule)) { + psErrorStackPrint(stderr, "Unable to do lensing parameters."); + psErrorClear(); + } + + if (!psphotEfficiency(config, view, filerule)) { // pass 1 + psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources"); + psErrorClear(); + } + + // replace failed sources? + // psphotReplaceUnfitSources (sources); + + // replace background in residual image + if (!psphotSkyReplace (config, view, filerule)) { // pass 1 + psErrorStackPrint(stderr, "Unable to replace sky"); + psErrorClear(); + } + + // drop the references to the image pixels held by each source + if (!psphotSourceFreePixels (config, view, filerule)) { // pass 1 + psErrorStackPrint(stderr, "Unable to free source pixels"); + psErrorClear(); + } + + psLogMsg ("psphot.readout", PS_LOG_WARN, "complete psphot readout : %f sec\n", psTimerMark ("psphotReadout")); + + // create the exported-metadata and free local data + return psphotReadoutCleanup(config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutCleanup.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutCleanup.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutCleanup.c (revision 41083) @@ -0,0 +1,214 @@ +# include "psphotInternal.h" + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotReadoutCleanup (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // remove internal pmFPAfiles, if created + if (psErrorCodeLast() == (psErrorCode) PSPHOT_ERR_DATA) { + psErrorStackPrint(stderr, "Error in the psphot readout analysis"); + psErrorClear(); + } + if (psErrorCodeLast() != PS_ERR_NONE) { + return false; + } + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotReadoutCleanupReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on psphotReadoutCleanup for %s entry %d", filerule, i); + return false; + } + } + + // XXX move this to top of loop? + pmKapaClose (); + + return true; +} + +// psphotReadoutCleanup is called on exit from psphotReadout. If the last raised error is +// not a DATA error, then there was a serious problem. Only in this case, or if the fail +// on the stats measurement, do we return false +bool psphotReadoutCleanupReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status = true; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // when psphotReadoutCleanup is called, these are not necessarily defined + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psArray *sources = detections ? detections->allSources : NULL; + // XXX where do we free these, in here (psMetadataRemove?) + + // XXX this is currently only set by psphotModelBackground / psphotModelBackgroundReadoutFileIndex / psphotModelBackgroundReadout + // if the image background cannot be measured (no valid pixels) + int quality = psMetadataLookupS32 (&status, readout->analysis, "PSPHOT_QUALITY"); + if (quality) { + // if there is no stats file this will be a no-op + psphotStatsFileSetQuality(quality); + } + + // use the psf-model to measure FWHM stats + if (psf) { + if (!psphotPSFstatsSources (readout, sources, psf)) { + psError(PSPHOT_ERR_PROG, false, "Failed to measure PSF shape parameters"); + return false; + } + } + // otherwise, use the source moments to measure FWHM stats + if (!psf && sources) { + if (!psphotMomentsStats (readout, sources)) { + psError(PSPHOT_ERR_PROG, false, "Failed to measure Moment shape parameters"); + return false; + } + } + + // Check to see if any sources were detected + // This is not necessarily a quality error: e.g., ppSub + if (0 && !psf && !sources) { + psError(PSPHOT_ERR_DATA, false, "Unable to detect sources in the image"); + return false; + } + + // Check to see if the image quality was measured + // XXX not sure we want / need this test + if (0 && !psf) { + bool mdok; // Status of MD lookup + int nIQ = psMetadataLookupS32(&mdok, recipe, "IQ_NSTAR"); // Number of stars for IQ measurement + if (!mdok || nIQ <= 0) { + psError(PSPHOT_ERR_DATA, false, "Unable to measure image quality"); + return false; + } + } + + // create an output header with stats results currently saved on readout->analysis + psMetadata *header = psphotDefineHeader (readout->analysis); + + // write NSTARS to the image header + psphotSetHeaderNstars (header, sources); + + // save the results of the analysis + // this should happen way up stream (when needed?) + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSPHOT.HEADER", PS_DATA_METADATA | PS_META_REPLACE, "header stats", header); + + if (psf) { + // XXX this seems a little silly : we saved the psf on readout->analysis above, but now + // we are moving it to chip->analysis. + // save the psf for possible output. if there was already an entry, it was loaded from external sources + // the new one may have been updated or modified, so replace the existing entry. We + // are required to save it on the chip, but this will cause problems if we ever want to + // run psphot on an unmosaiced image + pmCell *cell = readout->parent; + pmChip *chip = cell->parent; + psMetadataAdd (chip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE, "psphot psf", psf); + } + + if (psErrorCodeLast() != PS_ERR_NONE) { + psErrorStackPrint(stderr, "unexpected remaining errors"); + abort(); + } + + psFree (header); + if (quality) return false; + + return true; +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotReadoutCleanupMinimal (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // remove internal pmFPAfiles, if created + if (psErrorCodeLast() == (psErrorCode) PSPHOT_ERR_DATA) { + psErrorStackPrint(stderr, "Error in the psphot readout analysis"); + psErrorClear(); + } + if (psErrorCodeLast() != PS_ERR_NONE) { + return false; + } + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotReadoutCleanupReadoutMinimal (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on psphotReadoutCleanupMinimal for %s entry %d", filerule, i); + return false; + } + } + + // XXX move this to top of loop? + pmKapaClose (); + + return true; +} + +// psphotReadoutCleanupMinimal is called on exit from psphotReadoutMinimal. If the last raised error is +// not a DATA error, then there was a serious problem. Only in this case, or if the fail +// on the stats measurement, do we return false +bool psphotReadoutCleanupReadoutMinimal (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status = true; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // when psphotReadoutCleanupMinimal is called, these are not necessarily defined + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psArray *sources = detections ? detections->allSources : NULL; + // XXX where do we free these, in here (psMetadataRemove?) + + // create an output header with stats results currently saved on readout->analysis + psMetadata *header = psphotDefineHeader (readout->analysis); + + // write NSTARS to the image header + psphotSetHeaderNstars (header, sources); + + // save the results of the analysis + // this should happen way up stream (when needed?) + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSPHOT.HEADER", PS_DATA_METADATA | PS_META_REPLACE, "header stats", header); + + if (psf) { + // XXX this seems a little silly : we saved the psf on readout->analysis above, but now + // we are moving it to chip->analysis. + // save the psf for possible output. if there was already an entry, it was loaded from external sources + // the new one may have been updated or modified, so replace the existing entry. We + // are required to save it on the chip, but this will cause problems if we ever want to + // run psphot on an unmosaiced image + pmCell *cell = readout->parent; + pmChip *chip = cell->parent; + psMetadataAdd (chip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE, "psphot psf", psf); + } + + if (psErrorCodeLast() != PS_ERR_NONE) { + psErrorStackPrint(stderr, "unexpected remaining errors"); + abort(); + } + + psFree (header); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutFindPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutFindPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutFindPSF.c (revision 41083) @@ -0,0 +1,81 @@ +# include "psphotInternal.h" + +// in this psphotReadout-variant, we are only trying to determine the PSF given an existing set +// of input source positions to use as initial PSF stars. +bool psphotReadoutFindPSF(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources) { + + psTimerStart ("psphotReadout"); + + // set the photcode for the input + if (!psphotAddPhotcode(config, view, filerule)) { + psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and variance images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + + // Note that in this implementation, we do NOT model the background and we do not + // attempt to detect the sources in the image + + // include the externally-supplied sources (inSources) + // (we assume a single set of input sources is supplied) + if (!psphotDetectionsFromSources (config, view, filerule, inSources)) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Can't find PSF stars"); + return psphotReadoutCleanup (config, view, filerule); + } + + // construct detections->newSources and measure basic stats (moments, local sky) + if (!psphotSourceStats(config, view, filerule, true)) { + psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); + return false; + } + + // peak flux is wrong : use the peak measured in the moments analysis: + if (!psphotRepairLoadedSources(config, view, filerule)) { + psError(PSPHOT_ERR_UNKNOWN, false, "failure to repair sources"); + return false; + } + + // classify sources based on moments, brightness (psf is not known) + if (!psphotRoughClass (config, view, filerule)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough source class"); + return psphotReadoutCleanup (config, view, filerule); + } + + if (!psphotImageQuality (config, view, filerule)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to measure image quality"); + return psphotReadoutCleanup (config, view, filerule); + } + + if (!psphotChoosePSF(config, view, filerule, true)) { + psError(PSPHOT_ERR_PSF, false, "Failed to construct a psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + +# if 0 + // XXX if we want to determine the aperture residual correction here, we either + // need to carry it out of the PSF determination analysis above, or save the model + // fits from that analysis, or run the linear PSF fit for all objects currently in hand + // construct an initial model for each object, set the radius to fitRadius, set circular fit mask + psphotGuessModels (config, view, filerule); +# endif + +# if 0 + // measure aperture photometry corrections + if (!psphotApResid (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed on psphotApResid"); + return psphotReadoutCleanup (config, view, filerule); + } +# endif + + // drop the references to the image pixels held by each source + psphotSourceFreePixels(config, view, filerule); + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutForcedKnownSources.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutForcedKnownSources.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutForcedKnownSources.c (revision 41083) @@ -0,0 +1,57 @@ +# include "psphotInternal.h" + +// in this psphotReadout-variant, we are only measuring the photometry for known source +// position, using a supplied PSF +bool psphotReadoutForcedKnownSources(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources) { + + psTimerStart ("psphotReadout"); + + // remove cruft from the input analysis structure + if (!psphotCleanInputs (config, view, filerule)) { + psError (PSPHOT_ERR_PROG, false, "trouble setting up the inputs"); + return false; + } + + // set the photcode for this image + if (!psphotAddPhotcode(config, view, filerule)) { + psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + + // Note that in this implementation, we do NOT model the background and we do not + // attempt to detect the sources in the image + + if (!psphotAddKnownSources (config, view, filerule, inSources)) { + psError(PSPHOT_ERR_UNKNOWN, false, "failure to load supplied sources"); + return false; + } + + if (!psphotLoadPSF (config, view, filerule)) { + // this only happens if we had a programming error in psphotLoadPSF + psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, filerule); + + // linear PSF fit to source peaks + psphotFitSourcesLinear (config, view, filerule, false, false); + + // calculate source magnitudes + psphotMagnitudes(config, view, filerule); + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, filerule); + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutKnownSources.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutKnownSources.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutKnownSources.c (revision 41083) @@ -0,0 +1,78 @@ +# include "psphotInternal.h" + +// in this psphotReadout-variant, we are only measuring the photometry for known sources, using +// a PSF generated for this observation from those sources +bool psphotReadoutKnownSources(pmConfig *config, const pmFPAview *view, const char *filerule, psArray *inSources) { + + psTimerStart ("psphotReadout"); + + // set the photcode for this image + if (!psphotAddPhotcode(config, view, filerule)) { + psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + + // Note that in this implementation, we do NOT model the background and we do not + // attempt to detect the sources in the image + + // include externally-supplied sources (supplied as PSPHOT.INPUT.CMF) + if (!psphotDetectionsFromSources (config, view, filerule, inSources)) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Can't find PSF stars"); + return psphotReadoutCleanup (config, view, filerule); + } + + // construct sources and measure basic stats + if (!psphotSourceStats (config, view, filerule, true)) { + psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); + return false; + } + + // peak flux is wrong : use the peak measured in the moments analysis: + if (!psphotRepairLoadedSources(config, view, filerule)) { + psError(PSPHOT_ERR_UNKNOWN, false, "failure to repair sources"); + return false; + } + + // classify sources based on moments, brightness (psf is not known) + if (!psphotRoughClass (config, view, filerule)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough source class"); + return psphotReadoutCleanup (config, view, filerule); + } + + if (!psphotChoosePSF (config, view, filerule, true)) { + psError(PSPHOT_ERR_PSF, false, "Failed to construct a psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, filerule); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, filerule); + + // linear PSF fit to source peaks + psphotFitSourcesLinear (config, view, filerule, false, false); + + // measure aperture photometry corrections + if (!psphotApResid (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed on psphotApResid"); + return psphotReadoutCleanup (config, view, filerule); + } + + // calculate source magnitudes + psphotMagnitudes(config, view, filerule); + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, filerule); + + // set source->nFrames + psphotSetNFrames (config, view, filerule); + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutMinimal.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutMinimal.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReadoutMinimal.c (revision 41083) @@ -0,0 +1,118 @@ +# include "psphotInternal.h" + +// this badly-named function performs photometry assuming (a) a supplied PSF, (b) background +// subtraction, (c) linear psf-model fits only, (d) a prior analysis has supplied the moments +// window parameters. It is currently only being used by ppSub. + +// NOTE: ppSub needs to perform extended source analysis for comets and trails. + +bool psphotReadoutMinimal(pmConfig *config, const pmFPAview *view, const char *filerule) { + + // measure the total elapsed time in psphotReadout. XXX the current threading plan + // for psphot envisions threading within psphotReadout, not multiple threads calling + // the same psphotReadout. In the current plan, this dtime is the elapsed time used + // jointly by the multiple threads, not the total time used by all threads. + psTimerStart ("psphotReadout"); + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); + + // set the photcode for this image + if (!psphotAddPhotcode(config, view, filerule)) { + psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images, including the user-defined analysis region of interest + psphotSetMaskAndVariance (config, view, filerule); + + // only subtract background if needed? + // activate this for a clean test with psphotMinimal. (add to recipe!) + if (0) { + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + if (!psphotSubtractBackground (config, view, filerule)) { + return psphotReadoutCleanup (config, view, filerule); + } + } + + // load the psf model, if suppled. FWHM_X,FWHM_Y,etc are saved on readout->analysis + if (!psphotLoadPSF (config, view, filerule)) { + psError (PSPHOT_ERR_CONFIG, false, "missing psf model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // find the detections (by peak and/or footprint) in the image. (final pass) + if (!psphotFindDetections(config, view, filerule, false)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, filerule); + } + + // construct sources and measure basic stats (saved on detections->newSources) + if (!psphotSourceStats (config, view, filerule, false)) { // pass 1 + psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); + return psphotReadoutCleanup (config, view, filerule); + } + + // find blended neighbors of very saturated stars + psphotDeblendSatstars (config, view, filerule); + + // mark blended peaks PS_SOURCE_BLEND + if (!psphotBasicDeblend (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed on deblend analysis"); + return psphotReadoutCleanup (config, view, filerule); + } + + // classify sources based on moments, brightness (use supplied psf shape parameters) + if (!psphotRoughClass (config, view, filerule)) { + psLogMsg ("psphot", 3, "failed to find a valid PSF clump for image"); + return psphotReadoutCleanup (config, view, filerule); + } + + // merge the newly selected sources into the existing list + psphotMergeSources (config, view, filerule); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + if (!psphotGuessModels (config, view, filerule)) { + psLogMsg ("psphot", 3, "failure to Guess Model"); + return psphotReadoutCleanup (config, view, filerule); + } + + // linear PSF fit to source peaks + psphotFitSourcesLinear (config, view, filerule, false, false); + + // measure the radial profiles to the sky + psphotRadialProfileWings (config, view, filerule); + + // re-measure the kron mags with models subtracted and more appropriate windows + psphotKronIterate(config, view, filerule, 1); + + // MEH -- sensless for SSdfiffs... + psphotChipParams (config, view, filerule); + + // measure source size for the remaining sources + psphotSourceSize (config, view, filerule, false); + + // NOTE: Petrosian and Isophotal mags are not relevant at this time + // psphotExtendedSourceAnalysis (config, view, filerule); + + // in ppSub context, this is used to fit TRAILs (and maybe EXP for comets) + psphotExtendedSourceFits (config, view, filerule); + + // calculate source magnitudes + psphotMagnitudes(config, view, filerule); + + // XXX ensure this is measured if the analysis succeeds (even if quality is low) + if (!psphotEfficiency(config, view, filerule)) { + psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources"); + psErrorClear(); + } + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, filerule); + + // create the exported-metadata and free local data + return psphotReadoutCleanupMinimal (config, view, filerule); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReplaceUnfit.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReplaceUnfit.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotReplaceUnfit.c (revision 41083) @@ -0,0 +1,397 @@ +# include "psphotInternal.h" + +// replace the flux for sources which failed +bool psphotReplaceUnfitSources (psArray *sources, psImageMaskType maskVal) { + + pmSource *source; + + psTimerStart ("psphot.replace"); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // replace other sources? + if (source->mode & PM_SOURCE_MODE_FAIL) goto replace; + continue; + + replace: + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + } + psLogMsg ("psphot.replace", 3, "replace unfitted models: %f sec (%ld objects)\n", psTimerMark ("psphot.replace"), sources->n); + return true; +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotReplaceAllSources (pmConfig *config, const pmFPAview *view, const char *filerule, bool ignoreState) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + // ignore return value. False means "sources not subtracted", not a failure condition + (void) psphotReplaceAllSourcesReadout (config, view, filerule, i, recipe, ignoreState); + } + return true; +} + +// the return state indicates if any sources were actually replaced +bool psphotReplaceAllSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filename, int index, psMetadata *recipe, bool ignoreState) { + + bool status; + + psTimerStart ("psphot.replace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filename, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psLogMsg ("psphot", PS_LOG_DETAIL, "skipping replace all sources for input file %d", index); + return false; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + // if no work to do, should just return false + if (!sources) return false; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + int NpixTotal = 0; + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + if (ignoreState) { + // rely on the type of source to decide if we subtract it or not + + // skip non-astronomical objects (very likely defects) + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + + // do not include CRs in the full ensemble fit + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue; + + // do not include MOMENTS_FAILURES in the fit + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + } else { + // if we respect the state, do not replace unsubtracted sources + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) continue; + } + + pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + NpixTotal += source->pixels->numCols * source->pixels->numRows; + } + + psphotVisualShowImage(readout); + psLogMsg ("psphot.replace", PS_LOG_INFO, "replaced models for %ld objects: %f sec (%d pixels)\n", sources->n, psTimerMark ("psphot.replace"), NpixTotal); + return true; +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotRemoveAllSources (pmConfig *config, const pmFPAview *view, const char *filerule, bool ignoreState) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotRemoveAllSourcesReadout (config, view, filerule, i, recipe, ignoreState)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to replace all sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotRemoveAllSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filename, int index, psMetadata *recipe, bool ignoreState) { + + bool status; + + psTimerStart ("psphot.replace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filename, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + //psAssert (sources, "missing sources?"); + if (!sources) return true; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + if (ignoreState) { + // rely on the type of source to decide if we subtract it or not + + // skip non-astronomical objects (very likely defects) + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + + // do not include CRs in the full ensemble fit + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue; + + // do not include MOMENTS_FAILURES in the fit + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) continue; + } else { + // if we respect the state, only remove unsubtracted sources + if ((source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) continue; + } + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + + psphotVisualShowImage(readout); + psLogMsg ("psphot.replace", PS_LOG_INFO, "replaced models for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.replace")); + return true; +} + +bool psphotRemoveAllSourcesByArray (const psArray *sources, const psMetadata *recipe) { + + bool status; + pmSource *source; + + psTimerStart ("psphot.replace"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // replace other sources? + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) continue; + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + } + psLogMsg ("psphot.replace", PS_LOG_INFO, "replaced models for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.replace")); + return true; +} + +// modify the sources to point at the corresponding pixels for the given filerule +bool psphotRedefinePixels (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotRedefinePixelsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to replace all sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotRedefinePixelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + pmSource *source; + + psTimerStart ("psphot.replace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // XXX the sources have already been copied (merge into here?) + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // sources have not yet been subtracted in this image (but this flag may be raised) + source->tmpFlags &= ~PM_SOURCE_TMPF_SUBTRACTED; + if (!source->modelPSF) continue; + + float Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + float Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + float radius = source->modelPSF->fitRadius; + + // force a redefine to this image + pmSourceFreePixels(source); + pmSourceRedefinePixels (source, readout, Xo, Yo, radius); + } + return true; +} + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotResetModels (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotResetModelsReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to replace all sources for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotResetModelsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + pmSource *source; + + psTimerStart ("psphot.replace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psLogMsg ("psphot", PS_LOG_DETAIL, "skipping reset models for input file %d", index); + return true; + } + + pmSourceFitOptions *fitOptions = NULL; + if (psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS")) { + fitOptions = psMetadataLookupPtr (&status, readout->analysis, "PCM_FIT_OPTIONS"); + psAssert (fitOptions, "missing pcm fit options"); + } + + // XXX the sources have already been copied (merge into here?) + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + + // what fraction of the PSF is used? (radius in pixels : 2 -> 5x5 box) + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + for (int i = 0; i < sources->n; i++) { + source = sources->data[i]; + + // *** we need to cache the 'best' model, and we have 3 cases: + // 1) model is the psf model --> generate from the new psf + // 2) model is an unconvolved extended model --> just cache the copy (not perfect) + // 3) model is a convolved extended model --> re-generate + + // use the 'best' model to cache the model (PSF or EXT : EXT may point at one of modelFits + bool isPSF = false; + pmModel *model = pmSourceGetModel(&isPSF, source); + if (!model) continue; + + float radius = model->fitRadius; // save for future use below + + // regenerate the PSF if the model is a PSF, or if we need the PSF for a PCM + if (isPSF || model->isPCM) { + if (!source->modelPSF) continue; + // the guess central intensity comes from the peak: + float Io = source->peak->rawFlux; + float Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + float Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + + // generate a model for this object with Io = 1.0 + pmModel *modelPSF = pmModelFromPSFforXY(psf, Xo, Yo, Io); + if (modelPSF == NULL) { + psWarning ("Failed to determine PSF model for source at (%f,%f), skipping", Xo, Yo); + continue; + } + + // set the source PSF model + psFree (source->modelPSF); + source->modelPSF = modelPSF; + source->modelPSF->fitRadius = radius; + // model = source->modelPSF; + } + + if (model->isPCM) { + pmPCMdata *pcm = pmPCMinit (source, fitOptions, model, maskVal, psfSize); + if (pcm) { + pmPCMCacheModel (source, maskVal, psfSize, pcm->nsigma); + psFree(pcm); + } else { + psFree(source->modelEXT); + source->modelEXT = NULL; + } + + } else { + pmSourceCacheModel (source, maskVal); // ALLOC x14 (!) + } + } + + psLogMsg ("psphot.replace", PS_LOG_INFO, "subtracted models for %ld objects: %f sec\n", sources->n, psTimerMark ("psphot.replace")); + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRoughClass.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRoughClass.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotRoughClass.c (revision 41083) @@ -0,0 +1,243 @@ +# include "psphotInternal.h" + +# define CHECK_STATUS(S,MSG) { \ + if (!status) { \ + psError(PSPHOT_ERR_CONFIG, false, "missing PSF Clump entry: %s\n", MSG); \ + return false; \ + } } + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotRoughClass (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Rough Class ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotRoughClassReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on rough classification for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotRoughClassReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) { + + bool status; + + psTimerStart ("psphot.rough"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // if we have a PSF, use the existing PSF clump region below + bool havePSF = false; + if (psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF")) { + havePSF = true; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->newSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping rough classification"); + return true; + } + + int NstarsInClump = psMetadataLookupS32 (&status, readout->analysis, "PSF_CLUMP_NSTARS"); + // if NstarsInClump is not defined, use the user-selected option: + if (!status) { + NstarsInClump = 1000; + } + + int ScaleForClump = 1; + if (NstarsInClump >= 12) ScaleForClump = 2; // 4 cells + if (NstarsInClump >= 27) ScaleForClump = 3; // 9 cells + if (NstarsInClump >= 48) ScaleForClump = 4; // 16 cells + if (NstarsInClump > 75) ScaleForClump = 5; // 25 cells + + // we make this measurement on a NxM grid of regions across the readout + int NX = psMetadataLookupS32 (&status, recipe, "PSF_CLUMP_NX"); CHECK_STATUS (status, "PSF_CLUMP_NX"); + int NY = psMetadataLookupS32 (&status, recipe, "PSF_CLUMP_NY"); CHECK_STATUS (status, "PSF_CLUMP_NY"); + + int NXuse, NYuse; + + ScaleForClump = PS_MIN(ScaleForClump, PS_MAX(NX, NY)); + if (NX > NY) { + NXuse = ScaleForClump; + NYuse = (int) (ScaleForClump * (NX / NY) + 0.5); + } else { + NYuse = ScaleForClump; + NXuse = (int) (ScaleForClump * (NY / NX) + 0.5); + } + + psLogMsg ("psphot", 4, "With %d stars, using %d x %d grid for PSF clump\n", NstarsInClump, NXuse, NYuse); + + int dX = readout->image->numCols / NXuse; + int dY = readout->image->numRows / NYuse; + + int nRegion = 0; + for (int ix = 0; ix < NXuse; ix ++) { + for (int iy = 0; iy < NYuse; iy ++) { + + psRegion *region = psRegionAlloc (ix*dX, (ix + 1)*dX, iy*dY, (iy + 1)*dY); + if (!psphotRoughClassRegion (nRegion, region, sources, readout->analysis, recipe, havePSF)) { + psLogMsg ("psphot", 4, "Failed to determine rough classification for region %f,%f - %f,%f\n", + region->x0, region->y0, region->x1, region->y1); + + // If in doubt, it's a PSF + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (!source || !source->peak) { + continue; + } + if (source->peak->x < region->x0) continue; + if (source->peak->x >= region->x1) continue; + if (source->peak->y < region->y0) continue; + if (source->peak->y >= region->y1) continue; + source->type = PM_SOURCE_TYPE_STAR; + } + psFree (region); + continue; + } + psFree (region); + nRegion ++; + } + } + psMetadataAddS32 (readout->analysis, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", nRegion); + + // optional printout of source moments only + psphotDumpMoments (recipe, sources); + + psLogMsg ("psphot.roughclass", PS_LOG_WARN, "rough classification: %f sec\n", psTimerMark ("psphot.rough")); + + psphotVisualPlotMoments (recipe, readout->analysis, sources); + psphotVisualShowRoughClass (sources); + // XXX better visualization: psphotVisualShowFlags (sources); + + return true; +} + +bool psphotRoughClassRegion (int nRegion, psRegion *region, psArray *sources, psMetadata *analysis, psMetadata *recipe, const bool havePSF) { + + bool status; + char regionName[64]; + pmPSFClump psfClump; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskSat = psMetadataLookupImageMask(&status, recipe, "MASK.SAT"); // Mask value for bad pixels + assert (maskSat); + + // the regions are saved on the readout->analysis metadata folder (passed to us as analysis) + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", nRegion); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + if (!regionMD) { + // allocate the region metadata folder and add this region to it. + regionMD = psMetadataAlloc(); + psMetadataAddMetadata (analysis, PS_LIST_TAIL, regionName, PS_META_REPLACE, "psf clump region", regionMD); + psFree (regionMD); + } + psMetadataAddPtr (regionMD, PS_LIST_TAIL, "REGION", PS_DATA_REGION | PS_META_REPLACE, "psf clump region", region); + + if (!havePSF) { + // determine the PSF parameters from the source moment values + // XXX why not save the psfClump as a PTR? + + float PSF_SN_LIM = psMetadataLookupF32(&status, recipe, "PSF_SN_LIM"); psAssert (status, "missing PSF_SN_LIM"); + float MOMENTS_AR_MAX = psMetadataLookupF32(&status, recipe, "MOMENTS_AR_MAX"); psAssert (status, "missing MOMENTS_AR_MAX"); + + float PSF_CLUMP_GRID_SCALE = psMetadataLookupF32(&status, analysis, "PSF_CLUMP_GRID_SCALE"); + if (!status) { + PSF_CLUMP_GRID_SCALE = psMetadataLookupF32(&status, recipe, "PSF_CLUMP_GRID_SCALE"); + psAssert (status, "missing PSF_CLUMP_GRID_SCALE"); + } + float MOMENTS_SX_MAX = psMetadataLookupF32(&status, analysis, "MOMENTS_SX_MAX"); + if (!status) { + MOMENTS_SX_MAX = psMetadataLookupF32(&status, recipe, "MOMENTS_SX_MAX"); + psAssert (status, "missing MOMENTS_SX_MAX"); + } + float MOMENTS_SY_MAX = psMetadataLookupF32(&status, analysis, "MOMENTS_SY_MAX"); + if (!status) { + MOMENTS_SY_MAX = psMetadataLookupF32(&status, recipe, "MOMENTS_SY_MAX"); + psAssert (status, "missing MOMENTS_SY_MAX"); + } + float MOMENTS_SX_MIN = psMetadataLookupF32(&status, analysis, "MOMENTS_SX_MIN"); + if (!status) { + MOMENTS_SX_MIN = psMetadataLookupF32(&status, recipe, "MOMENTS_SX_MIN"); + if (!status) { + MOMENTS_SX_MIN = 0.5; + } + } + float MOMENTS_SY_MIN = psMetadataLookupF32(&status, analysis, "MOMENTS_SY_MIN"); + if (!status) { + MOMENTS_SY_MIN = psMetadataLookupF32(&status, recipe, "MOMENTS_SY_MIN"); + if (!status) { + MOMENTS_SY_MIN = 0.5; + } + } + + psfClump = pmSourcePSFClump (NULL, region, sources, PSF_SN_LIM, PSF_CLUMP_GRID_SCALE, MOMENTS_SX_MAX, MOMENTS_SY_MAX, MOMENTS_SX_MIN, MOMENTS_SY_MIN, MOMENTS_AR_MAX); + + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); + } else { + // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y + psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + if (!status) { + psLogMsg ("psphot", 4, "No PSF clump defined for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + return false; + } + psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); psAssert (status, "missing PSF.CLUMP.Y"); + psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); psAssert (status, "missing PSF.CLUMP.DX"); + psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); psAssert (status, "missing PSF.CLUMP.DY"); + } + + if (psfClump.X < 0) { + psError(PSPHOT_ERR_PROG, false, "programming error calling pmSourcePSFClump"); + return false; + } + if (!psfClump.X || !psfClump.Y || isnan(psfClump.X) || isnan(psfClump.Y)) { + psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + return false; + } + if (!havePSF) { + psLogMsg ("psphot", 3, "psf clump X, Y: %f, %f : DX, DY: %f, %f : nStars %d of %d\n", psfClump.X, psfClump.Y, psfClump.dX, psfClump.dY, psfClump.nStars, psfClump.nTotal); + } else { + psLogMsg ("psphot", 3, "psf clump X, Y: %f, %f : DX, DY: %f, %f : loaded from metadata\n", psfClump.X, psfClump.Y, psfClump.dX, psfClump.dY); + } + + // get basic parameters, or set defaults + float PSF_SN_LIM = psMetadataLookupF32 (&status, recipe, "PSF_SN_LIM"); psAssert (status, "missing PSF_SN_LIM"); + float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, recipe, "PSF_CLUMP_NSIGMA"); psAssert (status, "missing PSF_CLUMP_NSIGMA"); + + // group into STAR, COSMIC, EXTENDED, SATURATED, etc. + if (!pmSourceRoughClass (region, sources, PSF_SN_LIM, PSF_CLUMP_NSIGMA, psfClump, maskSat)) { + psError(PSPHOT_ERR_PROG, false, "programming error calling pmSourceRoughClass"); + return false; + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSavePSFStars.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSavePSFStars.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSavePSFStars.c (revision 41083) @@ -0,0 +1,148 @@ +# include "psphotInternal.h" + +bool psphotSourcePlots (pmReadout *readout, psArray *sources, psMetadata *recipe) { + + // bool status = false; + psTimerStart ("psphot"); + + // if this file is defined, create the necessary output data + pmFPAfile *file = psMetadataLookupPtr(&status, config->files, "PSPHOT.PSF.STARS"); + + // the source images are written to an image 10x the size of a PSF object + // float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + // PS_ASSERT (status, false); + + int DX = 21; + int DY = 21; + + // examine PSF sources in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // counters to track the size of the image and area used in a row + int dX = 0; // starting corner of next box + int dY = 0; // height of row so far + int NX = 20*DX; // full width of output image + int NY = 0; // total height of output image + + // first, examine the PSF and SAT stars: + // - determine bounding boxes for summary image + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + bool keep = false; + keep |= (source->mode & PM_SOURCE_MODE_PSFSTAR); + keep |= (source->mode & PM_SOURCE_MODE_SATSTAR); + if (!keep) continue; + + // how does this subimage get placed into the output image? + // DX = source->pixels->numCols + // DY = source->pixels->numRows + + if (dX + DX > NX) { + // too wide for the rest of this row + if (dX == 0) { + // alone on this row + NY += DY; + dX = 0; + dY = 0; + } else { + // start the next row + NY += dY; + dX = DX; + dY = DY; + } + } else { + // extend this row + dX += DX; + dY = PS_MAX (dY, DY); + } + } + + // allocate output image + psImage *outpos = psImageAlloc (NX, NY, PS_TYPE_F32); + psImage *outsub = psImageAlloc (NX, NY, PS_TYPE_F32); + + int Xo = 0; // starting corner of next box + int Yo = 0; // starting corner of next box + dY = 0; // height of row so far + + int nPSF = 0; + int nSAT = 0; + int kapa = 0; // file descriptor for plotting routine + + // first, examine the PSF and SAT stars: + // - generate radial plots (PS plots) + // - create output image array + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + bool keep = false; + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + nPSF ++; + keep = true; + } + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + nSAT ++; + keep = true; + } + if (!keep) continue; + + // how does this subimage get placed into the output image? + // DX = source->pixels->numCols + // DY = source->pixels->numRows + + if (Xo + DX > NX) { + // too wide for the rest of this row + if (Xo == 0) { + // place source alone on this row + psphotAddWithTest (source, true); // replace source if subtracted + psphotRadialPlot (&kapa, "radial.plots.ps", source); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + psphotSubWithTest (source, false); // remove source (force) + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + + psphotSetState (source, false); // replace source (has been subtracted) + Yo += DY; + Xo = 0; + dY = 0; + } else { + // start the next row + Yo += dY; + Xo = 0; + psphotAddWithTest (source, true); // replace source if subtracted + psphotRadialPlot (&kapa, "radial.plots.ps", source); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + psphotSubWithTest (source, false); // remove source (force) + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + psphotSetState (source, false); // replace source (has been subtracted) + + Xo = DX; + dY = DY; + } + } else { + // extend this row + psphotAddWithTest (source, true); // replace source if subtracted + psphotRadialPlot (&kapa, "radial.plots.ps", source); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + psphotSubWithTest (source, false); // remove source (force) + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + psphotSetState (source, false); // replace source (has been subtracted) + + Xo += DX; + dY = PS_MAX (dY, DY); + } + } + + psphotSaveImage (NULL, outpos, "outpos.fits"); + psphotSaveImage (NULL, outsub, "outsub.fits"); + psLogMsg ("psphot", PS_LOG_INFO, "plotted %d sources (%d psf, %d sat): %f sec\n", nPSF + nSAT, nPSF, nSAT, psTimerMark ("psphot")); + + psFree (outpos); + psFree (outsub); + return (sources); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.00.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.00.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.00.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: -0.445996271922, C1: 0.00257774203952 + class->IndexToMajor->coeff[0] = -0.445996271922; + class->IndexToMajor->coeff[1] = 0.00257774203952; + + // minor: C0: -0.442221950965, C1: 0.00310744939505 + class->IndexToMinor->coeff[0] = -0.442221950965; + class->IndexToMinor->coeff[1] = 0.00310744939505; + + // MagOffset: C0: -0.0096577152343, C1: -0.04709419632 + class->IndexToTotalMag->coeff[0] = -0.0096577152343; + class->IndexToTotalMag->coeff[1] = -0.04709419632; + + // KronMag: C0: -0.396760779784, C1: -0.0144654449553, C2: 0.00688206755937 + class->KronToIndex->coeff[0] = -0.396760779784; + class->KronToIndex->coeff[1] = -0.0144654449553; + class->KronToIndex->coeff[2] = 0.00688206755937; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.01.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.01.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.01.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: -0.150653602233, C1: 0.0583316775309 + class->IndexToMajor->coeff[0] = -0.150653602233; + class->IndexToMajor->coeff[1] = 0.0583316775309; + + // minor: C0: -0.146015984633, C1: 0.0584414894493 + class->IndexToMinor->coeff[0] = -0.146015984633; + class->IndexToMinor->coeff[1] = 0.0584414894493; + + // MagOffset: C0: -0.0516576246324, C1: -0.06308349488 + class->IndexToTotalMag->coeff[0] = -0.0516576246324; + class->IndexToTotalMag->coeff[1] = -0.06308349488; + + // KronMag: C0: -1.28498429043, C1: 0.255293518143, C2: -0.0209474725898 + class->KronToIndex->coeff[0] = -1.28498429043; + class->KronToIndex->coeff[1] = 0.255293518143; + class->KronToIndex->coeff[2] = -0.0209474725898; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.02.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.02.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.02.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.0700235952113, C1: 0.0832659513906 + class->IndexToMajor->coeff[0] = 0.0700235952113; + class->IndexToMajor->coeff[1] = 0.0832659513906; + + // minor: C0: 0.0778318261346, C1: 0.082428734924 + class->IndexToMinor->coeff[0] = 0.0778318261346; + class->IndexToMinor->coeff[1] = 0.082428734924; + + // MagOffset: C0: -0.392754912322, C1: -0.0271438531326 + class->IndexToTotalMag->coeff[0] = -0.392754912322; + class->IndexToTotalMag->coeff[1] = -0.0271438531326; + + // KronMag: C0: -2.24835974564, C1: 0.543586359075, C2: -0.0521214274524 + class->KronToIndex->coeff[0] = -2.24835974564; + class->KronToIndex->coeff[1] = 0.543586359075; + class->KronToIndex->coeff[2] = -0.0521214274524; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.03.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.03.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.00.03.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.302234153612, C1: 0.0812072518454 + class->IndexToMajor->coeff[0] = 0.302234153612; + class->IndexToMajor->coeff[1] = 0.0812072518454; + + // minor: C0: 0.318735595658, C1: 0.0782450906783 + class->IndexToMinor->coeff[0] = 0.318735595658; + class->IndexToMinor->coeff[1] = 0.0782450906783; + + // MagOffset: C0: -0.442722048421, C1: -0.0779760825812 + class->IndexToTotalMag->coeff[0] = -0.442722048421; + class->IndexToTotalMag->coeff[1] = -0.0779760825812; + + // KronMag: C0: -4.13616754915, C1: 1.31952307132, C2: -0.135246127816 + class->KronToIndex->coeff[0] = -4.13616754915; + class->KronToIndex->coeff[1] = 1.31952307132; + class->KronToIndex->coeff[2] = -0.135246127816; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.00.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.00.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.00.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: -0.206834286782, C1: -0.00161061048758 + class->IndexToMajor->coeff[0] = -0.206834286782; + class->IndexToMajor->coeff[1] = -0.00161061048758; + + // minor: C0: -0.206834286782, C1: -0.00161061048758 + class->IndexToMinor->coeff[0] = -0.206834286782; + class->IndexToMinor->coeff[1] = -0.00161061048758; + + // MagOffset: C0: -0.019943291705, C1: -0.032180165617 + class->IndexToTotalMag->coeff[0] = -0.019943291705; + class->IndexToTotalMag->coeff[1] = -0.032180165617; + + // KronMag: C0: -0.184713930007, C1: -0.0151666534408, C2: 0.00506733444285 + class->KronToIndex->coeff[0] = -0.184713930007; + class->KronToIndex->coeff[1] = -0.0151666534408; + class->KronToIndex->coeff[2] = 0.00506733444285; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.01.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.01.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.01.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.10199171121, C1: 0.0370901734428 + class->IndexToMajor->coeff[0] = 0.10199171121; + class->IndexToMajor->coeff[1] = 0.0370901734428; + + // minor: C0: -0.64374610969, C1: 0.0615101610281 + class->IndexToMinor->coeff[0] = -0.64374610969; + class->IndexToMinor->coeff[1] = 0.0615101610281; + + // MagOffset: C0: -0.0424533119058, C1: -0.051484524864 + class->IndexToTotalMag->coeff[0] = -0.0424533119058; + class->IndexToTotalMag->coeff[1] = -0.051484524864; + + // KronMag: C0: -0.79489754688, C1: 0.140962833359, C2: -0.0081614929638 + class->KronToIndex->coeff[0] = -0.79489754688; + class->KronToIndex->coeff[1] = 0.140962833359; + class->KronToIndex->coeff[2] = -0.0081614929638; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.02.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.02.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.02.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.253878808908, C1: 0.063184490483 + class->IndexToMajor->coeff[0] = 0.253878808908; + class->IndexToMajor->coeff[1] = 0.063184490483; + + // minor: C0: -0.386724307864, C1: 0.116445334002 + class->IndexToMinor->coeff[0] = -0.386724307864; + class->IndexToMinor->coeff[1] = 0.116445334002; + + // MagOffset: C0: -0.0872953973638, C1: -0.0728614138338 + class->IndexToTotalMag->coeff[0] = -0.0872953973638; + class->IndexToTotalMag->coeff[1] = -0.0728614138338; + + // KronMag: C0: -1.85587783211, C1: 0.447839373687, C2: -0.0374898116232 + class->KronToIndex->coeff[0] = -1.85587783211; + class->KronToIndex->coeff[1] = 0.447839373687; + class->KronToIndex->coeff[2] = -0.0374898116232; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.03.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.03.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.01.03.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.440682409806, C1: 0.0655243267517 + class->IndexToMajor->coeff[0] = 0.440682409806; + class->IndexToMajor->coeff[1] = 0.0655243267517; + + // minor: C0: -0.0588380249835, C1: 0.125842487938 + class->IndexToMinor->coeff[0] = -0.0588380249835; + class->IndexToMinor->coeff[1] = 0.125842487938; + + // MagOffset: C0: -0.559338014369, C1: -0.0595570268259 + class->IndexToTotalMag->coeff[0] = -0.559338014369; + class->IndexToTotalMag->coeff[1] = -0.0595570268259; + + // KronMag: C0: -3.19558263789, C1: 1.10848141303, C2: -0.127301553664 + class->KronToIndex->coeff[0] = -3.19558263789; + class->KronToIndex->coeff[1] = 1.10848141303; + class->KronToIndex->coeff[2] = -0.127301553664; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.00.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.00.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.00.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: -0.0912134941416, C1: -0.000669340733451 + class->IndexToMajor->coeff[0] = -0.0912134941416; + class->IndexToMajor->coeff[1] = -0.000669340733451; + + // minor: C0: -0.0912134941416, C1: -0.000669340733451 + class->IndexToMinor->coeff[0] = -0.0912134941416; + class->IndexToMinor->coeff[1] = -0.000669340733451; + + // MagOffset: C0: -0.0221727399583, C1: -0.0223898759145 + class->IndexToTotalMag->coeff[0] = -0.0221727399583; + class->IndexToTotalMag->coeff[1] = -0.0223898759145; + + // KronMag: C0: -0.104030916493, C1: 0.000705683288561, C2: 0.00180404239128 + class->KronToIndex->coeff[0] = -0.104030916493; + class->KronToIndex->coeff[1] = 0.000705683288561; + class->KronToIndex->coeff[2] = 0.00180404239128; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.01.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.01.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.01.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.231254014758, C1: 0.0296484411276 + class->IndexToMajor->coeff[0] = 0.231254014758; + class->IndexToMajor->coeff[1] = 0.0296484411276; + + // minor: C0: -1.02822862517, C1: 0.0541915833825 + class->IndexToMinor->coeff[0] = -1.02822862517; + class->IndexToMinor->coeff[1] = 0.0541915833825; + + // MagOffset: C0: -0.038775941514, C1: -0.0429605069959 + class->IndexToTotalMag->coeff[0] = -0.038775941514; + class->IndexToTotalMag->coeff[1] = -0.0429605069959; + + // KronMag: C0: -0.598266540255, C1: 0.144570984447, C2: -0.0107033922884 + class->KronToIndex->coeff[0] = -0.598266540255; + class->KronToIndex->coeff[1] = 0.144570984447; + class->KronToIndex->coeff[2] = -0.0107033922884; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.02.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.02.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.02.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.390176771343, C1: 0.047444773551 + class->IndexToMajor->coeff[0] = 0.390176771343; + class->IndexToMajor->coeff[1] = 0.047444773551; + + // minor: C0: -0.65406916551, C1: 0.124100778941 + class->IndexToMinor->coeff[0] = -0.65406916551; + class->IndexToMinor->coeff[1] = 0.124100778941; + + // MagOffset: C0: -0.0701907061504, C1: -0.0669032246792 + class->IndexToTotalMag->coeff[0] = -0.0701907061504; + class->IndexToTotalMag->coeff[1] = -0.0669032246792; + + // KronMag: C0: -1.41116916222, C1: 0.339914508589, C2: -0.0257471931693 + class->KronToIndex->coeff[0] = -1.41116916222; + class->KronToIndex->coeff[1] = 0.339914508589; + class->KronToIndex->coeff[2] = -0.0257471931693; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.03.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.03.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.02.03.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.56620879654, C1: 0.0423613195169 + class->IndexToMajor->coeff[0] = 0.56620879654; + class->IndexToMajor->coeff[1] = 0.0423613195169; + + // minor: C0: -0.206040045432, C1: 0.120238710256 + class->IndexToMinor->coeff[0] = -0.206040045432; + class->IndexToMinor->coeff[1] = 0.120238710256; + + // MagOffset: C0: -0.620063142691, C1: 0.00335965878994 + class->IndexToTotalMag->coeff[0] = -0.620063142691; + class->IndexToTotalMag->coeff[1] = 0.00335965878994; + + // KronMag: C0: -2.34864255166, C1: 0.766720904286, C2: -0.0877245543247 + class->KronToIndex->coeff[0] = -2.34864255166; + class->KronToIndex->coeff[1] = 0.766720904286; + class->KronToIndex->coeff[2] = -0.0877245543247; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.00.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.00.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.00.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.00265274254899, C1: -0.00199264363239 + class->IndexToMajor->coeff[0] = 0.00265274254899; + class->IndexToMajor->coeff[1] = -0.00199264363239; + + // minor: C0: 0.00265274254899, C1: -0.00199264363239 + class->IndexToMinor->coeff[0] = 0.00265274254899; + class->IndexToMinor->coeff[1] = -0.00199264363239; + + // MagOffset: C0: -0.0238043163179, C1: -0.00786669933119 + class->IndexToTotalMag->coeff[0] = -0.0238043163179; + class->IndexToTotalMag->coeff[1] = -0.00786669933119; + + // KronMag: C0: -0.0238043163179, C1: -0.00786669933119, C2: -0.0877245543247 + class->KronToIndex->coeff[0] = -0.0238043163179; + class->KronToIndex->coeff[1] = -0.00786669933119; + class->KronToIndex->coeff[2] = -0.0877245543247; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.01.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.01.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.01.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.391301293845, C1: 0.0144842856382 + class->IndexToMajor->coeff[0] = 0.391301293845; + class->IndexToMajor->coeff[1] = 0.0144842856382; + + // minor: C0: 0.391301293845, C1: 0.0144842856382 + class->IndexToMinor->coeff[0] = 0.391301293845; + class->IndexToMinor->coeff[1] = 0.0144842856382; + + // MagOffset: C0: -0.0471179132852, C1: -0.0233493549339 + class->IndexToTotalMag->coeff[0] = -0.0471179132852; + class->IndexToTotalMag->coeff[1] = -0.0233493549339; + + // KronMag: C0: -0.264947544849, C1: 0.0691808996903, C2: -0.00434994287966 + class->KronToIndex->coeff[0] = -0.264947544849; + class->KronToIndex->coeff[1] = 0.0691808996903; + class->KronToIndex->coeff[2] = -0.00434994287966; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.02.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.02.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.02.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.526962575335, C1: 0.0325858343301 + class->IndexToMajor->coeff[0] = 0.526962575335; + class->IndexToMajor->coeff[1] = 0.0325858343301; + + // minor: C0: -1.02251162695, C1: 0.118560104626 + class->IndexToMinor->coeff[0] = -1.02251162695; + class->IndexToMinor->coeff[1] = 0.118560104626; + + // MagOffset: C0: -0.0716688127431, C1: -0.0502414483962 + class->IndexToTotalMag->coeff[0] = -0.0716688127431; + class->IndexToTotalMag->coeff[1] = -0.0502414483962; + + // KronMag: C0: -0.985092751487, C1: 0.272660295656, C2: -0.0202614753082 + class->KronToIndex->coeff[0] = -0.985092751487; + class->KronToIndex->coeff[1] = 0.272660295656; + class->KronToIndex->coeff[2] = -0.0202614753082; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.03.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.03.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicGuess.03.03.h (revision 41083) @@ -0,0 +1,17 @@ + // major: C0: 0.694732619604, C1: 0.0222519100811 + class->IndexToMajor->coeff[0] = 0.694732619604; + class->IndexToMajor->coeff[1] = 0.0222519100811; + + // minor: C0: -0.346745057966, C1: 0.0949650739266 + class->IndexToMinor->coeff[0] = -0.346745057966; + class->IndexToMinor->coeff[1] = 0.0949650739266; + + // MagOffset: C0: -0.754034109564, C1: 0.071069480015 + class->IndexToTotalMag->coeff[0] = -0.754034109564; + class->IndexToTotalMag->coeff[1] = 0.071069480015; + + // KronMag: C0: -1.16844402687, C1: 0.174859870338, C2: -0.00252564582698 + class->KronToIndex->coeff[0] = -1.16844402687; + class->KronToIndex->coeff[1] = 0.174859870338; + class->KronToIndex->coeff[2] = -0.00252564582698; + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicModelClass.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicModelClass.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSersicModelClass.c (revision 41083) @@ -0,0 +1,662 @@ +# include "psphotInternal.h" + +psF64 psPolynomial1DSolve(const psPolynomial1D* poly, psF64 y, bool upper); +void psphotSersicModelNorm (pmPCMdata *pcm, const pmSource *source); + +static psArray *classArray = NULL; +static float PSFratioMin = 1.0; +static float PSFratioMax = 4.0; +static float ASPratioMin = 0.2; +static float ASPratioMax = 1.0; + +// structure to define the sersic model guess information +typedef struct { + float PSFratioMin; + float PSFratioMax; + float ASPratioMin; + float ASPratioMax; + + float indexMin; + float indexMax; + float kronMin; + float kronMax; + + bool KronUpper; + psPolynomial1D *KronToIndex; + psPolynomial1D *IndexToTotalMag; + psPolynomial1D *IndexToMajor; + psPolynomial1D *IndexToMinor; +} psphotSersicModelClass; + +static void psphotSersicModelClassFree (psphotSersicModelClass *class) { + psFree(class->KronToIndex); + psFree(class->IndexToTotalMag); + psFree(class->IndexToMajor); + psFree(class->IndexToMinor); + return; +} + +psphotSersicModelClass *psphotSersicModelClassAlloc() { + + psphotSersicModelClass *class = (psphotSersicModelClass *) psAlloc(sizeof(psphotSersicModelClass)); + psMemSetDeallocator(class, (psFreeFunc) psphotSersicModelClassFree); + + class->PSFratioMin = NAN; + class->PSFratioMax = NAN; + class->ASPratioMin = NAN; + class->ASPratioMax = NAN; + + class->KronUpper = false; + class->KronToIndex = NULL; + class->IndexToTotalMag = NULL; + class->IndexToMajor = NULL; + class->IndexToMinor = NULL; + + return class; +} + +void psphotSersicModelClassSetIndexRange(psphotSersicModelClass *class, float indexMin, float indexMax) { + + class->indexMin = indexMin; + class->indexMax = indexMax; + float modelMinX = -0.5 * class->KronToIndex->coeff[1] / class->KronToIndex->coeff[2]; + float modelMinY = psPolynomial1DEval (class->KronToIndex, modelMinX); + float kronIndexMin = psPolynomial1DEval (class->KronToIndex, class->indexMin); + float kronIndexMax = psPolynomial1DEval (class->KronToIndex, class->indexMax); + + // if modelMinX is between indexMin and indexMax, use the modelMinY as the min/max kron value + // else, use the kron values at the min and max positions + + if ((modelMinX > class->indexMin) && (modelMinX < class->indexMax)) { + if (class->KronToIndex->coeff[2] < 0.0) { + class->kronMax = modelMinY - 0.001; // pad slightly to avoid falling off the curve + class->kronMin = PS_MIN(kronIndexMin, kronIndexMax); + } else { + class->kronMin = modelMinY + 0.001; // pad slightly to avoid falling off the curve + class->kronMax = PS_MAX(kronIndexMin, kronIndexMax); + } + } else { + class->kronMin = PS_MIN(kronIndexMin, kronIndexMax); + class->kronMax = PS_MAX(kronIndexMin, kronIndexMax); + } + + // KronUpper specifies which of the 2 quadratic solutions to accept: + if (modelMinX < class->indexMin) { + if (class->KronToIndex->coeff[2] < 0.0) { + class->KronUpper = false; + } else { + class->KronUpper = true; + } + return; + } + if (modelMinX > class->indexMax) { + if (class->KronToIndex->coeff[2] < 0.0) { + class->KronUpper = true; + } else { + class->KronUpper = false; + } + return; + } + + if (fabs(modelMinX - class->indexMin) < fabs(modelMinX - class->indexMax)) { + if (class->KronToIndex->coeff[2] < 0.0) { + class->KronUpper = false; + } else { + class->KronUpper = true; + } + return; + } + if (class->KronToIndex->coeff[2] < 0.0) { + class->KronUpper = true; + } else { + class->KronUpper = false; + } + return; +} + +/* the Sersic model class guess system uses a set of empirically-determined fits to several + * relationships: index vs psf-kron mags, total mag vs index, major axis vs index, minor axis + * vs index. all are functions of the aspect ratio and of the ratio between the measure major + * axis and the size of the psf. Below we have the fits for these different ranges. + + * aspect ratio values: 1.0, 0.5, 0.33, 0.2 + * psf ratio values: 0.5, 1.0, 2.0, 4.0 + + * psf ratio ranges: 0.5-0.8, 0.8-1.5, 1.5-3.0, 3.0-4.0 + * apt ratio ranges: 1.0-0.71, 0.71-0.41, 0.41-0.25, 0.25-0.20 + + */ + +void psphotSersicModelClassInit () { + + psphotSersicModelClass *class; + + if (classArray) return; + + // hardwired trends for now (move into the recipe?) + classArray = psArrayAllocEmpty (4); + + // image.00.00.fit.dat: PSFratio : 0.5, ASPratio : 1.0 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.5; + class->PSFratioMax = 0.8; + class->ASPratioMin = 0.71; + class->ASPratioMax = 1.00; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.00.00.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.00.01.fit.dat: PSFratio : 1.0, ASPratio : 1.0 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.8; + class->PSFratioMax = 1.5; + class->ASPratioMin = 0.71; + class->ASPratioMax = 1.00; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.00.01.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.00.02.fit.dat: PSFratio : 2.0, ASPratio : 1.0 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 1.5; + class->PSFratioMax = 3.0; + class->ASPratioMin = 0.71; + class->ASPratioMax = 1.00; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.00.02.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.00.03.fit.dat: PSFratio : 4.0, ASPratio : 1.0 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 3.0; + class->PSFratioMax = 4.0; + class->ASPratioMin = 0.71; + class->ASPratioMax = 1.00; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.00.03.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.01.00.fit.dat: PSFratio : 0.5, ASPratio : 0.5 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.5; + class->PSFratioMax = 0.8; + class->ASPratioMin = 0.41; + class->ASPratioMax = 0.71; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.01.00.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.01.01.fit.dat: PSFratio : 1.0, ASPratio : 0.5 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.8; + class->PSFratioMax = 1.5; + class->ASPratioMin = 0.41; + class->ASPratioMax = 0.71; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.01.01.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.01.02.fit.dat: PSFratio : 2.0, ASPratio : 0.5 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 1.5; + class->PSFratioMax = 3.0; + class->ASPratioMin = 0.41; + class->ASPratioMax = 0.71; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.01.02.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.01.03.fit.dat: PSFratio : 4.0, ASPratio : 0.5 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 3.0; + class->PSFratioMax = 4.0; + class->ASPratioMin = 0.41; + class->ASPratioMax = 0.71; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.01.03.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.02.00.fit.dat: PSFratio : 0.5, ASPratio : 0.33 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.5; + class->PSFratioMax = 0.8; + class->ASPratioMin = 0.25; + class->ASPratioMax = 0.41; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.02.00.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.02.01.fit.dat: PSFratio : 1.0, ASPratio : 0.33 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.8; + class->PSFratioMax = 1.5; + class->ASPratioMin = 0.25; + class->ASPratioMax = 0.41; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.02.01.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.02.02.fit.dat: PSFratio : 2.0, ASPratio : 0.33 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 1.5; + class->PSFratioMax = 3.0; + class->ASPratioMin = 0.25; + class->ASPratioMax = 0.41; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.02.02.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.02.03.fit.dat: PSFratio : 4.0, ASPratio : 0.33 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 3.0; + class->PSFratioMax = 4.0; + class->ASPratioMin = 0.25; + class->ASPratioMax = 0.41; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.02.03.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.03.00.fit.dat: PSFratio : 0.5, ASPratio : 0.2 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.5; + class->PSFratioMax = 0.8; + class->ASPratioMin = 0.20; + class->ASPratioMax = 0.25; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.03.00.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.03.01.fit.dat: PSFratio : 1.0, ASPratio : 0.2 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 0.8; + class->PSFratioMax = 1.5; + class->ASPratioMin = 0.20; + class->ASPratioMax = 0.25; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.03.01.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.03.02.fit.dat: PSFratio : 2.0, ASPratio : 0.2 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 1.5; + class->PSFratioMax = 3.0; + class->ASPratioMin = 0.20; + class->ASPratioMax = 0.25; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.03.02.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); + + // image.03.03.fit.dat: PSFratio : 4.0, ASPratio : 0.2 + class = psphotSersicModelClassAlloc(); + + class->PSFratioMin = 3.0; + class->PSFratioMax = 4.0; + class->ASPratioMin = 0.20; + class->ASPratioMax = 0.25; + class->KronToIndex = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); + class->IndexToTotalMag = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMajor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + class->IndexToMinor = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 1); + +# include "psphotSersicGuess.03.03.h" + psphotSersicModelClassSetIndexRange(class, 1.0, 4.0); + + psArrayAdd(classArray, 4, class); + psFree (class); +} + +void psphotSersicModelClassCleanup () { + psFree (classArray); +} + +# define TIMING false + +bool psphotSersicModelClassGuessPCM (pmPCMdata *pcm, pmSource *source) { + + float t1, t2, t4, t5; + if (TIMING) { psTimerStart ("SersicGuess"); } + + psF32 *PAR = pcm->modelConv->params->data.F32; + + // XXX require: moments, psfMag, moments->kronFlux + + // we attempt to guess the Sersic model parameters based on a number of non-parametric + // measurements: moments, kron mag, etc + + // * first choose the model class based on (a) ratio of Moments Major axis to PSF major + // * axis and (b) moments axial ratio + + // convert the moments to Major,Minor,Theta + psEllipseMoments moments; + + moments.x2 = source->moments->Mxx; + moments.y2 = source->moments->Myy; + moments.xy = source->moments->Mxy; + + // limit axis ratio < 20.0 + psEllipseAxes momentAxes = psEllipseMomentsToAxes (moments, 20.0); + + // convert the PSF shape to Major,Minor,Theta + psEllipseShape shape; + + // XXX make sure this is consistent with the re-definition of PM_PAR_SXX + shape.sx = source->modelPSF->params->data.F32[PM_PAR_SXX] / M_SQRT2; + shape.sy = source->modelPSF->params->data.F32[PM_PAR_SYY] / M_SQRT2; + shape.sxy = source->modelPSF->params->data.F32[PM_PAR_SXY]; + psEllipseAxes psfAxes = psEllipseShapeToAxes (shape, 20.0); + + // get the PSFratio and the ASPratio : we use these to choose the model class + float PSFratio = source->moments->Mrf / (2.35 * psfAxes.major) ; + float ASPratio = momentAxes.minor / momentAxes.major; + + // saturate PSFratio and ASPratio to min / max values + PSFratio = PS_MAX(PSFratio, PSFratioMin); + PSFratio = PS_MIN(PSFratio, PSFratioMax); + ASPratio = PS_MAX(ASPratio, ASPratioMin); + ASPratio = PS_MIN(ASPratio, ASPratioMax); + + // find the containing model class: + + if (TIMING) { t1 = psTimerMark ("SersicGuess"); } + + psphotSersicModelClass *class = NULL; + for (int i = 0; i < classArray->n; i++) { + psphotSersicModelClass *thisClass = classArray->data[i]; + if (PSFratio < thisClass->PSFratioMin) continue; + if (PSFratio > thisClass->PSFratioMax) continue; + if (ASPratio < thisClass->ASPratioMin) continue; + if (ASPratio > thisClass->ASPratioMax) continue; + class = thisClass; + break; + } + + if (TIMING) { t2 = psTimerMark ("SersicGuess"); } + + psAssert (class, "PSFratio and ASPratio must be in range"); + + // get the index guess from the KronMag - psfMag: + + // get dKronMag & saturate dKronMag at limits of valid range + float dKronMag = -2.5*log10(source->moments->KronFlux) - source->psfMag; + dKronMag = PS_MIN(dKronMag, class->kronMax); + dKronMag = PS_MAX(dKronMag, class->kronMin); + + // get index (saturate at valid ends) + float index = psPolynomial1DSolve (class->KronToIndex, dKronMag, class->KronUpper); + index = PS_MIN (index, class->indexMax); + index = PS_MAX (index, class->indexMin); + + // float totalMagOffset = psPolynomial1DEval (class->IndexToTotalMag, index); + // float totalMag = -2.5*log10(source->moments->KronFlux) + totalMagOffset; + + // need to go from totalMag to Io in the sersic model + + float majorAxisFactor = psPolynomial1DEval (class->IndexToMajor, index); + float minorAxisFactor = psPolynomial1DEval (class->IndexToMinor, index); + + momentAxes.major *= (1.0 + majorAxisFactor); + momentAxes.minor *= (1.0 + minorAxisFactor); + + psEllipseShape extShapeGuess = psEllipseAxesToShape (momentAxes); + if (!isfinite(extShapeGuess.sx)) return false; + if (!isfinite(extShapeGuess.sy)) return false; + if (!isfinite(extShapeGuess.sxy)) return false; + + // set the actual model parameters: + + // index is standard sersic index (n = 1-4), but PAR7 = 1/2n + PAR[PM_PAR_7] = 0.5 / index; + + // sky is zero (no longer fitted, but not yet deprecated) + PAR[PM_PAR_SKY] = 0.0; + + // set the model position + if (!pmModelSetPosition(&PAR[PM_PAR_XPOS], &PAR[PM_PAR_YPOS], source)) { + return false; + } + PAR[PM_PAR_SXX] = extShapeGuess.sx * M_SQRT2; + PAR[PM_PAR_SXY] = extShapeGuess.sxy; + PAR[PM_PAR_SYY] = extShapeGuess.sy * M_SQRT2; + + // XXX this is a bit of a waste: calculate the flux with Io = 1.0, then renormalize. + PAR[PM_PAR_I0] = 1.0; + + if (TIMING) { t4 = psTimerMark ("SersicGuess"); } + + // set the normalization by linear fit between model and data + psphotSersicModelNorm (pcm, source); + if (!isfinite(PAR[PM_PAR_I0])) { + fprintf(stderr, "psphotSersicModelClassGuessPCM: psphotSerisicModelNorm set PM_PAR_I0 to NAN\n"); + return false; + } + + if (TIMING) { t5 = psTimerMark ("SersicGuess"); } + + if (TIMING) { + fprintf (stderr, "guess, t1: %6.4f, t2: %6.4f, t4: %6.4f, t5: %6.4f\n", t1, t2, t4, t5); + } + + // float flux = pcm->modelConv->modelFlux(pcm->modelConv->params); + // float normMag = -2.5*log10(flux); + // float Io = pow(10.0, -0.4*(totalMag - normMag)); + // PAR[PM_PAR_I0] = Io; + + return true; +} + +void psphotSersicModelNorm (pmPCMdata *pcm, const pmSource *source) { + + psVector *params = pcm->modelConv->params; + // XXX : not needed? psAssert (params->data.F32[PM_PAR_I0] == 1.0, "not normalized?"); + + // generate the convolved model image + // working vector to store local coordinate + psVector *coord = psVectorAlloc(2, PS_TYPE_F32); + + // create the convolved model in situ + psAssert (pcm->modelConvFlux, "not already allocated?"); + psImageInit (pcm->modelConvFlux, 0.0); + + // fill in the coordinate and value entries + for (psS32 i = 0; i < source->pixels->numRows; i++) { + for (psS32 j = 0; j < source->pixels->numCols; j++) { + + // Convert i/j to image space: + coord->data.F32[0] = (psF32) (j + source->pixels->col0); + coord->data.F32[1] = (psF32) (i + source->pixels->row0); + + pcm->modelConvFlux->data.F32[i][j] = pcm->modelConv->class->modelFunc (NULL, params, coord); + } + } + psFree(coord); + + psImageSmooth (pcm->modelConvFlux, pcm->sigma, pcm->nsigma); + + float YYmod = 0.0; + float Ymod2 = 0.0; + bool usePoisson = false; + + for (psS32 i = 0; i < source->pixels->numRows; i++) { + for (psS32 j = 0; j < source->pixels->numCols; j++) { + // XXX are we doing the right thing with the mask? + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[i][j]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[i][j] == 0) { + continue; + } + // skip nan value points + if (!isfinite(source->pixels->data.F32[i][j])) { + continue; + } + + float ymodel = pcm->modelConvFlux->data.F32[i][j]; + float yweight = (usePoisson) ? 1.0 / source->variance->data.F32[i][j] : 1.0; + + YYmod += ymodel * source->pixels->data.F32[i][j] * yweight; + Ymod2 += PS_SQR(ymodel) * yweight; + } + } + + float Io = YYmod / Ymod2; + + params->data.F32[PM_PAR_I0] *= Io; + + return; +} + +// Given a polynomial y = C0 + C1 x + C2 x^2 etc, solve for x given y +// actually: this only solves 1st and 2nd order polynomials +// for 2nd order, it always returns the (positive/negative) term + +psF64 psPolynomial1DSolve( + const psPolynomial1D* poly, + psF64 y, + bool upper) +{ + psF64 x, C; + switch (poly->nX) { + case 1: + // y = coeff[0] + coeff[1]*x + // x = (y - coeff[0]) / coeff[1] + x = (y - poly->coeff[0]) / poly->coeff[1]; + return x; + + case 2: + // y = coeff[0] + coeff[1]*x + coeff[2]*x^2 + // x = -coeff[1] +/- sqrt(coeff[1]^2 - 4*coeff[0]*coeff[2]) / (2 coeff[0]) + C = poly->coeff[0] - y; + if (upper) { + x = (-poly->coeff[1] + sqrt(PS_SQR(poly->coeff[1]) - 4*poly->coeff[2]*C)) / (2.0 * poly->coeff[2]); + } else { + x = (-poly->coeff[1] - sqrt(PS_SQR(poly->coeff[1]) - 4*poly->coeff[2]*C)) / (2.0 * poly->coeff[2]); + } + return x; + + default: + psAbort("invalid polynomial for 1D solver"); + } + psAbort("invalid polynomial for 1D solver"); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetMaskBits.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetMaskBits.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetMaskBits.c (revision 41083) @@ -0,0 +1,40 @@ +# include "psphotInternal.h" + +// XXX Should it be an error for any of these to not exist? + +// This function is called by the stand-alone psphot program to set the mask values in the +// config file. It sets the named mask values MASK.PSPHOT and MARK.PSPHOT in the PSPHOT +// recipe. Functions or programs which call psphotReadout as a library function must set these +// named mask values in the PSPHOT recipe on their own. + +bool psphotSetMaskBits (pmConfig *config) { + + psImageMaskType maskValue; + psImageMaskType markValue; + + if (!pmConfigMaskSetBits (&maskValue, &markValue, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + + bool status = psphotSetMaskRecipe (config, maskValue, markValue); + return status; +} + +bool psphotSetMaskRecipe (pmConfig *config, psImageMaskType maskValue, psImageMaskType markValue) { + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + + // set maskValue and markValue in the psphot recipe + psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MARK.PSPHOT", PS_META_REPLACE, "user-defined mask", markValue); + psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "user-defined mask", maskValue); + + return true; +} + +// XXX should these be in config->analysis or somewhere else besides 'recipe'? Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetNFrames.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetNFrames.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetNFrames.c (revision 41083) @@ -0,0 +1,75 @@ +# include "psphotInternal.h" + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + + +// Set source->nFrames based on the values in the EXPNUM image at the coordinates of each source's peak + +bool psphotSetNFrames (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Set N Frames ---"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the inputs + for (int i = 0; i < num; i++) { + if (!psphotSetNFramesReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to set nFrames for %s entry %d", filerule, i); + return false; + } + } + + return true; +} + +bool psphotSetNFramesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) +{ + bool status; + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + psImage *expnum = psMetadataLookupPtr(&status, readout->analysis, "EXPNUM"); + if (!status || !expnum) { + psLogMsg ("psphot", PS_LOG_INFO, "No EXPNUM image for input %d", index); + return true; + } + psLogMsg ("psphot", PS_LOG_INFO, "Found EXPNUM image for input %d", index); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + psLogMsg ("psphot", PS_LOG_INFO, "No detections image input %d", index); + return true; + } + + psArray *sources = detections->allSources; + if (!sources) { + psLogMsg ("psphot", PS_LOG_INFO, "No sources for input %d", index); + return true; + } + + int col0 = expnum->col0; + int row0 = expnum->row0; + int numCols = expnum->numCols; + int numRows = expnum->numRows; + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + int x = source->peak->x; + int y = source->peak->y; + if (x >= col0 && x < numCols && y >= row0 && y < numRows) { + source->nFrames = expnum->data.PS_TYPE_IMAGE_MASK_DATA[y][x]; + } else { + source->nFrames = 0; + } + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetThreads.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetThreads.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSetThreads.c (revision 41083) @@ -0,0 +1,100 @@ +# include "psphot.h" + +bool psphotSetThreads () { + + psThreadTask *task = NULL; + + pmPSFThreads (); + + task = psThreadTaskAlloc("PSPHOT_MODEL_BACKGROUND", 15); + task->function = &psphotModelBackground_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_GUESS_MODEL", 5); + task->function = &psphotGuessModel_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_ADD_NOISE", 6); + task->function = &psphotAddOrSubNoise_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_MAGNITUDES", 9); + task->function = &psphotMagnitudes_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_PSF_WEIGHTS", 3); + task->function = &psphotPSFWeights_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_APRESID_MAGS", 7); + task->function = &psphotApResidMags_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_SOURCE_STATS", 11); + task->function = &psphotSourceStats_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_KRON_ITERATE", 11); + task->function = &psphotKronIterate_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_KRON_FLUX", 4); + task->function = &psphotKronFlux_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_PETRO_FLUX", 4); + task->function = &psphotPetroFlux_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_GALAXY_SHAPES", 7); + task->function = &psphotGalaxyShape_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_BLEND_FIT", 10); + task->function = &psphotBlendFit_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_EXTENDED_FIT", 16); + task->function = &psphotExtendedSourceFits_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_EXTENDED_ANALYSIS", 8); + task->function = &psphotExtendedSourceAnalysis_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_RADIAL_APERTURES", 6); + task->function = &psphotRadialApertures_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_RADIAL_PROFILE_WINGS", 3); + task->function = &psphotRadialProfileWings_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_CHIP_PARAMS", 2); + task->function = &psphotChipParams_Threaded; + psThreadTaskAdd(task); + psFree(task); + + task = psThreadTaskAlloc("PSPHOT_GALAXY_PARAMS", 11); + task->function = &psphotGalaxyParams_Threaded; + psThreadTaskAdd(task); + psFree(task); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSignificanceImage.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSignificanceImage.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSignificanceImage.c (revision 41083) @@ -0,0 +1,155 @@ +# include "psphotInternal.h" + +// In this function, we smooth the image and variance, then generate the significance image : +// (S/N)^2. If FWMH_X,Y have been recorded, use them, otherwise use PEAKS_SMOOTH_SIGMA for the +// smoothing kernel. +pmReadout *psphotSignificanceImage (pmReadout *readout, psMetadata *recipe, psImageMaskType maskVal) { + + float SIGMA_SMTH, NSIGMA_SMTH; + bool status = false; + + // smooth the image and variance map + psTimerStart ("psphot.smooth"); + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading in psImageConvolve + + // XXX we can a) choose fft to convolve if needed and b) multithread fftw + + float minGauss = psMetadataLookupF32(NULL, recipe, "PEAKS_MIN_GAUSS"); // Minimum valid fraction of kernel + if (!isfinite(minGauss)) { + psWarning("PEAKS_MIN_GAUSS is not set in recipe; using default value"); + minGauss = 0.5; + } + + // NOTE: for a faint extended-source detection pass, we over-smooth by SOMETHING + + // if we have already determined the PSF model, then we have a better idea how to smooth this image + bool statusMajor, statusMinor; + float fwhmMajor = psMetadataLookupF32(&statusMajor, readout->analysis, "FWHM_MAJ"); + float fwhmMinor = psMetadataLookupF32(&statusMinor, readout->analysis, "FWHM_MIN"); + if (statusMajor && statusMinor) { + // if we know the FHWM, use that to set the smoothing kernel (XXX allow an optional override?) + if (!isfinite(fwhmMajor) || !isfinite(fwhmMinor) || fwhmMajor == 0.0 || fwhmMinor == 0.0) { + psWarning("fwhmMajor (%f) or fwhmMinor (%f) is bad!", fwhmMajor, fwhmMinor); + } + SIGMA_SMTH = 0.5*(fwhmMajor + fwhmMinor) / (2.0*sqrt(2.0*log(2.0))); + NSIGMA_SMTH = psMetadataLookupF32 (&status, recipe, "PEAKS_SMOOTH_NSIGMA"); + } else { + // if we do not know the FWHM, use the guess smoothing kernel supplied. + // it is a configuration error if these are not supplied + SIGMA_SMTH = psMetadataLookupF32 (&status, recipe, "PEAKS_SMOOTH_SIGMA"); + PS_ASSERT (status, NULL); + NSIGMA_SMTH = psMetadataLookupF32 (&status, recipe, "PEAKS_SMOOTH_NSIGMA"); + PS_ASSERT (status, NULL); + } + // record the actual smoothing sigma + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SIGMA_SMOOTH", PS_META_REPLACE, "Smoothing sigma for detections", SIGMA_SMTH); + + // smooth the image, applying the mask as we go + psImage *smooth_im = psImageCopy(NULL, readout->image, PS_TYPE_F32); + psImageSmoothMask_Threaded(smooth_im, smooth_im, readout->mask, maskVal, SIGMA_SMTH, NSIGMA_SMTH, minGauss); + psLogMsg("psphot", PS_LOG_MINUTIA, "smooth image: %f sec\n", psTimerMark("psphot.smooth")); + + // Smooth the variance, applying the mask as we go. The variance is smoothed by the PSF^2, + // renomalized to maintain the input level of the variance. We achieve this by smoothing + // with a Gaussian with sigma = SIGMA_SMTH/sqrt(2) with unity normalization. Note that + // this process yields a smoothed image with correlated errors. The pixel-to-pixel + // variations in smooth_im will be decreased by a factor of 4*pi*SIGMA_SMTH^2, but for + // measurements based on apertures comparable to or larger than the smoothing kernel, the + // effective per-pixel variance is maintained. + psImage *smooth_wt = psImageCopy(NULL, readout->variance, PS_TYPE_F32); + psImageSmoothMask_Threaded(smooth_wt, smooth_wt, readout->mask, maskVal, SIGMA_SMTH * M_SQRT1_2, NSIGMA_SMTH, minGauss); + psLogMsg("psphot", PS_LOG_MINUTIA, "smooth variance: %f sec\n", psTimerMark("psphot.smooth")); + + psImage *mask = readout->mask; + + // optionally save example images under trace + // XXX change these to recipe value checks + if (psTraceGetLevel("psphot") > 5) { + static int pass = 0; + char name[64]; + sprintf (name, "imsmooth.v%d.fits", pass); + psphotSaveImage(NULL, smooth_im, name); + sprintf (name, "wtsmooth.v%d.fits", pass); + psphotSaveImage(NULL, smooth_wt, name); + pass ++; + } + + // Calculate correction factor for the covariance produced by the (potentially multiple) smoothing + psKernel *kernel = psImageSmoothKernel(SIGMA_SMTH, NSIGMA_SMTH); // Kernel used for smoothing + double sum2 = 0.0; // Sum of kernel squared + for (int y = kernel->yMin; y <= kernel->yMax; y++) { + for (int x = kernel->xMin; x <= kernel->xMax; x++) { + sum2 += PS_SQR(kernel->kernel[y][x]); + } + } + float factor = 1.0 / (sum2 * psImageCovarianceCalculateFactor(kernel, readout->covariance)); + psFree(kernel); + + // record the effective area and significance scaling factor + float effArea = 8.0 * M_PI * PS_SQR(SIGMA_SMTH); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "EFFECTIVE_AREA", PS_META_REPLACE, "Effective Area", effArea); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SIGNIFICANCE_SCALE_FACTOR", PS_META_REPLACE, "Signicance scale factor", factor); + + // we are going to return both the image and the weight here: the image contains the signal + // while the 'weight' will contain the significance (NOTE the deviation from the usual + // definition) + + // save the smoothed significance image in the weight array + for (int j = 0; j < smooth_im->numRows; j++) { + for (int i = 0; i < smooth_im->numCols; i++) { + float value = smooth_im->data.F32[j][i]; + if (value < 0 || smooth_wt->data.F32[j][i] <= 0 || (mask->data.PS_TYPE_IMAGE_MASK_DATA[j][i] & maskVal)) { + smooth_wt->data.F32[j][i] = 0.0; + } else { + // XXX the value of 100 here (or 1000 before) must depend on the FWHM of the smoothing kernel, right?? + // float v2 = value + PS_SQR(value/100.0); + // CZW 2013-06-20: I don't think this hack was helping. + float v2 = value; + smooth_wt->data.F32[j][i] = factor * PS_SQR(v2) / smooth_wt->data.F32[j][i]; + } + } + } + psLogMsg ("psphot", PS_LOG_INFO, "built smoothed signficance image: %f sec\n", psTimerMark("psphot.smooth")); + + // optionally save example images under trace + // XXX change these to recipe value checks + if (psTraceGetLevel("psphot") > 5) { + char name[64]; + static int pass = 0; + sprintf (name, "snsmooth.v%d.fits", pass); + psphotSaveImage (NULL, smooth_wt, name); + pass ++; + } + psImageConvolveSetThreads(oldThreads); + + pmReadout *significanceRO = pmReadoutAlloc(NULL); + significanceRO->variance = smooth_wt; + significanceRO->image = smooth_im; + + return significanceRO; +} + +# if (0) +{ + // threadingdemo + // smooth the image, applying the mask as we go + psImage *smooth_im = psImageCopy(NULL, readout->image, PS_TYPE_F32); + psImageSmoothMask(smooth_im, smooth_im, readout->mask, maskVal, SIGMA_SMTH, NSIGMA_SMTH, minGauss); + + psLogMsg("psphot", PS_LOG_INFO, "smooth image: %f sec\n", psTimerMark("psphot.smooth")); + + psImageConvolveSetThreads(true); + + psTimerStart ("psphot.smooth"); + + // XXX a quick test of the threaded version of the function: + psImage *smooth_test = psImageCopy(NULL, readout->image, PS_TYPE_F32); + psImageSmoothMask_Threaded(smooth_test, smooth_test, readout->mask, maskVal, SIGMA_SMTH, NSIGMA_SMTH, minGauss); + + psLogMsg("psphot", PS_LOG_INFO, "smooth_threaded image: %f sec\n", psTimerMark("psphot.smooth")); + + psphotSaveImage (NULL, smooth_im, "smooth_im.fits"); + psphotSaveImage (NULL, smooth_test, "smooth_test.fits"); + exit (0); +} +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSkyReplace.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSkyReplace.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSkyReplace.c (revision 41083) @@ -0,0 +1,57 @@ +# include "psphotInternal.h" + +bool psphotSkyReplace (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = true; + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotSkyReplaceReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to replace sky for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// XXX make this an option? +// in order to successfully replace the sky, we must define a corresponding file... +bool psphotSkyReplaceReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + psTimerStart ("psphot.skyreplace"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // select background pixels, from output background file, or create + pmReadout *background = pmFPAfileThisReadout (config->files, view, psphotGetFilerule("PSPHOT.BACKGND")); + if (background == NULL) psAbort("background not defined"); + + // select the corresponding images + psF32 **image = readout->image->data.F32; + // psImageMaskType **mask = readout->mask->data.PS_TYPE_IMAGE_MASK_DATA; + psF32 **back = background->image->data.F32; + + // replace the background model + for (int j = 0; j < readout->image->numRows; j++) { + for (int i = 0; i < readout->image->numCols; i++) { + if (isfinite(image[j][i]) && isfinite(back[j][i])) { + image[j][i] += back[j][i]; + } + } + } + psLogMsg ("psphot.sky", PS_LOG_DETAIL, "replace background flux : %f sec\n", psTimerMark ("psphot.skyreplace")); + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSortBySN.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSortBySN.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSortBySN.c (revision 41083) @@ -0,0 +1,1 @@ + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceFits.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceFits.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceFits.c (revision 41083) @@ -0,0 +1,1414 @@ +# include "psphotInternal.h" + +// given a source with an existing modelPSF, attempt a full PSF fit, subtract if successful + +static int NfitBlend = 0; + +static int NfitPSF = 0; +static int NfitIterPSF = 0; + +static int NfitDBL = 0; +static int NfitIterDBL = 0; +static int NfitPixDBL = 0; + +static int NfitEXT = 0; +static int NfitIterEXT = 0; +static int NfitPixEXT = 0; + +static int NfitPCM = 0; +static int NfitIterPCM = 0; +static int NfitPixPCM = 0; + +bool psphotPCMfitCheckSize (pmPCMdata *pcm, pmSource *source, psImageMaskType maskVal, float psfSize); +bool psphotPCMfitRetry (pmPCMdata *pcm, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, float psfSize); + +bool psphotFitInit (int nThreads) { + psTimerStart ("psphot.fits"); + pmSourceFitSetInit (nThreads); + return true; +} + +bool psphotFitSummary () { + + psLogMsg ("psphot.pspsf", PS_LOG_INFO, "fitted %5d psf (%d iter), %5d blend: %5d ext (%d iter, %d pix), %5d dbl (%d iter, %d pix) : %6.2f sec\n", + NfitPSF, NfitIterPSF, NfitBlend, NfitEXT, NfitIterEXT, NfitPixEXT, NfitDBL, NfitIterDBL, NfitPixDBL, psTimerMark ("psphot.fits")); + return true; +} + +bool psphotFitSummaryExtended () { + + psLogMsg ("psphot.pspsf", PS_LOG_INFO, "fitted %5d ext (%d iter, %d pix), %5d pcm (%d iter, %d pix)\n", + NfitEXT, NfitIterEXT, NfitPixEXT, NfitPCM, NfitIterPCM, NfitPixPCM); + return true; +} + +bool psphotFitInitExtended () { + NfitEXT = 0; + NfitIterEXT = 0; + NfitPixEXT = 0; + NfitPCM = 0; + NfitIterPCM = 0; + NfitPixPCM = 0; + return true; +} + +bool psphotFitBlend (pmReadout *readout, pmSource *source, pmPSF *psf, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal) { + + float x, y, dR; + + pmSourceFitOptions options = *fitOptions; + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // if this source is not a possible blend, just fit as PSF + if ((source->blends == NULL) || (source->mode & PM_SOURCE_MODE_SATSTAR)) { + bool status = psphotFitPSF (readout, source, psf, fitOptions, maskVal, markVal); + return status; + } + + // save the PSF model from the Ensemble fit + pmModel *PSF = pmModelCopy (source->modelPSF); + + if (isnan(PSF->params->data.F32[PM_PAR_I0])) psAbort("nan in blend fit primary"); + + x = PSF->params->data.F32[PM_PAR_XPOS]; + y = PSF->params->data.F32[PM_PAR_YPOS]; + + psArray *modelSet = psArrayAllocEmpty (source->blends->n + 1); + psArrayAdd (modelSet, 16, PSF); + + psArray *sourceSet = psArrayAllocEmpty (source->blends->n + 1); + psArrayAdd (sourceSet, 16, source); + + psTrace ("psphot", 4, "fitting blended source at %f %f : %f\n", source->peak->xf, source->peak->yf, source->peak->rawFlux); + + // we need to include all blends in the fit (unless primary is saturated?) + dR = 0; + for (int i = 0; i < source->blends->n; i++) { + pmSource *blend = source->blends->data[i]; + + // find the blend which is furthest from source + dR = PS_MAX (dR, hypot (blend->peak->xf - x, blend->peak->yf - y)); + + // create the model and guess parameters for this blend + pmModel *model = pmModelAlloc (PSF->type); + for (int j = 0; j < model->params->n; j++) { + model->params->data.F32[j] = PSF->params->data.F32[j]; + model->dparams->data.F32[j] = PSF->dparams->data.F32[j]; + } + + // XXX assume local sky is 0.0? + model->params->data.F32[PM_PAR_I0] = blend->peak->rawFlux; + model->params->data.F32[PM_PAR_XPOS] = blend->peak->xf; + model->params->data.F32[PM_PAR_YPOS] = blend->peak->yf; + + // these should never be invalid values + // XXX drop these tests eventually + if (isnan(model->params->data.F32[PM_PAR_I0])) psAbort("nan in blend fit"); + if (isnan(model->params->data.F32[PM_PAR_XPOS])) psAbort("nan in blend fit"); + if (isnan(model->params->data.F32[PM_PAR_YPOS])) psAbort("nan in blend fit"); + + // add this blend to the list + psArrayAdd (modelSet, 16, model); + psArrayAdd (sourceSet, 16, blend); + + psTrace ("psphot", 5, "adding source at %f %f : %f\n", blend->peak->xf, blend->peak->yf, blend->peak->rawFlux); + + // free to avoid double counting model + psFree (model); + } + + // extend source radius as needed + psphotCheckRadiusPSFBlend (readout, source, PSF, markVal, dR); + + // fit PSF model + options.mode = PM_SOURCE_FIT_PSF; + pmSourceFitSet (source, modelSet, &options, maskVal); + + // clear the circular mask + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + if (!isfinite(PSF->params->data.F32[PM_PAR_I0])) psAbort("nan in fit"); + + // evaluate the blend objects, subtract if good, free otherwise + for (int i = 1; i < modelSet->n; i++) { + pmSource *blend = sourceSet->data[i]; + pmModel *model = modelSet->data[i]; + + if (!isfinite(model->params->data.F32[PM_PAR_I0])) psAbort("nan in fit"); + + // if this one failed, skip it + if (!psphotEvalPSF (blend, model)) { + psTrace ("psphot", 4, "failed on blend %d of %ld\n", i, modelSet->n); + continue; + } + + // otherwise, supply the resulting model to the corresponding blend + psFree(blend->modelPSF); + blend->modelPSF = psMemIncrRefCounter (model); + psTrace ("psphot", 5, "fitted blend as PSF\n"); + + // build cached model and subtract + pmSourceCacheModel (blend, maskVal); + pmSourceSub (blend, PM_MODEL_OP_FULL, maskVal); + blend->mode |= PM_SOURCE_MODE_BLEND_FIT; + } + NfitBlend += modelSet->n; + + // evaluate the primary object + if (!psphotEvalPSF (source, PSF)) { + psTrace ("psphot", 4, "failed on blend 0 of %ld\n", modelSet->n); + psFree (PSF); + psFree (modelSet); + psFree (sourceSet); + return false; + } + psFree (modelSet); + psFree (sourceSet); + + // save the new, successful model + psFree (source->modelPSF); + source->modelPSF = PSF; + psTrace ("psphot", 5, "fitted primary as PSF\n"); + + // build cached model and subtract + pmSourceCacheModel (source, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + source->mode |= PM_SOURCE_MODE_BLEND_FIT; + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + return true; +} + +bool psphotFitPSF (pmReadout *readout, pmSource *source, pmPSF *psf, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal) { + + pmSourceFitOptions options = *fitOptions; + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + NfitPSF ++; + + // save the PSF model from the Ensemble fit + pmModel *PSF = pmModelCopy (source->modelPSF); + if (isnan(PSF->params->data.F32[1])) psAbort("nan in psf fit"); + + // extend source radius as needed + psphotCheckRadiusPSF (readout, source, PSF, markVal); + + // fit PSF model (set/unset the pixel mask) + options.mode = PM_SOURCE_FIT_PSF; + pmSourceFitModel (source, PSF, &options, maskVal); + NfitIterPSF += PSF->nIter; + + if (!isfinite(PSF->params->data.F32[PM_PAR_I0])) psAbort("nan in fit"); + + // clear the circular mask + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // does the PSF model succeed? + if (!psphotEvalPSF (source, PSF)) { + psFree (PSF); + return false; + } + + // free old model, save new model + psFree (source->modelPSF); + source->modelPSF = PSF; + psTrace ("psphot", 5, "fitted as PSF\n"); + + // build cached model and subtract + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + pmSourceCacheModel (source, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + return true; +} + +// save a local, static copy of the EXT model type so we don't have to lookup for each object +static pmModelType modelTypeEXT; + +bool psphotInitLimitsEXT (psMetadata *recipe, pmReadout *readout) { + + bool status; + + // extended source model descriptions + char *modelNameEXT = psMetadataLookupStr (&status, recipe, "EXT_MODEL"); + modelTypeEXT = pmModelClassGetType (modelNameEXT); + + psphotInitRadiusEXT (recipe, readout); + + return true; +} + +bool psphotFitBlob (pmReadout *readout, pmSource *source, psArray *newSources, pmPSF *psf, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal) { + + float fitRadius, windowRadius; + bool okEXT, okDBL; + pmModel *ONE = NULL; + pmSource *tmpSrc = NULL; + pmModel *EXT = NULL; + psArray *DBL = NULL; + // pmMoments psfMoments; + + // skip the source if we don't think it is extended + // XXX are these robust, or do we have better info from the source-size analysis?? + if (source->type == PM_SOURCE_TYPE_UNKNOWN) return false; + if (source->type == PM_SOURCE_TYPE_DEFECT) return false; + if (source->type == PM_SOURCE_TYPE_SATURATED) return false; + + // set the radius based on the footprint (also sets the mask pixels) + if (!psphotSetRadiusMoments(&fitRadius, &windowRadius, readout, source, markVal)) return false; + // fprintf (stderr, "rad: %6.1f %6.1f | %5.2f %5.2f %5.2f ", source->peak->xf, source->peak->yf, source->moments->Mrf, fitRadius, windowRadius); + + psTrace ("psphot", 5, "trying blob...\n"); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + float chiEXT = NAN; + float chiDBL = NAN; + + // this temporary source is used as a place-holder by the psphotEval functions below + tmpSrc = pmSourceAlloc (); + { + // DBL will always be defined, but DBL->data[n] might not + DBL = psphotFitDBL (readout, source, fitOptions, maskVal, markVal); + if (!DBL) goto escape; + if (!DBL->n) goto escape; + + okDBL = psphotEvalDBL (tmpSrc, DBL->data[0]); + okDBL &= psphotEvalDBL (tmpSrc, DBL->data[1]); + okDBL = false; // XXX this is failing badly... + // XXX should I keep / save the flags set in the eval functions? + + // correct first model chisqs for flux trend + ONE = DBL->data[0]; + if (ONE) { + psAssert (isfinite(ONE->params->data.F32[PM_PAR_I0]), "nan in fit"); + chiDBL = ONE->chisqNorm; // save chisq for double-star/galaxy comparison + ONE->fitRadius = fitRadius; + } + + // correct second model chisqs for flux trend + ONE = DBL->data[1]; + if (ONE) { + psAssert (isfinite(ONE->params->data.F32[PM_PAR_I0]), "nan in fit"); + ONE->fitRadius = fitRadius; + } + } + + { + // XXX need to handle failures better here + EXT = psphotFitEXT (NULL, readout, source, fitOptions, modelTypeEXT, maskVal, markVal); + if (!EXT) goto escape; + if (!isfinite(EXT->params->data.F32[PM_PAR_I0])) goto escape; + + okEXT = psphotEvalEXT (tmpSrc, EXT); + chiEXT = EXT ? EXT->chisqNorm : NAN; + EXT->fitRadius = fitRadius; + } + + // clear the circular mask + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + + psFree (tmpSrc); + + // (void) psTraceSetLevel("psModules.objects.pmSourceFitSet", 0); + + if (okEXT && okDBL) { + // XXX EAM : a bogus bias: need to examine this better + if (3*chiEXT > chiDBL) goto keepDBL; + goto keepEXT; + } + + if (okEXT && !okDBL) goto keepEXT; + if (!okEXT && okDBL) goto keepDBL; + + psTrace ("psphot", 4, "both failed: blob chisq: %f vs %f for %f,%f\n", chiEXT, chiDBL, source->peak->xf, source->peak->yf); + + // both models failed; reject them both + // XXX -- change type flags to psf in this case, and make sure we subtract it? + // reset the psf moments + // XXX *source->moments = psfMoments; + + psFree (EXT); + psFree (DBL); + return false; + +keepEXT: + psTrace ("psphot", 4, "goto EXT : blob chisq: %f vs %f for %f,%f\n", chiEXT, chiDBL, source->peak->xf, source->peak->yf); + // sub EXT + psFree (DBL); + + // save new model + // XXX save the correct radius... + source->modelEXT = EXT; + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXTMODEL; + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + + // build cached model and subtract + pmSourceCacheModel (source, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + +# if (PS_TRACE_ON) + psTrace ("psphot", 5, "blob as EXT: %f %f\n", EXT->params->data.F32[PM_PAR_XPOS], EXT->params->data.F32[PM_PAR_YPOS]); + if (psTraceGetLevel("psphot") >= 6) { + psLogMsg ("psphot", 1, "source 2:\n"); + for (int i = 0; i < source->modelEXT->params->n; i++) { + psLogMsg ("psphot", 1, "PAR %d : %f +/- %f\n", i, source->modelEXT->params->data.F32[i], source->modelEXT->dparams->data.F32[i]); + } + } +# endif + + // reset the psf moments + // XXX *source->moments = psfMoments; + return true; + +keepDBL: + psTrace ("psphot", 4, "goto DBL : blob chisq: %f vs %f for %f,%f\n", chiEXT, chiDBL, source->peak->xf, source->peak->yf); + // sub DLB + psFree (EXT); + + // drop old model, save new second model... + psFree (source->modelPSF); + source->modelPSF = psMemIncrRefCounter (DBL->data[0]); + source->mode |= PM_SOURCE_MODE_PAIR; + source->mode |= PM_SOURCE_MODE_NONLINEAR_FIT; + + // copy most data from the primary source (modelEXT, blends stay NULL) + pmSource *newSrc = pmSourceCopy (source); + newSrc->modelPSF = psMemIncrRefCounter (DBL->data[1]); + newSrc->peak->footprint = source->peak->footprint; // just a reference; the peak does not own the footprint + psArrayAdd(newSrc->peak->footprint->peaks, 1, newSrc->peak); // the footprint owns the peak + + // build cached models and subtract + pmSourceCacheModel (source, maskVal); + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + pmSourceCacheModel (newSrc, maskVal); + pmSourceSub (newSrc, PM_MODEL_OP_FULL, maskVal); + +# if (PS_TRACE_ON) + psTrace ("psphot", 5, "blob as DBL: %f %f\n", ONE->params->data.F32[PM_PAR_XPOS], ONE->params->data.F32[PM_PAR_YPOS]); + if (psTraceGetLevel("psphot") >= 6) { + psLogMsg ("psphot", 1, "source 1:\n"); + for (int i = 0; i < newSrc->modelPSF->params->n; i++) { + psLogMsg ("psphot", 1, "PAR %d : %f +/- %f\n", i, newSrc->modelPSF->params->data.F32[i], newSrc->modelPSF->dparams->data.F32[i]); + } + psLogMsg ("psphot", 1, "source 2:\n"); + for (int i = 0; i < source->modelPSF->params->n; i++) { + psLogMsg ("psphot", 1, "PAR %d : %f +/- %f\n", i, source->modelPSF->params->data.F32[i], source->modelPSF->dparams->data.F32[i]); + } + } +# endif + + psArrayAdd (newSources, 100, newSrc); + psFree (newSrc); + psFree (DBL); + return true; + +escape: + psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(markVal)); + psFree (tmpSrc); + psFree (EXT); + psFree (DBL); + return false; +} + +// fit a double PSF source to an extended blob +psArray *psphotFitDBL (pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal) { + + float dx, dy; + pmModel *DBL; + pmModel *PSF; + psEllipseAxes axes; + psEllipseMoments moments; + psArray *modelSet; + + pmSourceFitOptions options = *fitOptions; + + NfitDBL ++; + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // XXX this is really poor: if we don't have moments for the source, we have no guess. + // force the measurement? + psAssert (source->moments, "moments are re-calculated for any extended source"); + + // make a guess at the position of the two sources + moments.x2 = source->moments->Mxx; + moments.xy = source->moments->Mxy; + moments.y2 = source->moments->Myy; + axes = psEllipseMomentsToAxes (moments, 20.0); + + if (isnan(axes.major)) return NULL; + if (isnan(axes.minor)) return NULL; + if (isnan(axes.theta)) return NULL; + + // XXX this is really arbitrary: 4 pixel separation? + dx = 2 * cos (axes.theta); + dy = 2 * sin (axes.theta); + + // save the PSF model from the Ensemble fit + PSF = source->modelPSF; + if (isnan(PSF->params->data.F32[1])) psAbort("nan in dbl fit"); + + modelSet = psArrayAlloc (2); + + DBL = pmModelCopy (PSF); + DBL->params->data.F32[PM_PAR_I0] *= 0.5; + DBL->params->data.F32[PM_PAR_XPOS] = source->peak->xf + dx; + DBL->params->data.F32[PM_PAR_YPOS] = source->peak->yf + dy; + modelSet->data[0] = DBL; + + DBL = pmModelCopy (PSF); + DBL->params->data.F32[PM_PAR_I0] *= 0.5; + DBL->params->data.F32[PM_PAR_XPOS] = source->peak->xf - dx; + DBL->params->data.F32[PM_PAR_YPOS] = source->peak->yf - dy; + modelSet->data[1] = DBL; + + // fit PSF model (set/unset the pixel mask) + options.mode = PM_SOURCE_FIT_PSF; + pmSourceFitSet (source, modelSet, &options, maskVal); + NfitIterDBL += DBL->nIter; + NfitPixDBL += DBL->nDOF; + + return (modelSet); +} + +pmModel *psphotFitEXT (pmModel *guessModel, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, pmModelType modelType, psImageMaskType maskVal, psImageMaskType markVal) { + + if ((source->moments->Mxx < 1e-3) || (source->moments->Myy < 1e-3)) { + psTrace ("psphot", 5, "problem source: moments: %f %f\n", source->moments->Mxx, source->moments->Myy); + } + + pmSourceFitOptions options = *fitOptions; + + NfitEXT ++; + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // use the source moments, etc to guess basic model parameters + pmModel *model = guessModel ? guessModel : pmSourceModelGuess (source, modelType, maskVal, markVal); + if (!model) { + psTrace ("psphot", 5, "failed to generate a model for source: moments: %f %f\n", source->moments->Mxx, source->moments->Myy); + return NULL; + } + +# if (PS_TRACE_ON) + if (psTraceGetLevel ("psphot") >= 6) { + + // Moments-based shapes parameters + psEllipseMoments moments; + moments.x2 = source->moments->Mxx; + moments.xy = source->moments->Mxy; + moments.y2 = source->moments->Myy; + // force the axis ratio to be < 20.0 + psEllipseAxes axes = psEllipseMomentsToAxes (moments, 20.0); + + // PSF shape parameters + psEllipseShape psfShape; + psfShape.sx = source->modelPSF->params->data.F32[PM_PAR_SXX] / M_SQRT2; + psfShape.sxy = source->modelPSF->params->data.F32[PM_PAR_SXY]; + psfShape.sy = source->modelPSF->params->data.F32[PM_PAR_SYY] / M_SQRT2; + psEllipseAxes psfAxes = psEllipseShapeToAxes (psfShape, 20.0); + + fprintf (stderr, "--- guess values ---\n"); + fprintf (stderr, "(x,y): %f, %f Mxx: %f, Myy: %f, Mxy: %f -> major: %f, minor: %f, theta: %f (%f deg)\n", source->peak->xf, source->peak->yf, source->moments->Mxx, source->moments->Myy, source->moments->Mxy, axes.major, axes.minor, axes.theta, axes.theta*PS_DEG_RAD); + fprintf (stderr, "psf: major: %f, minor: %f, theta: %f (%f deg)\n", psfAxes.major, psfAxes.minor, psfAxes.theta, psfAxes.theta*PS_DEG_RAD); + for (int i = 0; i < model->params->n; i++) { + fprintf (stderr, "par %d: %f\n", i, model->params->data.F32[i]); + } + } + if (psTraceGetLevel ("psphot") >= 7) { + psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5); + } +# endif + + // for sersic models, use a grid search to choose an index, then float the params there + if (modelType == pmModelClassGetType("PS_MODEL_SERSIC")) { + psphotFitSersicIndex (model, readout, source, fitOptions, maskVal, markVal); + } + + options.mode = PM_SOURCE_FIT_EXT; + if (modelType == pmModelClassGetType("PS_MODEL_SERSIC")) { + options.mode = PM_SOURCE_FIT_NO_INDEX; + } + if (modelType == pmModelClassGetType("PS_MODEL_TRAIL")) { + options.mode = PM_SOURCE_FIT_TRAIL; + } + + pmSourceFitModel (source, model, &options, maskVal); + NfitIterEXT += model->nIter; + NfitPixEXT += model->nDOF; + +# if (PS_TRACE_ON) + if (psTraceGetLevel ("psphot") >= 5) { + if (psTraceGetLevel ("psphot") >= 6) { + fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + } + fprintf (stderr, "--- fitted values ---\n"); + for (int i = 0; i < model->params->n; i++) { + fprintf (stderr, "par %d: %f\n", i, model->params->data.F32[i]); + } + psTraceSetLevel("psLib.math.psMinimizeLMChi2", 0); + } +# endif + + return (model); +} + +# define TIMING 0 +# define EXTRA_VERBOSE 0 + +bool psphotSersicModelGuessPCM (pmPCMdata *pcm, pmSource *source, psImageMaskType maskVal, float psfSize); +bool psphotFitSersicShapeAndIndex (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize); +bool psphotFitSersicShapeAndIndexGrid (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize); +bool psphotFitSersicShapeAndIndexGridAuto (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize); + +pmModel *psphotFitPCM (pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, pmModelType modelType, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + if ((source->moments->Mxx < 1e-3) || (source->moments->Myy < 1e-3)) { + psTrace ("psphot", 5, "problem source: moments: %f %f\n", source->moments->Mxx, source->moments->Myy); + } + + pmSourceFitOptions options = *fitOptions; + + NfitPCM ++; + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // allocate the model (this can only fail on a config error) + pmModel *model = pmModelAlloc(modelType); + psAssert (model, "invalid extended model name"); + + float t1, t2, t3, t4, t5; + t1 = t2 = t3 = t4 = t5 = 0.0; + if (TIMING) { psTimerStart ("psphotFitPCM"); } + + // if we are ever (in a given psphot implementation) going to fit a parameter, we must set the options here to include + // that parameter (otherwise pmPCMupdate will fail to allocate the dmodelFlux image) + // thus, if the sersic analysis below uses an index fit, need to use this EXT_AND_SKY mode for init + + options.mode = PM_SOURCE_FIT_EXT; + if (modelType == pmModelClassGetType("PS_MODEL_SERSIC")) { + options.mode = PM_SOURCE_FIT_EXT_AND_SKY; + } + if (modelType == pmModelClassGetType("PS_MODEL_DEV")) { + options.mode = PM_SOURCE_FIT_SHAPE; + options.mode = PM_SOURCE_FIT_EXT_AND_SKY; + } + if (modelType == pmModelClassGetType("PS_MODEL_EXP")) { + options.mode = PM_SOURCE_FIT_EXT_AND_SKY; + } + + pmPCMdata *pcm = pmPCMinit (source, &options, model, maskVal, psfSize); + if (!pcm) { + psTrace ("psphot", 5, "failed to generate a model for source: moments: %f %f\n", source->moments->Mxx, source->moments->Myy); + model->flags |= PM_MODEL_STATUS_BADARGS; // XXX this is probably already set in pmPCMinit + return model; + } + if (TIMING) { t1 = psTimerMark ("psphotFitPCM"); } + + // NOTE : 65 allocs to here + // get the guess for sersic models + if (modelType == pmModelClassGetType("PS_MODEL_SERSIC")) { + // use the source moments, etc to guess basic model parameters + if (!psphotSersicModelGuessPCM (pcm, source, maskVal, psfSize)) { + psFree (pcm); + model->flags |= PM_MODEL_SERSIC_PCM_FAIL_GUESS; + return model; + } + if (TIMING) { t2 = psTimerMark ("psphotFitPCM"); } + + // psphotFitSersicShapeAndIndex (pcm, readout, source, fitOptions, maskVal, markVal, psfSize); + options.mode = PM_SOURCE_FIT_NO_INDEX; + if (!psphotFitSersicShapeAndIndexGridAuto (pcm, readout, source, &options, maskVal, markVal, psfSize)) { + psFree (pcm); + model->flags |= PM_MODEL_SERSIC_PCM_FAIL_GRID; + psError(PS_ERR_UNKNOWN, true, "Failed to find a index & shape"); + psErrorClear (); // clear the polynomial error + return model; + } + } else { + // use the source moments, etc to guess basic model parameters + if (!pmSourceModelGuessPCM (pcm, source, maskVal, markVal)) { + psFree (pcm); + model->flags |= PM_MODEL_PCM_FAIL_GUESS; + return model; + } + } + + if (TIMING) { t3 = psTimerMark ("psphotFitPCM"); } + + // psTraceSetLevel("psLib.math.psMinimizeLMChi2", 5); + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); // NOTE : 1687 allocs in here + NfitIterPCM += pcm->modelConv->nIter; + NfitPixPCM += pcm->modelConv->nDOF; + if (TIMING) { t4 = psTimerMark ("psphotFitPCM"); } + + // XXX we might make this more efficient by setting NITER to be fairly small. if we hit the iteration + // limit, then we could do a small grid search on the size and try again from the best fit + + if (options.isInteractive) psphotPCMfitCheckSize (pcm, source, maskVal, psfSize); + // if (pcm->modelConv->nIter == fitOptions->nIter) { + // psphotPCMfitRetry (pcm, source, &options, maskVal, markVal, psfSize); + // } + if (TIMING) { t5 = psTimerMark ("psphotFitPCM"); } + + if (TIMING) { + int nPixBig = source->pixels->numCols * source->pixels->numRows; + fprintf (stderr, "psphotFitPCM : nIter: %2d, radius: %6.1f, npix: %5d of %5d, t1: %6.4f, t2: %6.4f, t3: %6.4f, t4: %6.4f, t5: %6.4f\n", model->nIter, model->fitRadius, model->nPix, nPixBig, t1, t2, t3, t4, t5); + } + if (EXTRA_VERBOSE && !TIMING) { + int nPixBig = source->pixels->numCols * source->pixels->numRows; + float *PAR = model->params->data.F32; + fprintf (stderr, "source %d : %f - %f %f - %f %f %f - %f | nIter: %2d, radius: %6.1f, npix: %5d of %5d, chisq %f\n", source->id, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1], model->nIter, model->fitRadius, model->nPix, nPixBig, model->chisqNorm); + } + + psFree (pcm); + + return model; +} + +// note that these should be 1/2n of the standard sersic index +// float indexGuess[] = {0.8, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0, 5.5, 6.0}; +// float indexGuess[] = {0.5, 0.33, 0.25, 0.167, 0.125, 0.083}; +float indexGuess[] = {1.0, 2.0, 3.0, 4.0}; +# define N_INDEX_GUESS 4 + +// A sersic model is very sensitive to the index. attempt to find the index first by grid search in just the index +// for a sersic model, attempt to fit just the index and normalization with a modest number of iterations +bool psphotFitSersicIndex (pmModel *model, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal) { + + assert (model->type == pmModelClassGetType("PS_MODEL_SERSIC")); + + pmSourceFitOptions options = *fitOptions; + + // fit EXT (not PSF) model (set/unset the pixel mask) + options.mode = PM_SOURCE_FIT_NO_INDEX; + options.nIter = 4; + + int iMin = -1; + float xMin = NAN; + float chiSquare[N_INDEX_GUESS]; + + for (int i = 0; i < N_INDEX_GUESS; i++) { + model->params->data.F32[PM_PAR_7] = 0.5/indexGuess[i]; + + if (!model->class->modelGuess(model, source, maskVal, markVal)) { + model->flags |= PM_MODEL_STATUS_BADARGS; + return false; + } + + pmSourceFitModel (source, model, &options, maskVal); + // fprintf (stderr, "index: %f, chisq: %f, nIter: %d, radius: %f, npix: %d\n", indexGuess[i], model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + + chiSquare[i] = model->chisqNorm; + if (i == 0) { + xMin = chiSquare[i]; + iMin = i; + } else { + if (chiSquare[i] < xMin) { + xMin = chiSquare[i]; + iMin = i; + } + } + } + assert (iMin >= 0); + + model->flags = PM_MODEL_STATUS_NONE; // do not attempt to handle failures here, let the next iteration deal with it + model->params->data.F32[PM_PAR_7] = 0.5/indexGuess[iMin]; + model->class->modelGuess(model, source, maskVal, markVal); + + return true; +} + +// A sersic model is very sensitive to the index. attempt to find the index first by grid search in just the index +// for a sersic model, attempt to fit just the index and normalization with a modest number of iterations +bool psphotFitSersicIndexPCM (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + pmModel *model = pcm->modelConv; + + assert (model->type == pmModelClassGetType("PS_MODEL_SERSIC")); + + pmSourceFitOptions options = *fitOptions; + + // fit EXT (not PSF) model (set/unset the pixel mask) + options.mode = PM_SOURCE_FIT_NO_INDEX; + options.nIter = 4; + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, &options, model); + + int iMin = -1; + float xMin = NAN; + float chiSquare[N_INDEX_GUESS]; + + for (int i = 0; i < N_INDEX_GUESS; i++) { + model->params->data.F32[PM_PAR_7] = indexGuess[i]; + + if (!model->class->modelGuess(model, source, maskVal, markVal)) { + model->flags |= PM_MODEL_STATUS_BADARGS; + return false; + } + +# if (0) + // this block is to test the relative speed of straight and PCM fits + pmSourceFitModel (source, model, &options, maskVal); +# else + pmSourceModelGuessPCM(pcm, source, maskVal, markVal); + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); +# endif + fprintf (stderr, "index: %f, chisq: %f, nIter: %d, radius: %f, npix: %d\n", indexGuess[i], model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + // fprintf (stderr, "chisq: %f, nIter: %d, radius: %f, npix: %d\n", model->chisqNorm, model->nIter, model->fitRadius, model->nPix); + + chiSquare[i] = model->chisq; + if (i == 0) { + xMin = chiSquare[i]; + iMin = i; + } else { + if (chiSquare[i] < xMin) { + xMin = chiSquare[i]; + iMin = i; + } + } + } + assert (iMin >= 0); + + model->flags = PM_MODEL_STATUS_NONE; // do not attempt to handle failures here, let the next iteration deal with it + model->params->data.F32[PM_PAR_7] = indexGuess[iMin]; + + pmSourceModelGuessPCM(pcm, source, maskVal, markVal); + + return true; +} + +// 0.5 / n for (1.0, 1.25, 1.66, 2.0, 3.33, 4.0) +// float indexGuessInv[] = {0.5, 0.4, 0.3, 0.25, 0.20, 0.15, 0.125}; + +// 0.5 / n for (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 5.0, 6.0) +float indexGuessInv[] = {1.00, 0.50, 0.333, 0.25, 0.166, 0.125, 0.10, 0.0833}; +float indexGuessR1q[] = {1.06, 1.19, 1.335, 1.48, 1.840, 2.290, 2.84, 3.5300}; +# define N_INDEX_GUESS_INV 8 + +// we are going to guess in fractions about the R1-based guess +float reffGuess[] = {0.8, 0.9, 1.0, 1.12, 1.25}; +# define N_REFF_GUESS 5 + +// A sersic model is very sensitive to the index. attempt to find the index first by grid search in just the index +// for a sersic model, attempt to fit just the index and normalization with a modest number of iterations +bool psphotSersicModelGuessPCM (pmPCMdata *pcm, pmSource *source, psImageMaskType maskVal, float psfSize) { + + // we get a reasonable guess from: + // * Reff = Kron R1 / Q(index) -- Q comes from Graham & Driver + // * Rmajor / Rminor & Theta from moments + // * Io from total Kron flux + + // the guesses are used to fill in PAR: + psF32 *PAR = pcm->modelConv->params->data.F32; + + // convert the moments to Major,Minor,Theta + psEllipseMoments moments; + + if (!isfinite(source->moments->Mrf)) return false; + if (!isfinite(source->moments->Mxx)) return false; + if (!isfinite(source->moments->Mxy)) return false; + if (!isfinite(source->moments->Myy)) return false; + + moments.x2 = source->moments->Mxx; + moments.y2 = source->moments->Myy; + moments.xy = source->moments->Mxy; + + // limit axis ratio < 20.0 + psEllipseAxes momentAxes = psEllipseMomentsToAxes (moments, 20.0); + + // set the model position + if (!pmModelSetPosition(&PAR[PM_PAR_XPOS], &PAR[PM_PAR_YPOS], source)) { + return false; + } + + // sky is zero (no longer fitted, but not yet deprecated) + PAR[PM_PAR_SKY] = 0.0; + + // for the index loop, use Io = 1.0, use fitted values to determine Io + PAR[PM_PAR_I0] = 1.0; + + float xMin = NAN; + float iMin = NAN; + float sMin = NAN; + float rMin = NAN; + + // loop over index and Reff, keeping the ARatio and Theta constant? + // loop over index guesses and find the best fit + for (int j = 0; j < N_REFF_GUESS; j++) { + for (int i = 0; i < N_INDEX_GUESS_INV; i++) { + PAR[PM_PAR_7] = indexGuessInv[i]; + + psEllipseAxes guessAxes; + guessAxes.major = reffGuess[j] * source->moments->Mrf / indexGuessR1q[i]; + guessAxes.minor = guessAxes.major * (momentAxes.minor / momentAxes.major); + guessAxes.theta = momentAxes.theta; + + if (!isfinite(guessAxes.major)) return false; + if (!isfinite(guessAxes.minor)) return false; + if (!isfinite(guessAxes.theta)) return false; + + // convert the major,minor,theta to shape parameters for an Reff-like model + pmModelAxesToParams (&PAR[PM_PAR_SXX], &PAR[PM_PAR_SXY], &PAR[PM_PAR_SYY], guessAxes, true); + + // generated the modelFlux + // XXX note that this does not add sky to model + pmPCMMakeModel (source, pcm->modelConv, pcm->nsigma, maskVal, psfSize); + + float YY = 0.0; + float YM = 0.0; + float MM = 0.0; + bool usePoisson = false; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[iy][ix] == 0) { + continue; + } + // skip nan value points + if (!isfinite(source->pixels->data.F32[iy][ix])) { + continue; + } + + float fy = source->pixels->data.F32[iy][ix]; + float fm = source->modelFlux->data.F32[iy][ix]; + float wt = (usePoisson) ? 1.0 / source->variance->data.F32[iy][ix] : 1.0; + + YY += PS_SQR(fy) * wt; + YM += fm * fy * wt; + MM += PS_SQR(fm) * wt; + } + } + + float Io = YM / MM; + float Chisq = YY - 2 * Io * YM + Io * Io * MM; + if (isnan(xMin) || (Chisq < xMin)) { + xMin = Chisq; + iMin = Io; + sMin = indexGuessInv[i]; + rMin = reffGuess[j] / indexGuessR1q[i]; + } + if (EXTRA_VERBOSE) { + fprintf (stderr, "%d | %f %f %f %f | %f %f %f %f", i, indexGuessInv[i], reffGuess[j], Io, Chisq, sMin, rMin, iMin, xMin); + fprintf (stderr, "\n"); + } + } + } + + { + psEllipseAxes guessAxes; + guessAxes.major = rMin * source->moments->Mrf; + guessAxes.minor = guessAxes.major * (momentAxes.minor / momentAxes.major); + guessAxes.theta = momentAxes.theta; + + if (!isfinite(guessAxes.major)) return false; + if (!isfinite(guessAxes.minor)) return false; + if (!isfinite(guessAxes.theta)) return false; + + // convert the major,minor,theta to shape parameters for an Reff-like model + pmModelAxesToParams (&PAR[PM_PAR_SXX], &PAR[PM_PAR_SXY], &PAR[PM_PAR_SYY], guessAxes, true); + } + + PAR[PM_PAR_I0] = iMin; + PAR[PM_PAR_7] = sMin; + + return true; +} + +// we have a set of guess parameters, do a small number of iterations fitting only SHAPE then only INDEX +bool psphotFitSersicShapeAndIndex (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + pmModel *model = pcm->modelConv; + + assert (model->type == pmModelClassGetType("PS_MODEL_SERSIC")); + + pmSourceFitOptions options = *fitOptions; + + for (int i = 0; i < 3; i++) { + // fit EXT (not PSF) model (set/unset the pixel mask) + options.mode = PM_SOURCE_FIT_SHAPE; + options.nIter = 2; + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, &options, model); + + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) { + float *PAR = model->params->data.F32; + fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + } + + // fit EXT (not PSF) model (set/unset the pixel mask) + options.mode = PM_SOURCE_FIT_INDEX; + // options.mode = PM_SOURCE_FIT_EXT_AND_SKY; + options.nIter = 30; + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, &options, model); + + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) { + float *PAR = model->params->data.F32; + fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + } + } + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, fitOptions, model); + + return true; +} + +// we have a set of guess parameters, do a small number of iterations fitting only SHAPE then only INDEX +bool psphotFitSersicShapeAndIndexGridAuto (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + pmModel *model = pcm->modelConv; + + assert (model->type == pmModelClassGetType("PS_MODEL_SERSIC")); + + pmSourceFitOptions options = *fitOptions; + + psF32 *PAR = pcm->modelConv->params->data.F32; + + options.mode = PM_SOURCE_FIT_SHAPE; + options.nIter = 7; + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, &options, model); + + // we have been provided a guess at the index (P[7]) from the list of indexGuessInv + + // find the matching indexGuessInv + int nStart = -1; + for (int i = 0; i < N_INDEX_GUESS_INV; i++) { + if (fabs(PAR[PM_PAR_7] - indexGuessInv[i]) < 0.01) { + nStart = i; + break; + } + } + if (nStart == -1) { + fprintf (stderr, "WARNING: could not find start guess %f\n", PAR[PM_PAR_7]); + return false; + } + + psVector *chi2 = psVectorAllocEmpty (16, PS_TYPE_F32); + psVector *Sidx = psVectorAllocEmpty (16, PS_TYPE_F32); + + float Sm = NAN, Sp = NAN, So = NAN; + if (nStart == 0) { + Sm = indexGuessInv[nStart]; + So = 0.5*(indexGuessInv[nStart + 1] + indexGuessInv[nStart]); + Sp = indexGuessInv[nStart + 1]; + } else if (nStart == N_INDEX_GUESS_INV - 1) { + Sp = indexGuessInv[nStart]; + So = 0.5*(indexGuessInv[nStart - 1] + indexGuessInv[nStart]); + Sm = indexGuessInv[nStart - 1]; + } else { + Sm = 0.5*(indexGuessInv[nStart - 1] + indexGuessInv[nStart]); + So = indexGuessInv[nStart]; + Sp = 0.5*(indexGuessInv[nStart + 1] + indexGuessInv[nStart]); + } + + PAR[PM_PAR_7] = Sm; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + + PAR[PM_PAR_7] = So; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + + PAR[PM_PAR_7] = Sp; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + + psPolynomial1D *poly = psPolynomial1DAlloc (PS_POLYNOMIAL_ORD, 2); + if (!psVectorFitPolynomial1D (poly, NULL, 0, chi2, NULL, Sidx)) { + psError(PS_ERR_UNKNOWN, true, "Failed to find a good chisq parabola"); + psFree (chi2); + psFree (Sidx); + psFree (poly); + return false; + } + + // where is the minimum of this polynomial fit? + float Smin = -0.5 * poly->coeff[1] / poly->coeff[2] / 100.0; + + // constrain Smin to be in a valid range: allow the fitted range to go a bit beyond the 3 trial points, but no further + float Smx = Sm - 0.25*(So - Sm); + float Spx = Sp + 0.25*(Sp - So); + Smin = PS_MAX(PS_MIN(Smin, Smx), Spx); + PAR[PM_PAR_7] = Smin; + + // XXX I could set the error on PAR_7 here if I knew how to roughly convert these chisq values to true chisq values + + // return to the original fitting mode (fitOptions) + pmPCMupdate(pcm, source, fitOptions, model); + + psFree (chi2); + psFree (Sidx); + psFree (poly); + + return true; +} + + +// we have a set of guess parameters, do a small number of iterations fitting only SHAPE then only INDEX +bool psphotFitSersicShapeAndIndexGridAutoScaled (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + pmModel *model = pcm->modelConv; + + assert (model->type == pmModelClassGetType("PS_MODEL_SERSIC")); + + pmSourceFitOptions options = *fitOptions; + + psF32 *PAR = pcm->modelConv->params->data.F32; + + options.mode = PM_SOURCE_FIT_SHAPE; + options.nIter = 5; + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, &options, model); + + float parStart[8]; + for (int i = 0; i < 8; i++) parStart[i] = PAR[i]; + + // we start with a guess at the index (P[7]) + + // get chisq for P[7], P[7]*1.1, P[7]*1.25 (or *0.75 depending on the result of 1.1) + + psVector *chi2 = psVectorAllocEmpty (16, PS_TYPE_F32); + psVector *Sidx = psVectorAllocEmpty (16, PS_TYPE_F32); + + PAR[PM_PAR_7] = parStart[7]; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + + float fI = 1.1; + PAR[PM_PAR_7] = parStart[7]*fI; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + + if (chi2->data.F32[1] < chi2->data.F32[0]) { + fI = 1.3; + } else { + fI = 1.0 / 1.3; + } + + PAR[PM_PAR_7] = parStart[7]*fI; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + + // can we fit the 3 pts with a parabola? + int nTry = 0; + psPolynomial1D *poly = psPolynomial1DAlloc (PS_POLYNOMIAL_ORD, 2); + while (!psVectorFitPolynomial1D (poly, NULL, 0, chi2, NULL, Sidx)) { + psErrorClear (); // clear the polynomial error + if (nTry > 4) { + psError(PS_ERR_UNKNOWN, true, "Failed to find a good chisq parabola"); + psFree (chi2); + psFree (Sidx); + psFree (poly); + return false; + } + fI = (fI < 1.0) ? fI / 1.3 : fI * 1.3; + PAR[PM_PAR_7] = parStart[7]*fI; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, 100*PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + nTry ++; + } + + // where is the minimum of this polynomial fit? + float Smin = -0.5 * poly->coeff[1] / poly->coeff[2] / 100.0; + + // constrain Smin to be in a valid range (1.0 - 0.1, corresponding to 0.5 (Gauss) to 5.0 (slightly peakier than Dev) + Smin = PS_MAX(PS_MIN(Smin, 1.0), 0.1); + PAR[PM_PAR_7] = Smin; + + // pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + // if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + + //// for (int i = 0; i < 8; i++) PAR[i] = parStart[i]; + //// + //// for (float fI = 0.0; fI < 0.15; fI += 0.01) { + //// PAR[PM_PAR_7] = parStart[7] - fI; + //// + //// // fit EXT (not PSF) model (set/unset the pixel mask) + //// + //// pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + //// if (TIMING) { + //// float *PAR = model->params->data.F32; + //// fprintf (stderr, "%d %f : %f - %f %f %f - %f\n", model->nIter, model->chisqNorm, PAR[7], PAR[4], PAR[5], PAR[6], PAR[1]); + //// } + //// } + + // return to the original fitting mode (fitOptions) + pmPCMupdate(pcm, source, fitOptions, model); + + psFree (chi2); + psFree (Sidx); + psFree (poly); + + return true; +} + + +// we have a set of guess parameters, do a small number of iterations fitting only SHAPE then only INDEX +bool psphotFitSersicShapeAndIndexGrid (pmPCMdata *pcm, pmReadout *readout, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, int psfSize) { + + pmModel *model = pcm->modelConv; + + assert (model->type == pmModelClassGetType("PS_MODEL_SERSIC")); + + pmSourceFitOptions options = *fitOptions; + + psF32 *PAR = pcm->modelConv->params->data.F32; + + options.mode = PM_SOURCE_FIT_SHAPE; + options.nIter = 10; + + // update the pcm elements if we have changed the circumstance (here, options.mode) + pmPCMupdate(pcm, source, &options, model); + + psVector *chi2 = psVectorAllocEmpty (16, PS_TYPE_F32); + psVector *Sidx = psVectorAllocEmpty (16, PS_TYPE_F32); + + float par7[] = {0.100, 0.125, 0.150, 0.175, 0.200, 0.225, 0.250}; + for (int i = 0; i < 7; i++) { + PAR[PM_PAR_7] = par7[i]; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + psVectorAppend (Sidx, PAR[PM_PAR_7]); + psVectorAppend (chi2, model->chisqNorm); + } + + psPolynomial1D *poly = psPolynomial1DAlloc (PS_POLYNOMIAL_ORD, 2); + if (!psVectorFitPolynomial1D (poly, NULL, 0, chi2, NULL, Sidx)) { + psError(PS_ERR_UNKNOWN, true, "Failed to find a good chisq parabola"); + psFree (chi2); + psFree (Sidx); + psFree (poly); + return false; + } + + // where is the minimum of this polynomial fit? + fprintf (stderr, "fit1d: %f + %f x + %f x^2\n", poly->coeff[0], poly->coeff[1], poly->coeff[2]); + float Smin = -0.5 * poly->coeff[1] / poly->coeff[2]; + + // constrain Smin to be in a valid range (1.0 - 0.1, corresponding to 0.5 (Gauss) to 5.0 (slightly peakier than Dev) + Smin = PS_MAX(PS_MIN(Smin, 1.0), 0.1); + PAR[PM_PAR_7] = Smin; + pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + if (EXTRA_VERBOSE) fprintf (stderr, "%d >>> %d %f : %f - %f %f - %f %f %f - %f\n", source->id, model->nIter, model->chisqNorm, PAR[7], PAR[2], PAR[3], PAR[4], PAR[5], PAR[6], PAR[1]); + + //// for (int i = 0; i < 8; i++) PAR[i] = parStart[i]; + //// + //// for (float fI = 0.0; fI < 0.15; fI += 0.01) { + //// PAR[PM_PAR_7] = parStart[7] - fI; + //// + //// // fit EXT (not PSF) model (set/unset the pixel mask) + //// + //// pmSourceFitPCM (pcm, source, &options, maskVal, markVal, psfSize); + //// if (TIMING) { + //// float *PAR = model->params->data.F32; + //// fprintf (stderr, "%d %f : %f - %f %f %f - %f\n", model->nIter, model->chisqNorm, PAR[7], PAR[4], PAR[5], PAR[6], PAR[1]); + //// } + //// } + + // return to the original fitting mode (fitOptions) + pmPCMupdate(pcm, source, fitOptions, model); + + psFree (chi2); + psFree (Sidx); + psFree (poly); + + return true; +} + +// # define N_REFF_CHECK 11 +// float drefCheck[] = {-0.02, -0.04, -0.06, 0.0, 0.85, 0.90, 0.95, 1.00, 1.05, 1.10, 1.15, 1.20, 1.25}; + +// we have an initial fit, check to see if the current size is besst +bool psphotPCMfitCheckSize (pmPCMdata *pcm, pmSource *source, psImageMaskType maskVal, float psfSize) { + + // PAR is already at my current best guess + psF32 *PAR = pcm->modelConv->params->data.F32; + + // store best guess as a shape + psEllipseAxes centerAxes; + pmModelParamsToAxes (¢erAxes, PAR[PM_PAR_SXX], PAR[PM_PAR_SXY], PAR[PM_PAR_SYY], true); + + float xMin = NAN; + float iMin = NAN; + float rMin = NAN; + + // loop over Reff, keeping the ARatio and Theta constant + for (int j = -4; j <= 4; j++) { + + float dref = j * 0.01; + + psEllipseAxes guessAxes; + guessAxes.major = centerAxes.major + dref; + guessAxes.minor = guessAxes.major * centerAxes.minor / centerAxes.major; + guessAxes.theta = centerAxes.theta; + + if (!isfinite(guessAxes.major)) return false; + if (!isfinite(guessAxes.minor)) return false; + if (!isfinite(guessAxes.theta)) return false; + + // convert the major,minor,theta to shape parameters for an Reff-like model + pmModelAxesToParams (&PAR[PM_PAR_SXX], &PAR[PM_PAR_SXY], &PAR[PM_PAR_SYY], guessAxes, true); + + // generated the modelFlux + // XXX note that this does not add sky to model + pmPCMMakeModel (source, pcm->modelConv, pcm->nsigma, maskVal, psfSize); + + float YY = 0.0; + float YM = 0.0; + float MM = 0.0; + bool usePoisson = false; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[iy][ix] == 0) { + continue; + } + // skip nan value points + if (!isfinite(source->pixels->data.F32[iy][ix])) { + continue; + } + + float fy = source->pixels->data.F32[iy][ix]; + float fm = source->modelFlux->data.F32[iy][ix]; + float wt = (usePoisson) ? 1.0 / source->variance->data.F32[iy][ix] : 1.0; + + YY += PS_SQR(fy) * wt; + YM += fm * fy * wt; + MM += PS_SQR(fm) * wt; + } + } + + float Io = YM / MM; + float Chisq = YY - 2 * Io * YM + Io * Io * MM; + if (isnan(xMin) || (Chisq < xMin)) { + xMin = Chisq; + iMin = Io; + rMin = dref; + } + // fprintf (stderr, "%d | %f %f %f | %f %f %f\n", j, dref, Io, Chisq, rMin, iMin, xMin); + } + + psEllipseAxes guessAxes; + guessAxes.major = centerAxes.major + rMin; + guessAxes.minor = guessAxes.major * centerAxes.minor / centerAxes.major; + guessAxes.theta = centerAxes.theta; + + if (!isfinite(guessAxes.major)) return false; + if (!isfinite(guessAxes.minor)) return false; + if (!isfinite(guessAxes.theta)) return false; + + // convert the major,minor,theta to shape parameters for an Reff-like model + pmModelAxesToParams (&PAR[PM_PAR_SXX], &PAR[PM_PAR_SXY], &PAR[PM_PAR_SYY], guessAxes, true); + PAR[PM_PAR_I0] = iMin; + + return true; +} + +// we have an initial fit, check to see if the current size is besst +bool psphotPCMfitRetry (pmPCMdata *pcm, pmSource *source, pmSourceFitOptions *fitOptions, psImageMaskType maskVal, psImageMaskType markVal, float psfSize) { + + // PAR is already at my current best guess + psF32 *PAR = pcm->modelConv->params->data.F32; + + // store best guess as a shape + psEllipseAxes centerAxes; + pmModelParamsToAxes (¢erAxes, PAR[PM_PAR_SXX], PAR[PM_PAR_SXY], PAR[PM_PAR_SYY], true); + + // retry with axes smaller by 1 pixel + psEllipseAxes guessAxes; + guessAxes.major = centerAxes.major - 0.08; + guessAxes.minor = guessAxes.major * centerAxes.minor / centerAxes.major; + guessAxes.theta = centerAxes.theta; + + if (!isfinite(guessAxes.major)) return false; + if (!isfinite(guessAxes.minor)) return false; + if (!isfinite(guessAxes.theta)) return false; + + // convert the major,minor,theta to shape parameters for an Reff-like model + pmModelAxesToParams (&PAR[PM_PAR_SXX], &PAR[PM_PAR_SXY], &PAR[PM_PAR_SYY], guessAxes, true); + + // generated the modelFlux + // XXX note that this does not add sky to model + pmPCMMakeModel (source, pcm->modelConv, pcm->nsigma, maskVal, psfSize); + + float YY = 0.0; + float YM = 0.0; + float MM = 0.0; + bool usePoisson = false; + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + // skip masked points + if (source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]) { + continue; + } + // skip zero-variance points + if (source->variance->data.F32[iy][ix] == 0) { + continue; + } + // skip nan value points + if (!isfinite(source->pixels->data.F32[iy][ix])) { + continue; + } + + float fy = source->pixels->data.F32[iy][ix]; + float fm = source->modelFlux->data.F32[iy][ix]; + float wt = (usePoisson) ? 1.0 / source->variance->data.F32[iy][ix] : 1.0; + + YY += PS_SQR(fy) * wt; + YM += fm * fy * wt; + MM += PS_SQR(fm) * wt; + } + } + + float Io = YM / MM; + PAR[PM_PAR_I0] = Io; + + pmSourceFitPCM (pcm, source, fitOptions, maskVal, markVal, psfSize); // NOTE : 1687 allocs in here + + return true; +} + + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceFreePixels.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceFreePixels.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceFreePixels.c (revision 41083) @@ -0,0 +1,39 @@ +# include "psphotInternal.h" + +bool psphotSourceFreePixels (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotSourceFreePixelsReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to free source pixels for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotSourceFreePixelsReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + bool status; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + pmSourceFreePixels (source); + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceMatch.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceMatch.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceMatch.c (revision 41083) @@ -0,0 +1,762 @@ +# include "psphotInternal.h" + +// Structure for storing the contents of the PSPHOT.STACK.MATCH.FILTERS list from recipe +#define MATCH_INFO_FILTER_STR_LEN 16 +typedef struct { + int inputNum; + int order; + bool matchAll; + long nSources; + float yRatioMax; + char filterID[MATCH_INFO_FILTER_STR_LEN]; +} psphotStackMatchInfo; + +static psphotStackMatchInfo * psphotStackGetMatchInfo (pmConfig *config, const pmFPAview *view, const char *filerule); + +// functions for sorting the match info array +static int compareMatchInfoByInputNum(const void *a, const void *b); +static int compareMatchInfoByOrder(const void *a, const void *b); + +bool psphotMatchSourcesAddMissing (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects, psphotStackMatchInfo *matchInfo); +bool psphotMatchSourcesSetIDs (psArray *objects); + +static psArray *psphotMatchFootprintCacheAlloc (int nImages); +static pmFootprint *psphotMatchLookupFootprint (psArray *cache, int footprintID, int imageID); +static pmFootprint *psphotMatchCopyFootprint (psArray *cache, pmFootprint *footprint, int imageID, psImage *image, psArray *footprints); + +psArray *psphotMatchSources (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + psArray *objects = psArrayAllocEmpty(100); + + int num = psphotFileruleCount(config, filerule); + + psphotStackMatchInfo *matchInfo = psphotStackGetMatchInfo(config, view, filerule); + + // loop over the available readouts matching sources found to objects. The inputs are processed + // in the order specified by the recipe + for (int j = 0; j < num; j++) { + int i = matchInfo[j].inputNum; + if (!psphotMatchSourcesReadout (objects, config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to merge sources for %s entry %d", filerule, i); + psFree (objects); + return NULL; + } + } + + // Now re-order the matchInfo array by input number so we can find each inputs entry easily + qsort(matchInfo, num, sizeof(psphotStackMatchInfo), compareMatchInfoByInputNum); + + // create sources for images where an object has been detected in the other images + psphotMatchSourcesAddMissing (config, view, filerule, objects, matchInfo); + + // choose a consistent position; set common sequence values + psphotMatchSourcesSetIDs (objects); + + psFree(matchInfo); + + return objects; +} + +bool psphotMatchSourcesReadout (psArray *objects, pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int RADIUS = psMetadataLookupF32 (&status, recipe, "PSPHOT.STACK.MATCH.RADIUS"); + psAssert (status, "programming error: must define PSPHOT.STACK.MATCH.RADIUS"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + psAssert (detections->allSources, "all sources not defined?"); + + psphotMatchSourcesToObjects(objects, detections->allSources, RADIUS); + + return true; +} + +# define NEXT1 { i++; continue; } +# define NEXT2 { j++; continue; } +bool psphotMatchSourcesToObjects (psArray *objects, psArray *sources, float RADIUS) { + + float dx, dy; + + float RADIUS2 = RADIUS*RADIUS; + + // sort the source list by X + sources = psArraySort (sources, pmSourceSortByX); + objects = psArraySort (objects, pmPhotObjSortByX); + + psVector *foundSrc = psVectorAlloc(sources->n, PS_TYPE_U8); + psVectorInit (foundSrc, 0); + + psVector *foundObj = psVectorAlloc(objects->n, PS_TYPE_U8); + psVectorInit (foundObj, 0); + + // match sources to existing objects + + psLogMsg ("psphot", PS_LOG_DETAIL, "attempt to match sources (%ld vs %ld)", sources->n, objects->n); + + int i, j; + for (i = j = 0; (i < sources->n) && (j < objects->n); ) { + + pmSource *src = sources->data[i]; + pmPhotObj *obj = objects->data[j]; + + if (!src) NEXT1; + if (!src->peak) NEXT1; + if (!isfinite(src->peak->xf)) NEXT1; + if (!isfinite(src->peak->yf)) NEXT1; + + if (!obj) NEXT2; + if (!isfinite(obj->x)) NEXT2; + if (!isfinite(obj->y)) NEXT2; + + dx = src->peak->xf - obj->x; + if (dx < -1.02*RADIUS) NEXT1; + if (dx > +1.02*RADIUS) NEXT2; + + /* this block will match a given detection to the closest object within range of that detection. + XXX note that this matches ALL detections within range of the single object to that same object + this is bad, but I cannot just go in linear order (ie, mark off each object as they are + used). I should make a list of all Nobj * Ndet pairs in range and choose the matches + based on their separations. UGH + */ + + // we are within match range, look for matches: + int Jmin = -1; + float Rmin = RADIUS2; + for (int J = j; (dx > -1.02*RADIUS) && (J < objects->n); J++) { + + // skip objects that are already assigned: + if (foundObj->data.U8[J]) continue; + obj = objects->data[J]; + + dx = src->peak->xf - obj->x; + dy = src->peak->yf - obj->y; + + float dr = dx*dx + dy*dy; + if (dr > RADIUS2) continue; + if (dr < Rmin) { + Rmin = dr; + Jmin = J; + } + } + + // no match, try next source + if (Jmin == -1) { + i++; + continue; + } + obj = objects->data[Jmin]; + foundObj->data.U8[Jmin] = 1; + + // add to object + pmPhotObjAddSource (obj, src); + foundSrc->data.U8[i] = 1; + i++; + } + + // create new objects for unmatched sources + for (i = 0; i < sources->n; i++) { + + if (foundSrc->data.U8[i]) continue; + + pmSource *src = sources->data[i]; + + pmPhotObj *obj = pmPhotObjAlloc(); + pmPhotObjAddSource(obj, src); + psArrayAdd (objects, 100, obj); + psFree (obj); + } + psLogMsg ("psphot", PS_LOG_DETAIL, "matched sources (%ld vs %ld)", sources->n, objects->n); + + psFree (foundSrc); + psFree (foundObj); + return true; +} + +bool psphotMatchSourcesAddMissing (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects, psphotStackMatchInfo *matchInfo) { + + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int minDetectionsForForced = psMetadataLookupS32 (&status, recipe, "PSPHOT.STACK.MIN.DETECT.FOR.FORCED"); + if (!status) { + minDetectionsForForced = 2; + } + + // determine properties (sky, moments) of initial sources + float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + psAssert (status, "missing SKY_OUTER_RADIUS in recipe?"); + + int nImages = psphotFileruleCount(config, filerule); + + // generate look-up arrays for detections and readouts + psArray *detArrays = psArrayAlloc(nImages); + psArray *readouts = psArrayAlloc(nImages); + + for (int i = 0; i < nImages; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + psAssert (detections->allSources, "all sources not defined?"); + + detArrays->data[i] = psMemIncrRefCounter(detections); + readouts->data[i] = psMemIncrRefCounter(readout); + } + + // We are having a problem with large number of false y band detections causing large numbers of + // matched detections to be created. + // The matchInfo recipe each entry contains a value for Y.RATIO.MAX. For each filter with a non-zero value + // we compare the ratio of the number of y detections to the number of detections in that filter to Y.RATIO.MAX. + // If the ratio in all filters with a non-zero is above its limit we consider the y band stack to + // be suspect and so disable the y band matchAll recipe (if set) + + int iy = -1; + // note now we are looping over matchInfo + for (int i = 0; i < nImages; i++) { + // Remember which input is y + if (!strcmp(matchInfo[i].filterID, "y")) { + iy = i; + } + pmDetections *detections = detArrays->data[matchInfo[i].inputNum]; + matchInfo[i].nSources = detections->allSources->n; + } + + if (iy >= 0) { + int nCut = 0; + int nTry = 0; + for (int i = 0; i < nImages; i++) { + if (i == iy) continue; + if (matchInfo[i].yRatioMax != 0.0 && matchInfo[i].nSources > 0) { + ++nTry; + float ratio = (float) matchInfo[iy].nSources / (float) matchInfo[i].nSources; + psLogMsg ("psphot", PS_LOG_DETAIL, "nSrc_y / nSrc_%s: %6.3f max: %6.3f", matchInfo[i].filterID, ratio, + matchInfo[i].yRatioMax); + if (ratio > matchInfo[i].yRatioMax) { + // This one is above it's threshold + ++nCut; + } + } + } + if (nTry > 0 && nCut == nTry) { + // All filters with a cut were above the threshold. Disable matchAll for y band + psLogMsg ("psphot", PS_LOG_INFO, "Clearing y band matchAll because y ratio was too high in %d filters", nTry); + matchInfo[iy].matchAll = false; + } + } + + // vector to track if source for an image is found + psVector *found = psVectorAlloc(nImages, PS_TYPE_U8); + psArray *footprintCache = psphotMatchFootprintCacheAlloc(nImages); + + for (int i = 0; i < objects->n; i++) { + pmPhotObj *obj = objects->data[i]; + + // we will find the input source with the max number of spans and reproduce that footprint + int nSpansMax = 0; + int iSpansMax = -1; + + bool matchAll = false; + + // mark the images for which sources have been found + psVectorInit (found, 0); + for (int j = 0; j < obj->sources->n; j++) { + + pmSource *src = obj->sources->data[j]; + int index = src->imageID; + psAssert (index >= 0, "invalid index"); + psAssert (index < found->n, "invalid index"); + + if (src->peak && src->peak->footprint && src->peak->footprint->nspans > nSpansMax) { + nSpansMax = src->peak->footprint->nspans; + iSpansMax = j; + } + // If this detection was on an input that has the matchAll flag set we create matched sources + // irespective of the number of inputs in which the object was found. + if (matchInfo[index].matchAll) { + matchAll = true; + } + + found->data.U8[index] = 1; + } + + // skip adding matched sources for this object if the number of detections for less than + // the supplied mininum unless one of the detections was in a band for which the matchAll flag is set + if (!matchAll && obj->sources->n < minDetectionsForForced) { + continue; + } + + // we make a copy of the largest footprint; this will be used for all new sources associated with this object + pmFootprint *largestFootprint = NULL; + if (iSpansMax != -1) { // copy the footprint info + pmSource *src = obj->sources->data[iSpansMax]; + psAssert(src->peak, "source does not exist?"); + psAssert(src->peak->footprint, "footprint does not exist"); + psAssert(src->peak->footprint->nspans == nSpansMax, "wrong footprint?"); + + largestFootprint = src->peak->footprint; + } + + // generate new sources for the image that are missing + for (int index = 0; index < found->n; index++) { + if (found->data.U8[index]) continue; + + pmDetections *detections = detArrays->data[index]; + pmReadout *readout = readouts->data[index]; + int row0 = readout->image->row0; + int col0 = readout->image->col0; + + // The peak type is not used in psphot. PM_PEAK_LONE may be wrong, but irrelevant + float peakFlux = readout->image->data.F32[(int)(obj->y-row0-0.5)][(int)(obj->x-col0-0.5)]; + pmPeak *peak = pmPeakAlloc(obj->x, obj->y, peakFlux, PM_PEAK_LONE); + peak->xf = obj->x; + peak->yf = obj->y; + peak->dx = NAN; + peak->dy = NAN; + + // assign to a footprint on this readout->image + if (largestFootprint) { + // we save the copies that we make of the of the footprints in a hash so that we can reuse them + // for all sources that share the fooprint. Without this we had serious memory explosion in + // dense fields with lots of footprints (spans are small but when you have enough of them ...) + peak->footprint = psphotMatchLookupFootprint(footprintCache, largestFootprint->id, index); + if (!peak->footprint) { + // the peak does not claim ownership of the footprint (it does not free it). + // psphotMatchCopyFootprint saves a copy of this + // footprint on detections->footprints so we can free it later + peak->footprint = psphotMatchCopyFootprint(footprintCache, largestFootprint, + index, readout->image, detections->footprints); + } + } + + // create a new source + pmSource *source = pmSourceAlloc(); + source->imageID = index; + source->mode2 |= PM_SOURCE_MODE2_MATCHED; // source is generated based on another image + source->type = PM_SOURCE_TYPE_STAR; // until we know more, assume a PSF fit + + // add the peak + source->peak = peak; + + // allocate space for moments + source->moments = pmMomentsAlloc(); + + // allocate image, weight, mask arrays for each peak (square of radius OUTER) + pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); + +#if (0) + fprintf(stderr, "Add mising source for obj: %5d %5d image: %d flux: %f size: %4d %4d\n", + i, obj->id, index, peakFlux, source->pixels->numRows, source->pixels->numCols); +#endif + + peak->assigned = true; + pmPhotObjAddSource(obj, source); + psArrayAdd (detections->allSources, 100, source); + psFree (source); + } + } + psFree(footprintCache); + + // how many sources do we have now? + int nSources = 0; + for (int i = 0; i < objects->n; i++) { + pmPhotObj *obj = objects->data[i]; + nSources += obj->sources->n; + if (minDetectionsForForced <= 1) { + psAssert (obj->sources->n == nImages, "failed to match sources?"); + } + } + psLogMsg ("psphot", PS_LOG_DETAIL, "total of %d sources for %d images", nSources, nImages); + + + psFree (found); + psFree (detArrays); + psFree (readouts); + return true; +} + +bool psphotMatchSourcesSetIDs (psArray *objects) { + + for (int i = 0; i < objects->n; i++) { + pmPhotObj *obj = objects->data[i]; + + // set the source->seq values + for (int j = 0; j < obj->sources->n; j++) { + pmSource *src = obj->sources->data[j]; + src->seq = i; + } + } + return true; +} + +// Cache of footprints created for unmatched sources. +static psArray * psphotMatchFootprintCacheAlloc (int nImages) { + psArray *cache = psArrayAlloc(nImages); + for (int i = 0; i < nImages; i++) { + psHash *hash = psHashAlloc(5000); + cache->data[i] = hash; + } + return cache; +} + +// Find copy of footprint with given ID made for a given image +static pmFootprint *psphotMatchLookupFootprint (psArray *cache, int footprintID, int imageID) { + psHash *hash = (psHash *) cache->data[imageID]; + + psAssert(hash != NULL, "missing hash for image %d", imageID); + + // footprintID is the id of the original footprint. + char key[32]; + sprintf(key, "%d", footprintID); + + // The footprints in our hashes are the copies of that footprint for the respective images + pmFootprint *copy = psHashLookup(hash, key); + + return copy; +} + +// Create a copy of a given footprint for a given image +static pmFootprint *psphotMatchCopyFootprint (psArray *cache, pmFootprint *footprint, int imageID, psImage *image, psArray *footprints) { + + psAssert((imageID >= 0 && imageID < cache->n), "invalid imageID %d", imageID); + + psHash *hash = (psHash *) cache->data[imageID]; + + psAssert(hash != NULL, "missing hash for image %d", imageID); + + char key[32]; + sprintf(key, "%d", footprint->id); + + pmFootprint *copy = pmFootprintCopyData (footprint, image); + + // save a copy in this image's hash + psHashAdd(hash, key, copy); + + // save a copy of this footprint on the passed in footprints Array so we can free it later + psArrayAdd(footprints, 100, copy); + + // drop our ref + psFree(copy); + + return copy; +} + +// qsort function to sort match info by order +static int compareMatchInfoByOrder(const void *a, const void *b) { + psphotStackMatchInfo *infoA = (psphotStackMatchInfo *) a; + psphotStackMatchInfo *infoB = (psphotStackMatchInfo *) b; + + int result; + if (infoA->order < infoB->order) { + result = -1; + } else if (infoA->order == infoB->order) { + result = 0; + } else { + result = 1; + } + return result; +} + +// qsort function to sort match info by input number +static int compareMatchInfoByInputNum(const void *a, const void *b) { + psphotStackMatchInfo *infoA = (psphotStackMatchInfo *) a; + psphotStackMatchInfo *infoB = (psphotStackMatchInfo *) b; + + int result; + if (infoA->inputNum < infoB->inputNum) { + result = -1; + } else if (infoA->inputNum == infoB->inputNum) { + result = 0; + } else { + result = 1; + } + return result; +} + +// psphotStackGetMatchInfo +// process the recipe PSPHOT.STACK.MATCH.FILTERS which controls the order that the inputs +// are processed for source matching and the rules for creating matched sources for sources that +// are only detected in a single band. +static psphotStackMatchInfo *psphotStackGetMatchInfo (pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status = false; + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + + psMetadata *filterInfo = psMetadataLookupPtr (&status, recipe, "PSPHOT.STACK.MATCH.FILTERS"); + if (!status || !filterInfo) { + psLogMsg ("psphot", PS_LOG_WARN, "PSPHOT.MATCH.STACK.FILTERS not found in the recipe. Will process inputs in order supplied."); + filterInfo = NULL; + } + + int num = psphotFileruleCount(config, filerule); + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // Find the filter for each of the inputs in fpa concepts + psArray *inputFilters = psArrayAlloc(num); + for (int i = 0 ; i < num; i++) { + if (i != chisqNum) { + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); + psAssert (file, "missing file?"); + + psString filterid = psMetadataLookupStr(&status, file->fpa->concepts, "FPA.FILTERID"); + psAssert (filterid, "missing FPA.FILTERID?"); + psArraySet(inputFilters, i, filterid); + } else { + // The chisq image inherits its filterid from the first input so we shouldn't use that. + psString tmp = psStringCopy("chisq"); + psArraySet(inputFilters, i, tmp); + psFree(tmp); + } + } + // Allocate the match info array + psphotStackMatchInfo *matchInfo = psAlloc(num *sizeof(psphotStackMatchInfo)); + memset(matchInfo, 0, num*sizeof(psphotStackMatchInfo)); + int highest_order = -1; + if (filterInfo) { + // PSPHOT.STACK.MATCH.FILTERS is a metadata which contains a list of metadata objects each + // entry pertaining to a filter. + // The objects contained in each filter's metadata are strings. + + // Loop over the entries in the recipe and find the entry for each our our input readouts. + // XXX: Will this work if more than one input with a given filter is supplied? + // I think so. + psMetadataIterator *iter = psMetadataIteratorAlloc(filterInfo, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + if (item->type != PS_DATA_METADATA) { + psAbort ("Invalid type for PSPHOT.STACK.MATCH.FILTER: %s, not a metadata folder", item->name); + } + psString thisFilter = psMetadataLookupStr (&status, item->data.md, "FILTER.ID"); + psAssert(thisFilter, "missing FILTER.ID"); + + psString orderStr = psMetadataLookupStr (&status, item->data.md, "ORDER"); + psAssert(orderStr, "missing ORDER"); + psS32 order = atoi(orderStr); + + psString matchAllStr = psMetadataLookupStr (&status, item->data.md, "MATCH.ALL"); + psAssert(matchAllStr, "missing MATCH.ALL"); + bool matchAll = (strcasecmp("true", matchAllStr) == 0); + + psString yRatioStr = psMetadataLookupStr (&status, item->data.md, "Y.RATIO.MAX"); + psF32 yRatioMax = atof(yRatioStr); + + // look for this filter in the inputs + for (int i = 0; i < num; i++) { + if (!strcmp((char *)inputFilters->data[i], thisFilter)) { + // We have an input for this one + matchInfo[i].inputNum = i; + strncpy(matchInfo[i].filterID, thisFilter, MATCH_INFO_FILTER_STR_LEN - 1 ); + matchInfo[i].order = order; + matchInfo[i].matchAll = matchAll; + matchInfo[i].yRatioMax = yRatioMax; + matchInfo[i].nSources = 0; + psLogMsg ("psphot", PS_LOG_DETAIL, "input: %d %s match order: %d match all: %d\n", + i, thisFilter, order, matchAll); + if (order > highest_order) { + highest_order = order; + } + break; + } + } + } + psFree(iter); + } + + // Make sure we have a matchInfo for all of the inputs. Fill in an entry for any that were not found. + for (int i = 0; i < num; i++) { + if (!*matchInfo[i].filterID) { + matchInfo[i].inputNum = i; + strncpy(matchInfo[i].filterID, inputFilters->data[i], MATCH_INFO_FILTER_STR_LEN - 1); + matchInfo[i].order = ++highest_order; + matchInfo[i].matchAll = false; + psLogMsg ("psphot", PS_LOG_WARN, "Entry in PSPHOT.MATCH.STACK.FILTERS not found for input: %d filter: %s using order %d", + i, (char *) inputFilters->data[i], highest_order); + } + } + psFree(inputFilters); + + if (filterInfo) { + // Sort the array by ORDER. + qsort(matchInfo, num, sizeof(psphotStackMatchInfo), compareMatchInfoByOrder); + } else { + // no need to sort we just built the list in the order that the inputs were supplied + } + + return matchInfo; +} + +bool psphotFilterMatchedSources (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects) { + + bool status = false; + + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Filter Matched Sources ---"); + + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + + // select the appropriate recipe information + bool keepBadMatches = psMetadataLookupBool (&status, recipe, "PSPHOT.STACK.KEEP.BAD.MATCHES"); + if (!status) { + keepBadMatches = true; + } + + if (keepBadMatches) { + psLogMsg ("psphot", PS_LOG_INFO, "keeping bad matches"); + return true; + } + + int numImages = psphotFileruleCount(config, filerule); + psVector *dropped = psVectorAlloc(numImages, PS_TYPE_U32); + psVectorInit(dropped, 0); + + int nDropped = 0; + for (int i = 0; i < objects->n; i++) { + pmPhotObj *obj = objects->data[i]; + + // traverse the array from the end so that sources don't move until after we've processed them + for (int j = obj->sources->n - 1; j >= 0; j--) { + + pmSource *source = obj->sources->data[j]; + // This applies only to matched sources + if (!(source->mode2 & PM_SOURCE_MODE2_MATCHED)) continue; + + if (isfinite(source->apFlux)) continue; + + psTrace ("psphot", 7, "Dropping matched source from image %d at (%d, %d) no valid flux", + source->imageID, source->peak->x, source->peak->y); + + psAssert(source->imageID >= 0 && source->imageID < numImages, "bad imageID %d", source->imageID); + + dropped->data.U32[source->imageID]++; + + nDropped++; + + psArrayRemoveIndex(obj->sources, j); + } + } + + psLogMsg ("psphot", PS_LOG_DETAIL, "Dropped %d matched sources with no valid flux", nDropped); + psLogMsg ("psphot", PS_LOG_DETAIL, " Input Num Dropped"); + for (int i=0; idata.U32[i]); + } + psFree(dropped); + + // find the "best" Mrf from the detected sources. + // Currently we use the smallest positive value + for (int i=0; i< objects->n; i++) { + pmPhotObj *obj = objects->data[i]; + + float minMrf = 1000.; + bool hasMatched = false; + for (int j = 0; j < obj->sources->n; j++) { + pmSource *source = obj->sources->data[j]; + if (source->mode2 & PM_SOURCE_MODE2_MATCHED) { + hasMatched = true; + continue; + } + float Mrf = source->moments->Mrf; + if (isfinite(Mrf) && Mrf < minMrf && Mrf > 0) { + minMrf = Mrf; + } + } + + if (!hasMatched || minMrf > 120.) { + continue; + } + + // set Mrf for matched sources to the value found above + for (int j = 0; j < obj->sources->n; j++) { + pmSource *source = obj->sources->data[j]; + if (source->mode2 & PM_SOURCE_MODE2_MATCHED) { + source->moments->Mrf = minMrf; + } + } + } + + return true; +} + + +psArray *psphotLinkSources (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + + int num = psphotFileruleCount(config, filerule); + + psArray *sourcesArrays = psArrayAlloc(num); + + // loop over inputs find the maximum sequence number and save pointers to the arrays + int seqMax = -1; + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (NULL, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + psAssert (detections->allSources, "all sources not defined?"); + + // allSources value won't change but should be careful + detections->allSources = psArraySort(detections->allSources, pmSourceSortBySeq); + + sourcesArrays->data[i] = psMemIncrRefCounter(detections->allSources); + pmSource *lastSrc = detections->allSources->data[detections->allSources->n -1 ]; + int thisSeq = lastSrc->seq; + if (thisSeq > seqMax) { + seqMax = thisSeq; + } + } + + if (seqMax < 0) { + psError (PSPHOT_ERR_UNKNOWN, true, "failed to find maximum sequence number\n"); + return NULL; + } + + // allocate objects array + psArray *objects = psArrayAlloc(seqMax + 1); + // loop over inputs and create objects that reference all of the sources + for (int i = 0; i < num; i++) { + // sources for this input + psArray *sources = sourcesArrays->data[i]; + for (int j = 0; j < sources->n ; j++) { + pmSource *src = sources->data[j]; + // XXX This is no longer needed I think. I added it to work around something else. Check + src->imageID = i; + int seq = src->seq; + pmPhotObj *obj = objects->data[seq]; + if (!obj) { + // first source for this object + obj = objects->data[seq] = pmPhotObjAlloc(); + } + pmPhotObjAddSource(obj, src); + } + } + psFree(sourcesArrays); + + return objects; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceMemory.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceMemory.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceMemory.c (revision 41083) @@ -0,0 +1,54 @@ +# include "psphotInternal.h" + + +bool psphotSourceMemory (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotSourceMemoryReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on memory usage measurement for %s entry %d", filerule, i); + return false; + } + } + return true; +} +bool psphotSourceMemoryReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index) +{ + bool status; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + return true; + } + + psArray *sources = detections->allSources; + if (!sources) { + return true; + } + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping memory measurement"); + return true; + } + + psU64 bytes = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + bytes += pmSourceMemoryUse(source); + } + + psLogMsg ("psphot", PS_LOG_INFO, "input %s %d: %ld sources %.1f MB. %.1f bytes per source.\n", + filerule, index, sources->n, (psF32)bytes/1024/1024, (psF32) bytes / sources->n); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourcePlots.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourcePlots.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourcePlots.c (revision 41083) @@ -0,0 +1,167 @@ +# include "psphotInternal.h" + +bool psphotSourcePlots (pmReadout *readout, psArray *sources, psMetadata *recipe) { + + bool status = false; + psTimerStart ("psphot"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // the source images are written to an image 10x the size of a PSF object + // float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + // PS_ASSERT (status, false); + + int DX = 21; + int DY = 21; + + // examine PSF sources in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // counters to track the size of the image and area used in a row + int dX = 0; // starting corner of next box + int dY = 0; // height of row so far + int NX = 20*DX; // full width of output image + int NY = DY; // total height of output image so far + + // first, examine the PSF and SAT stars to set output image size: + // - add stamp widths until we exceed output image width, + // - then start a new row offset by max height + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + bool keep = false; + keep |= (source->mode & PM_SOURCE_MODE_PSFSTAR); + keep |= (source->mode & PM_SOURCE_MODE_SATSTAR); + if (!keep) { + psTrace ("psphot", 4, "not plotting star %d: %x", i, source->mode); + continue; + } + + // how does this subimage get placed into the output image? + // DX = source->pixels->numCols + // DY = source->pixels->numRows + + if (dX + DX > NX) { + // too wide for the rest of this row + if (dX == 0) { + // alone on this row + NY += DY; + dX = 0; + dY = 0; + } else { + // start the next row + NY += dY; + dX = DX; + dY = DY; + } + } else { + // extend this row + dX += DX; + dY = PS_MAX (dY, DY); + } + } + + if (NY == 0) { + psWarning ("no PSF or SAT stars to plot? skipping.\n"); + return false; + } + + // allocate output image + psImage *outpos = psImageAlloc (NX, NY, PS_TYPE_F32); + psImage *outsub = psImageAlloc (NX, NY, PS_TYPE_F32); + + psImageInit (outpos, 0.0); + psImageInit (outsub, 0.0); + + int Xo = 0; // starting corner of next box + int Yo = 0; // starting corner of next box + dY = 0; // height of row so far + + int nPSF = 0; + int nSAT = 0; + int kapa = 0; // file descriptor for plotting routine + + // first, examine the PSF and SAT stars: + // - generate radial plots (PS plots) + // - create output image array + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + bool keep = false; + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + nPSF ++; + keep = true; + } + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + nSAT ++; + keep = true; + } + if (!keep) continue; + + // how does this subimage get placed into the output image? + // DX = source->pixels->numCols + // DY = source->pixels->numRows + + if (Xo + DX > NX) { + // too wide for the rest of this row + if (Xo == 0) { + // place source alone on this row + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotRadialPlot (&kapa, "radial.plots.ps", source); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + Yo += DY; + Xo = 0; + dY = 0; + } else { + // start the next row + Yo += dY; + Xo = 0; + + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotRadialPlot (&kapa, "radial.plots.ps", source); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + Xo = DX; + dY = DY; + } + } else { + // extend this row + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotRadialPlot (&kapa, "radial.plots.ps", source); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + Xo += DX; + dY = PS_MAX (dY, DY); + } + } + + psphotSaveImage (NULL, outpos, "outpos.fits"); + psphotSaveImage (NULL, outsub, "outsub.fits"); + psLogMsg ("psphot", PS_LOG_INFO, "plotted %d sources (%d psf, %d sat): %f sec\n", nPSF + nSAT, nPSF, nSAT, psTimerMark ("psphot")); + + psFree (outpos); + psFree (outsub); + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceSize.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceSize.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceSize.c (revision 41083) @@ -0,0 +1,1148 @@ +# include "psphotInternal.h" + +// this structure is only used internally to simplify the function parameters +typedef struct { + psImageMaskType maskVal; + psImageMaskType markVal; + psImageMaskType crMask; + float ApResid; + float ApSysErr; + float nSigmaApResid; + float nSigmaMoments; + float nSigmaCR; + bool altDiffExt; + float altDiffExtThresh; + float soft; + int grow; + int xtest, ytest; + bool applyCRmask; // apply CR mask? + bool dynamicLimitsCR; // apply CR mask? + float sizeLimitCR; + float magLimitCR; + int maxWindowCR; +} psphotSourceSizeOptions; + +// local functions: +bool psphotSourceSizePSF (psphotSourceSizeOptions *options, pmReadout *readout, psArray *sources, pmPSF *psf, psMetadata *recipe); +bool psphotDynamicLimitsCR (psphotSourceSizeOptions *options, pmReadout *readout, psArray *sources, pmPSF *psf, psMetadata *recipe); +bool psphotSourceClass (pmReadout *readout, psArray *sources, psMetadata *recipe, pmPSF *psf, psphotSourceSizeOptions *options); +bool psphotSourceClassRegion (psRegion *region, pmPSFClump *psfClump, psArray *sources, psMetadata *recipe, pmPSF *psf, psphotSourceSizeOptions *options); +bool psphotSourceSelectCR (pmReadout *readout, psArray *sources, psphotSourceSizeOptions *options); +bool psphotMaskCosmicRay (pmReadout *readout, pmSource *source, psImageMaskType maskVal, int maxWindowCR); +int psphotMaskCosmicRayConnected (int xPeak, int yPeak, psImage *mymask, psImage *myvar, psImage *edges, int binning, float sigma_thresh); +float psphotSourceSizeFindThreshold (psVector *value, psVector *mask, int maskValue, float minValue, float maxValue, float delta, float guess, float fraction); + +// save test output images? +# define DUMPPICS 0 + +// we need to call this function after sources have been fitted to the PSF model and +// subtracted. + +// for now, let's store the detections on the readout->analysis for each readout +bool psphotSourceSize (pmConfig *config, const pmFPAview *view, const char *filerule, bool getPSFsize) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Source Size ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotSourceSizeReadout (config, view, filerule, i, recipe, getPSFsize)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on source size analysis for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +// this function use an internal flag to mark sources which have already been measured +bool psphotSourceSizeReadout(pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool getPSFsize) +{ + bool status; + psphotSourceSizeOptions options; + + psTimerStart ("psphot.size"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping source size"); + return true; + } + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + options.maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (options.maskVal); + + options.markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + assert (options.markVal); + + // bit to mask the cosmic-ray pixels + options.crMask = pmConfigMaskGet("CR", config); // Mask value for cosmic rays + + options.nSigmaCR = psMetadataLookupF32 (&status, recipe, "PSPHOT.CR.NSIGMA.LIMIT"); + assert (status); + + // XXX recipe name is not great + options.nSigmaApResid = psMetadataLookupF32 (&status, recipe, "PSPHOT.EXT.NSIGMA.LIMIT"); + assert (status); + + // XXX recipe name is not great (NOTE : not used!) + options.nSigmaMoments = psMetadataLookupF32 (&status, recipe, "PSPHOT.EXT.NSIGMA.MOMENTS"); + assert (status); + + // Optional algorithm to define if a source is extended (used by DIFF analysis) + options.altDiffExt = psMetadataLookupBool(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE"); + assert (status); + + // Threshold for this alternate method + options.altDiffExtThresh = psMetadataLookupF32(&status, recipe, "PSPHOT.EXT.DIFF.ALTERNATE.THRESH"); + assert (status); + + // location of a single test source + options.xtest = psMetadataLookupS32 (&status, recipe, "PSPHOT.CRMASK.XTEST"); + options.ytest = psMetadataLookupS32 (&status, recipe, "PSPHOT.CRMASK.YTEST"); + // These are optional recipe values don't assert (status); + + options.grow = psMetadataLookupS32(&status, recipe, "PSPHOT.CR.GROW"); // Growth size for CRs + if (!status || options.grow < 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "PSPHOT.CR.GROW is not positive."); + return false; + } + + options.soft = psMetadataLookupF32(&status, recipe, "PSPHOT.CR.NSIGMA.SOFTEN"); // Softening parameter + if (!status || !isfinite(options.soft) || options.soft < 0.0) { + psWarning("PSPHOT.CR.NSIGMA.SOFTEN not set; defaulting to zero."); + options.soft = 0.0; + } + + options.applyCRmask = psMetadataLookupBool(&status, recipe, "PSPHOT.CRMASK.APPLY"); // Growth size for CRs + if (!status) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "PSPHOT.CRMASK.APPLY is not defined."); + return false; + } + options.maxWindowCR = psMetadataLookupS32 (&status, recipe, "PSPHOT.CR.MAX.WINDOW"); + if (!status) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "PSPHOT.CR.MAX.WINDOW is not defined."); + return false; + } + + // determine the distribution of (PSF_mag - KRON_mag) for the PSF sources (saved on readout->analysis) + psphotSourceSizePSF (&options, readout, sources, psf, recipe); + + // adjust the user-supplied limits based on the distribution of CRs in the (Mminor, mKron) space + psphotDynamicLimitsCR(&options, readout, sources, psf, recipe); + + // classify the sources based on ApResid and Moments + // NOTE: only sources not already measured !(source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) + psphotSourceClass(readout, sources, recipe, psf, &options); + + // attempt to mask the candidate CRs; flag if CR nature is confirmed + psphotSourceSelectCR(readout, sources, &options); + + // XXX fix this (was source->n - first) + psLogMsg ("psphot.size", PS_LOG_WARN, "measure source sizes for %ld sources: %f sec\n", sources->n, psTimerMark ("psphot.size")); + + psphotVisualPlotSourceSize (recipe, readout->analysis, sources); + psphotVisualPlotSourceSizeAlt (recipe, readout->analysis, sources); + psphotVisualShowSourceSize (readout, sources); + psphotVisualPlotApResid (sources, options.ApResid, options.ApSysErr, false); + psphotVisualShowSatStars (recipe, psf, sources); + + return true; +} + +# define SAVE_PSF_OPTIONS(RO,OPT) \ + /* save these on readout->analysis (a) for output to the header and (b) to prevent re-calculating in another pass */ \ + psMetadataAddF32(RO->analysis, PS_LIST_TAIL, "PSPHOT.PSF.APRESID", PS_META_REPLACE, "locus of PSF stars in PSF_MAG - KRON_MAG", OPT->ApResid); \ + psMetadataAddF32(RO->analysis, PS_LIST_TAIL, "PSPHOT.PSF.APRESID.SYSERR", PS_META_REPLACE, "systematic error of PSF_MAG - KRON_MAG", OPT->ApSysErr); + +// model the apmifit distribution for the psf stars: +bool psphotSourceSizePSF (psphotSourceSizeOptions *options, pmReadout *readout, psArray *sources, pmPSF *psf, psMetadata *recipe) { + + // We are using the value PSF_MAG - KRON_MAG as a measure of the extendedness of an object. + // We need to model this distribution for the PSF stars before we can test the significance + // for a specific object. + + // NOTE: we require that objects have had moments measured, and we also require psfMags to + // be calculated. but, we do not require aperture mags or any other photometry values that + // require pixel analysis. + + bool status1 = false; + bool status2 = false; + options->ApResid = psMetadataLookupF32 (&status1, readout->analysis, "PSPHOT.PSF.APRESID"); + options->ApSysErr = psMetadataLookupF32 (&status2, readout->analysis, "PSPHOT.PSF.APRESID.SYSERR"); + psAssert ((status1 && status2) || (!status1 && !status2), "inconsistent record of PSF ApResid values"); + + // if they are saved on readout->analysis, we have already calculated these values + if (status1 && status2) { + return true; + } + + // select stats from the psf stars + psVector *ApOff = psVectorAllocEmpty (100, PS_TYPE_F32); + psVector *ApErr = psVectorAllocEmpty (100, PS_TYPE_F32); + + psImageMaskType markVal = options->markVal; + psImageMaskType maskVal = options->maskVal | options->markVal; + + // with PSFONLY, we do not need modify the pixels + pmSourcePhotometryMode photMode = PM_SOURCE_PHOT_PSFONLY; + + int num = 0; // Number of sources measured + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!(source->mode & PM_SOURCE_MODE_PSFSTAR)) continue; + num++; + + pmSourceMagnitudes (source, psf, photMode, maskVal, markVal, source->apRadius); + + float kMag = -2.5*log10(source->moments->KronFluxPSF); + float dMag = source->psfMag - kMag; + + psVectorAppend (ApOff, dMag); + psVectorAppend (ApErr, source->psfMagErr); + } + if (num == 0) { + // if we cannot determine the PSF distribution, call all objects PSFs... + options->ApResid = NAN; + options->ApSysErr = NAN; + psFree(ApOff); + psFree(ApErr); + SAVE_PSF_OPTIONS(readout, options); + return false; + } + + // model the distribution as a mean or median value and a systematic error from that value: + // psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); + psStats *stats = psStatsAlloc(PS_STAT_CLIPPED_MEAN); + psVectorStats (stats, ApOff, NULL, NULL, 0); + + psVector *dAp = psVectorAlloc (ApOff->n, PS_TYPE_F32); + for (int i = 0; i < ApOff->n; i++) { + dAp->data.F32[i] = ApOff->data.F32[i] - stats->clippedMean; + } + + options->ApResid = stats->clippedMean; + options->ApSysErr = psVectorSystematicError(dAp, ApErr, 0.1); + + // this is quite arbitrary... a large value means fewer things classified as extended. + if (!isfinite(options->ApSysErr)) options->ApSysErr = 0.05; + psLogMsg ("psphot", PS_LOG_DETAIL, "psf - Sum: %f +/- %f\n", options->ApResid, options->ApSysErr); + + psFree (ApOff); + psFree (ApErr); + psFree (stats); + psFree (dAp); + + SAVE_PSF_OPTIONS(readout, options); + return true; +} + +# define SAVE_CR_OPTIONS(RO,OPT) \ + /* save these on readout->analysis (a) for output to the header and (b) to prevent re-calculating in another pass */ \ + psMetadataAddF32(RO->analysis, PS_LIST_TAIL, "PSPHOT.CR.MAX.SIZE", PS_META_REPLACE, "dynamically-set minor axis size limit for cosmic rays", OPT->sizeLimitCR); \ + psMetadataAddF32(RO->analysis, PS_LIST_TAIL, "PSPHOT.CR.MAX.MAG", PS_META_REPLACE, "dynamically-set kron magnitude limit for cosmic rays", OPT->magLimitCR); + +// model the size and magnitude distribution of the Cosmic Rays +// ** CRs are reliably flagged by a combination on Mminor < X && mag (or flux) > Y +bool psphotDynamicLimitsCR (psphotSourceSizeOptions *options, pmReadout *readout, psArray *sources, pmPSF *psf, psMetadata *recipe) { + + /* attempt to describe the CR sources: + - input parameters are sizeLimit, magLimit + - first try to refine the sizeLimit: + -- select objects which meet the magLimit and exceed the sizeLimit by a factor of 1.5 + -- generate the histogram + -- look for a peak in the histogram + -- look for the min between valley and upper limit + -- look for first bin within 5% of the valley floor after peak (new sizeLimit) + + - next try to refine the magLimit + -- select objects which meet the sizeLimit and go fainter than the magLimit by 1.0 mag + -- generate the histogram + -- look for a peak in the histogram + -- look for the min between valley and upper limit + -- look for first bin within 5% of the valley floor after peak (new magLimit) + */ + + bool status = false; + bool status1 = false; + bool status2 = false; + options->sizeLimitCR = psMetadataLookupF32 (&status1, readout->analysis, "PSPHOT.CR.MAX.SIZE"); + if (!status1) { + options->sizeLimitCR = psMetadataLookupF32 (&status, recipe, "PSPHOT.CR.MAX.SIZE"); + if (!status) { + options->sizeLimitCR = 1.0; + } + } + options->magLimitCR = psMetadataLookupF32 (&status2, readout->analysis, "PSPHOT.CR.MAX.MAG"); + if (!status2) { + options->magLimitCR = psMetadataLookupF32 (&status, recipe, "PSPHOT.CR.MAX.MAG"); + if (!status) { + options->magLimitCR = -8.0; + } + } + psAssert ((status1 && status2) || (!status1 && !status2), "inconsistent record of dynamic CR limits"); + + // if they are saved on readout->analysis, we have already calculated these values + if (status1 && status2) { + return true; + } + + // if we do not want to dynamically set these, save the user-supplied values and exit + options->dynamicLimitsCR = psMetadataLookupBool (&status, recipe, "PSPHOT.CR.AUTOSCALE"); + if (!status) { + options->dynamicLimitsCR = true; + } + if (!options->dynamicLimitsCR) { + SAVE_CR_OPTIONS(readout, options); // macro defined above + return true; + } + + psVector *minor = psVectorAllocEmpty (100, PS_TYPE_F32); + psVector *mKron = psVectorAllocEmpty (100, PS_TYPE_F32); + psVector *mask = NULL; + + psImageMaskType markVal = options->markVal; + psImageMaskType maskVal = options->maskVal | options->markVal; + + // with PSFONLY, we do not need modify the pixels + pmSourcePhotometryMode photMode = PM_SOURCE_PHOT_PSFONLY; + + // generate vectors for all the objects of possible interest (so we can just access those + // vectors in the next sections) + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // XXX can we test if psfMag is set and calculate only if needed? + pmSourceMagnitudes (source, psf, photMode, maskVal, markVal, source->apRadius); + + // convert to Mmaj, Mmin: + psF32 Mxx = source->moments->Mxx; + psF32 Myy = source->moments->Myy; + psF32 Mxy = source->moments->Mxy; + + float KronMag = -2.5*log10(source->moments->KronFluxPSF); + + float Mminor = 0.5*(Mxx + Myy) - 0.5*sqrt(PS_SQR(Mxx - Myy) + 4.0*PS_SQR(Mxy)); + + if (Mminor > options->sizeLimitCR * 1.5) continue; + if (KronMag > options->magLimitCR + 2.5) continue; + + psVectorAppend (mKron, KronMag); + psVectorAppend (minor, Mminor); + } + + // if too few objects meet the criterion, give up.. + if (mKron->n < 50) goto escape; + + // set distinct masks for (minor > sizeLimit) or (mKron > magLimit) + mask = psVectorAlloc(mKron->n, PS_TYPE_U8); + psVectorInit(mask, 0); + for (int i = 0; i < mKron->n; i++) { + if (mKron->data.F32[i] > options->magLimitCR) { + mask->data.U8[i] |= 0x01; + } + if (minor->data.F32[i] > options->sizeLimitCR) { + mask->data.U8[i] |= 0x02; + } + } + + float delta1 = PS_MAX(0.02, PS_MIN(0.2, 5.0 * 0.5 / minor->n)); + float newSizeLimit = psphotSourceSizeFindThreshold(minor, mask, 0x01, 0.0, options->sizeLimitCR * 1.5, delta1, options->sizeLimitCR, 0.05); + if (isfinite(newSizeLimit)) { + options->sizeLimitCR = newSizeLimit; + } + + float delta2 = PS_MAX(0.02, PS_MIN(0.2, 5.0 * 2.0 / minor->n)); + float newMagLimit = psphotSourceSizeFindThreshold(mKron, mask, 0x02, -15.0, options->magLimitCR + 2.5, delta2, options->magLimitCR, 0.05); + if (isfinite(newMagLimit)) { + options->magLimitCR = newMagLimit; + } + + psLogMsg ("psphot", PS_LOG_DETAIL, "CR limits : %f mag | %f pix^2\n", options->magLimitCR, options->sizeLimitCR); + + psFree (mKron); + psFree (minor); + psFree (mask); + + // save these on readout->analysis (a) for output to the header and (b) to prevent re-calculating in another pass + SAVE_CR_OPTIONS(readout, options); // macro defined above + return true; + + escape: + SAVE_CR_OPTIONS(readout, options); // macro defined above + return false; +} + +// classify sources based on the combination of psf-mag, Mxx, Myy +bool psphotSourceClass (pmReadout *readout, psArray *sources, psMetadata *recipe, pmPSF *psf, psphotSourceSizeOptions *options) { + + bool status; + pmPSFClump psfClump; + char regionName[64]; + + psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4s %4s %4s %4s %4s", "Npsf", "Next", "Nsat", "Ncr", "Nskip"); + + if (!psphotSourceClassRegion (NULL, &psfClump, sources, recipe, psf, options)) { + psLogMsg ("psphot", 4, "Failed to determine source classification for full image\n"); + } else { + psLogMsg ("psphot", 4, "source classification for full image\n"); + } + return true; + + // NOTE : this section is deactivated (EAM : I think we were getting poor boundary effects?) + int nRegions = psMetadataLookupS32 (&status, readout->analysis, "PSF.CLUMP.NREGIONS"); + for (int i = 0; i < nRegions; i ++) { + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", i); + psMetadata *regionMD = psMetadataLookupPtr (&status, readout->analysis, regionName); + psAssert (regionMD, "regions must be defined by earlier call to psphotRoughClassRegion"); + + psRegion *region = psMetadataLookupPtr (&status, regionMD, "REGION"); + psAssert (region, "regions must be defined by earlier call to psphotRoughClassRegion"); + + // pull FWHM_X,Y from the recipe, use to define psfClump.X,Y + psfClump.X = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); psAssert (status, "missing PSF.CLUMP.X"); + psfClump.Y = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); psAssert (status, "missing PSF.CLUMP.Y"); + psfClump.dX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); psAssert (status, "missing PSF.CLUMP.DX"); + psfClump.dY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); psAssert (status, "missing PSF.CLUMP.DY"); + + if ((psfClump.X < 0) || (psfClump.Y < 0) || !psfClump.X || !psfClump.Y || isnan(psfClump.X) || isnan(psfClump.Y)) { + psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + continue; + } + + if (!psphotSourceClassRegion (region, &psfClump, sources, recipe, psf, options)) { + psLogMsg ("psphot", 4, "Failed to determine source classification for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + continue; + } + psLogMsg ("psphot", 4, "source classification for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1); + // psphotVisualPlotSourceSize (recipe, readout->analysis, sources); + } + + return true; +} + +bool psphotSourceClassRegion (psRegion *region, pmPSFClump *psfClump, psArray *sources, psMetadata *recipe, pmPSF *psf, psphotSourceSizeOptions *options) { + + PS_ASSERT_PTR_NON_NULL(sources, false); + PS_ASSERT_PTR_NON_NULL(recipe, false); + + int Nsat = 0; + int Next = 0; + int Npsf = 0; + int Ncr = 0; + int Nskip = 0; + + pmSourceMode noMoments = PM_SOURCE_MODE_MOMENTS_FAILURE | PM_SOURCE_MODE_SKYVAR_FAILURE | PM_SOURCE_MODE_SKY_FAILURE | PM_SOURCE_MODE_BELOW_MOMENTS_SN; + + // request the pixWeight values as well as the magnitudes + pmSourcePhotometryMode photMode = PM_SOURCE_PHOT_WEIGHT; + + psImageMaskType markVal = options->markVal; + psImageMaskType maskVal = options->maskVal | options->markVal; + + // in the ppSub context, do we get sensible values for ApResid? + float ApResidPSF = options->ApResid; + if (!isfinite(ApResidPSF)) { + ApResidPSF = 0.0; + } + float ApSysErrPSF = options->ApSysErr; + if (!isfinite(ApSysErrPSF)) { + ApSysErrPSF = 0.0; + } + + for (psS32 i = 0 ; i < sources->n ; i++) { + + pmSource *source = (pmSource *) sources->data[i]; + + // psfClumps are found for image subregions: + // skip sources not in this region + if (region) { + if (source->peak->x < region->x0) continue; + if (source->peak->x >= region->x1) continue; + if (source->peak->y < region->y0) continue; + if (source->peak->y >= region->y1) continue; + } + + // skip source if it was already measured + if (source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED) { + psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); + continue; + } + + // source must have been subtracted + if (!(source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED)) { + source->mode |= PM_SOURCE_MODE_SIZE_SKIPPED; + psTrace("psphot", 7, "Not calculating source size since source is not subtracted\n"); + Nskip ++; + continue; + } + + // skip saturated stars modeled with a radial profile + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + // we are classifying by moments and PSF_MAG - KRON_MAG + psAssert (source->moments, "why is this source missing moments?"); + if (source->mode & noMoments) { + Nskip ++; + continue; + } + + // convert to Mmaj, Mmin: + psF32 Mxx = source->moments->Mxx; + psF32 Myy = source->moments->Myy; + psF32 Mxy = source->moments->Mxy; + float Mminor = 0.5*(Mxx + Myy) - 0.5*sqrt(PS_SQR(Mxx - Myy) + 4.0*PS_SQR(Mxy)); + + // replace object in image to measure mag & psfWeights + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, options->maskVal); + } + + // clear the mask bit and set the circular mask pixels + // psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + // psImageKeepCircle (source->maskObj, source->peak->x, source->peak->y, source->apRadius, "OR", options->markVal); + pmSourceMagnitudes (source, psf, photMode, maskVal, markVal, source->apRadius); + + // clear the mask bit + // psImageMaskPixels (source->maskObj, "AND", PS_NOT_IMAGE_MASK(options->markVal)); + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, options->maskVal); + + float kMag = -2.5*log10(source->moments->KronFluxPSF); + float dMag = source->psfMag - kMag; + + // sources without a valid magnitudes cannot have their size measured + if (!isfinite(kMag) || !isfinite(source->psfMag)) { + Nskip ++; + continue; + } + + // set nSigmaMAG to include both systematic and poisson error terms. we include a hard + // floor on the Ap Sys Err (to be a bit generous). XXX put the floor in the recipe... + float nSigmaMAG = (dMag - ApResidPSF) / hypot(source->psfMagErr, hypot(ApSysErrPSF, 0.02)); + source->extNsigma = nSigmaMAG; + + // notes to clarify the source size classification rules: + // * a defect should be functionally equivalent to a cosmic ray + // * CR & defect should have a faintess limit (min S/N) + // * SAT stars should not be faint, but defects may? + + // Defects may not always match CRs from peak curvature analysis + // Defects may also be marked as SATSTAR -- XXX deactivate this flag? + // XXX this rule is not great + // XXX only accept brightish detections as CRs + // (nSigmaMAG < -options->nSigmaApResid) || + + // saturated star (too many saturated pixels or peak above saturation limit). These + // may also be saturated galaxies, or just large saturated regions. They are never + // marked as 'extended' + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + psTrace("psphotSourceClassRegion.SAT",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g SAT\n", + source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid); + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Nsat ++; + continue; + } + + // any sources missing a large fraction should just be treated as PSFs. They are never + // marked as 'extended' + if ((source->pixWeightNotBad < 0.9) || (source->pixWeightNotPoor < 0.9)) { + psTrace("psphotSourceClassRegion.PSF",4,"CLASS: %g %g\t%g %g %g %g %g %g\t%g %g\t%g PSF\t%g %g\n", + source->peak->xf,source->peak->yf,Mxx,Myy,psfClump->X,psfClump->Y,psfClump->dX,psfClump->dY,kMag,dMag,nSigmaMAG, + options->nSigmaApResid,options->nSigmaMoments); + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Npsf ++; + continue; + } + + // CRs are flagged by a combination on Mminor < options->sizeLimitCR && kmag < options->magLimitCR + // NOTE: we only flag the CRs here; when we mask them we verify their CR nature (otherwise -> PSF) + // bool isCR = (kMag < options->magLimitCR) && (Mminor < options->sizeLimitCR); + // XXX skip if we have already marked it?? + bool isCR = (source->moments->SN > 7.0) && (Mminor < options->sizeLimitCR); + if (isCR) { + psTrace("psphotSourceClassRegion.CR",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g CR\n", + source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid); + source->mode |= PM_SOURCE_MODE_DEFECT; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_CR_CANDIDATE; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Ncr ++; + continue; + } + + // Likely extended source (PSF_MAG - KRON_MAG is larger than limit) + bool isEXT = (nSigmaMAG > options->nSigmaApResid); + if (isEXT) { + psTrace("psphotSourceClassRegion.EXT",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g EXT\n", + source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid); + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXT_LIMIT; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Next ++; + continue; + } + + // Alternate extended source limit calculation + if (options->altDiffExt) { + // ratio of major to minor axes + // MRV = Major / Minor + // MRV = (0.5 * (Mxx + Myy) + 0.5 * sqrt( (Mxx + Myy)^2 + 4 Mxy^2)) / + // (0.5 * (Mxx + Myy) - 0.5 * sqrt( (Mxx + Myy)^2 + 4 Mxy^2)) + // MRV = (2 * 0.5 * (Mxx + Myy) - Minor) / Minor + float momentRatioVeres = (Mxx + Myy - Mminor) / Mminor; + bool isAltEXT = (momentRatioVeres > options->altDiffExtThresh); + if (isAltEXT) { + psTrace("psphotSourceClassRegion.EXT",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g ALTEXT\t%g %g\n", + source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid, + momentRatioVeres,options->altDiffExtThresh); + source->type = PM_SOURCE_TYPE_EXTENDED; + source->mode |= PM_SOURCE_MODE_EXT_LIMIT; + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Next ++; + continue; + } + } + + // Everything else should just be treated as a PSF + psTrace("psphotSourceClassRegion.PSF",4,"CLASS: %g %g\t%g %g\t%g %g\t%g %g\t%g PSF\n", + source->peak->xf, source->peak->yf, Mminor, kMag, dMag, nSigmaMAG, options->sizeLimitCR, options->magLimitCR, options->nSigmaApResid); + source->tmpFlags |= PM_SOURCE_TMPF_SIZE_MEASURED; + Npsf ++; + } + + psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4d %4d %4d %4d %4d", Npsf, Next, Nsat, Ncr, Nskip); + + return true; +} + +// given an object suspected to be a defect, generate a pixel mask using the Lapacian transform +// if enough of the object is detected as 'sharp', consider the object a cosmic ray +bool psphotSourceSelectCR (pmReadout *readout, psArray *sources, psphotSourceSizeOptions *options) { + + psTimerStart ("psphot.cr"); + + int nMasked = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + // only check candidates marked above + if (!(source->tmpFlags & PM_SOURCE_TMPF_SIZE_CR_CANDIDATE)) { + psTrace("psphot", 7, "Not calculating source size since it has already been measured\n"); + continue; + } + + // once we are here, remove the temporary flag (this allows us to try again if we do not mark it now) + source->tmpFlags &= ~PM_SOURCE_TMPF_SIZE_CR_CANDIDATE; + + // Integer position of peak + int xPeak = source->peak->xf - source->pixels->col0 + 0.5; + int yPeak = source->peak->yf - source->pixels->row0 + 0.5; + + // Skip sources which are too close to a boundary. These are mostly caught as DEFECT + if (xPeak < 1 || xPeak > source->pixels->numCols - 2 || + yPeak < 1 || yPeak > source->pixels->numRows - 2) { + psTrace("psphot", 7, "Not calculating crNsigma due to edge\n"); + continue; + } + + // XXX for testing, only CRMASK a single source: + if (options->xtest && (fabs(source->peak->xf - options->xtest) > 5)) continue; + if (options->ytest && (fabs(source->peak->yf - options->ytest) > 5)) continue; + + // replace object in image + if (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED) { + pmSourceAdd (source, PM_MODEL_OP_FULL, options->maskVal); + } + + // XXX this is running slowly and is too agressive, but it more-or-less works + psTrace("psphot", 6, "mask cosmic ray at %f, %f\n", source->peak->xf, source->peak->yf); + if (options->applyCRmask) { + psphotMaskCosmicRay(readout, source, options->crMask, options->maxWindowCR); + } else { + source->mode |= PM_SOURCE_MODE_CR_LIMIT; + } + nMasked ++; + + // re-subtract the object, leave local sky + pmSourceSub (source, PM_MODEL_OP_FULL, options->maskVal); + } + + // now that we have masked pixels associated with CRs, we can grow the mask + if (options->grow > 0) { + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolveMask + psImage *newMask = psImageConvolveMask(NULL, readout->mask, options->crMask, options->crMask, -options->grow, options->grow, -options->grow, options->grow); + psImageConvolveSetThreads(oldThreads); + if (!newMask) { + psError(PS_ERR_UNKNOWN, false, "Unable to grow CR mask"); + return false; + } + // Copy the new mask pixel values to the old mask array. NOTE: we cannot replace + // the mask pointer because the source->maskView objects point to the old data + // area + psImage *oldMask = readout->mask; + readout->mask = psImageCopy (readout->mask, newMask, PS_TYPE_IMAGE_MASK); + psAssert (oldMask == readout->mask, "should have copied the values in-situ"); + psFree(newMask); + } + + psLogMsg ("psphot.cr", PS_LOG_INFO, "mask CR: %d masked in %f sec\n", nMasked, psTimerMark ("psphot.cr")); + + // XXX test : save the mask image +# if (DUMPPICS) + psphotSaveImage (NULL, readout->mask, "crmask.fits"); +# endif + + return true; +} + +# define LIMIT_XRANGE(X, IMAGE) { X = PS_MIN(PS_MAX(0, X), IMAGE->numCols); } +# define LIMIT_YRANGE(Y, IMAGE) { Y = PS_MIN(PS_MAX(0, Y), IMAGE->numRows); } + +// Comments by CZW 20091209 : Mechanics of how to identify CR pixels taken from "Cosmic-Ray +// Rejection by Laplacian Edge Detection" by Pieter van Dokkum, arXiv:astro-ph/0108003. This +// does no repair or recovery of the CR pixels, it only masks them out. My test code can be +// found at /data/ipp031.0/watersc1/psphot.20091209/algo_check.c +bool psphotMaskCosmicRay (pmReadout *readout, pmSource *source, psImageMaskType maskVal, int maxWindowCR) { + + // Get the actual images and information about the peak. + psImage *mask = readout->mask; + pmPeak *peak = source->peak; + pmFootprint *footprint = peak->footprint; + + // Bounding boxes are inclusive of final pixel + int xs = footprint->bbox.x0; + int xe = footprint->bbox.x1 + 1; + int ys = footprint->bbox.y0; + int ye = footprint->bbox.y1 + 1; + + // We occasionally get very large footprints. When the footprint bounding box is large this function will + // do an incredible amount of work for no benefit. + // Limit the size of the area we examine. + if (xe - xs > maxWindowCR) { + xs = peak->x - maxWindowCR / 2; + xe = xs + maxWindowCR; + } + if (ye - ys > maxWindowCR) { + ys = peak->y - maxWindowCR / 2; + ye = ys + maxWindowCR; + } + + LIMIT_XRANGE(xs, mask); + LIMIT_XRANGE(xe, mask); + LIMIT_YRANGE(ys, mask); + LIMIT_YRANGE(ye, mask); + + // the peak must be contained in the mask.. + if (peak->x < xs) return false; + if (peak->y < ys) return false; + if (peak->x >= xe) return false; + if (peak->y >= ye) return false; + + int dx = xe - xs; + int dy = ye - ys; + + psImage *image= readout->image; + psImage *variance = readout->variance; + + int binning = 2; + float sigma_thresh = 3.0; + int max_iter = 1; // XXX with isophot masking, we only want to do a single pass + + // Temporary images. + psImage *mypix = psImageAlloc(dx,dy,image->type.type); + psImage *myfix = psImageAlloc(dx,dy,image->type.type); + psImage *myvar = psImageAlloc(dx,dy,image->type.type); + psImage *binned = psImageAlloc(dx * binning,dy * binning,image->type.type); + psImage *conved = psImageAlloc(dx * binning,dy * binning,image->type.type); + psImage *edges = psImageAlloc(dx,dy,image->type.type); + psImage *mymask = psImageAlloc(dx,dy,PS_TYPE_IMAGE_MASK); + + // Load my copy of things. + for (int y = 0; y < dy; y++) { + for (int x = 0; x < dx; x++) { + mypix->data.F32[y][x] = image->data.F32[y+ys][x+xs]; + myvar->data.F32[y][x] = variance->data.F32[y+ys][x+xs]; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0x00; + } + } + // Mask so I can see on the output image where the footprint is. + for (int i = 0; i < footprint->spans->n; i++) { + pmSpan *sp = footprint->spans->data[i]; + int y = sp->y - ys; + if (y < 0 || y >= mymask->numRows) { + // can we break if y >= numRows? + continue; + } + for (int x = PS_MAX(sp->x0 - xs, 0); x <= PS_MIN(sp->x1 - xs, mymask->numCols-1); x++) { + mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= 0x01; + } + } + + int nCRpix = 1; // force at least one pass... + for (int iteration = 0; (iteration < max_iter) && (nCRpix > 0); iteration++) { + nCRpix = 0; + psImageInit (binned, 0.0); + psImageInit (conved, 0.0); + psImageInit (edges, 0.0); + + // Make subsampled image. Maybe this should be called "unbinned" or something + for (int y = 0; y < binning * dy; y++) { + int yraw = y / binning; + for (int x = 0; x < binning * dx; x++) { + int xraw = x / binning; + binned->data.F32[y][x] = mypix->data.F32[yraw][xraw]; + } + } + + // Apply Laplace transform (kernel = [[0 -0.25 0][-0.25 1 -0.25][0 -0.25 0]]), clipping at zero + for (int y = 1; y < binning * dy - 1; y++) { + for (int x = 1; x < binning * dx - 1; x++) { + float value = binned->data.F32[y][x] - 0.25 * + (binned->data.F32[y+0][x-1] + binned->data.F32[y+0][x+1] + + binned->data.F32[y-1][x+0] + binned->data.F32[y+1][x+0]); + value = PS_MAX(0.0, value); + + conved->data.F32[y][x] = value; + } + } + + // Create an edge map by rebinning + for (int y = 0; y < binning * dy; y++) { + int yraw = y / binning; + for (int x = 0; x < binning * dx; x++) { + int xraw = x / binning; + edges->data.F32[yraw][xraw] += conved->data.F32[y][x]; + } + } + + // coordinate of peak in subimage pixels: + int xPeak = peak->x - xs; + int yPeak = peak->y - ys; + + // Modify my mask if we're above the significance threshold, but only for connected pixels + nCRpix = psphotMaskCosmicRayConnected (xPeak, yPeak, mymask, myvar, edges, binning, sigma_thresh); + +# if DUMPPICS + psphotSaveImage (NULL, mypix, "crmask.pix.fits"); +# endif + +// XXX do not repair the pixels in isophot version +# if 0 + // "Repair" Masked pixels for the next round. + for (int y = 1; y < dy - 1; y++) { + for (int x = 1; x < dx - 1; x++) { + if (!(mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & 0x40)) { + myfix->data.F32[y][x] = mypix->data.F32[y][x]; + continue; + } + myfix->data.F32[y][x] = 0.25 * + (mypix->data.F32[y+0][x-1] + mypix->data.F32[y+0][x+1] + + mypix->data.F32[y-1][x+0] + mypix->data.F32[y+1][x+0]); + } + } + + // "Repair" Masked pixels for the next round. + for (int y = 1; y < dy - 1; y++) { + for (int x = 1; x < dx - 1; x++) { + mypix->data.F32[y][x] = myfix->data.F32[y][x]; + } + } +# endif + +# if DUMPPICS + fprintf (stderr, "CRMASK %d %d %d %d %d\n", xs, ys, dx, dy, iteration); + psphotSaveImage (NULL, mypix, "crmask.fix.fits"); + psphotSaveImage (NULL, myvar, "crmask.var.fits"); + psphotSaveImage (NULL, binned, "crmask.binn.fits"); + psphotSaveImage (NULL, conved, "crmask.conv.fits"); + psphotSaveImage (NULL, edges, "crmask.edge.fits"); + psphotSaveImage (NULL, mymask, "crmask.mask.fits"); +# endif + psTrace("psphot.czw",2,"Iter: %d Count: %d",iteration, nCRpix); + } + +# if 0 + // A solitary masked pixel is likely a lie. Remove those + // XXX can't we use nCRpix == 1 to test for these? + for (int x = 0; x < dx; x++) { + for (int y = 0; y < dy; y++) { + if (!(mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & 0x40)) continue; + if ((x-1 >= 0) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x-1] & 0x40)) { + continue; + } + if ((y-1 >= 0) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[y-1][x] & 0x40)) { + continue; + } + if ((x+1 < dx) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x+1] & 0x40)) { + continue; + } + if ((y+1 < dy) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[y+1][x] & 0x40)) { + continue; + } + mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] ^= 0x40; + } + } +# endif + + // transfer temporary mask to real mask & count masked pixels + nCRpix = 0; + for (int x = 0; x < dx; x++) { + for (int y = 0; y < dy; y++) { + if (mymask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & 0x40) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y+ys+mask->row0][x+xs+mask->col0] |= maskVal; + nCRpix ++; + } + } + } + + // XXX if we decide this REALLY is a cosmic ray, set the CR_LIMIT bit + if (nCRpix > 1) { + source->mode |= PM_SOURCE_MODE_CR_LIMIT; + } + // fprintf (stderr, "CRMASK %d %d %d %d %d\n", peak->x, peak->y, dx, dy, nCRpix); + + psFree(mypix); + psFree(myfix); + psFree(myvar); + psFree(binned); + psFree(conved); + psFree(edges); + psFree(mymask); + + return true; +} + +# define VERBOSE 0 +int psphotMaskCosmicRayConnected (int xPeak, int yPeak, psImage *mymask, psImage *myvar, psImage *edges, int binning, float sigma_thresh) { + + int xLo, xRo; + int nCRpix = 0; + + float noise_factor = 5.0 / 4.0; // Intrinsic to the Laplacian making noise spikes spikier. + + // mark the pixels in this row to the left, then the right. stay within footprint + int xL = xPeak; // find the range of valid pixels in this row + int xR = xPeak; + for (int ix = xPeak; (ix >= 0) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[yPeak][ix] & 0x01); ix--) { + float noise = binning * sqrt(noise_factor * myvar->data.F32[yPeak][ix]); + float value = edges->data.F32[yPeak][ix] / noise; + if (value < sigma_thresh ) break; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[yPeak][ix] |= 0x40; + xL = ix; + nCRpix ++; + if (VERBOSE) fprintf (stderr, "mark %d,%d (%d) : %d - %d\n", ix, yPeak, nCRpix, xL, xR); + } + for (int ix = xPeak; (ix < mymask->numCols) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[yPeak][ix] & 0x01); ix++) { + float noise = binning * sqrt(noise_factor * myvar->data.F32[yPeak][ix]); + float value = edges->data.F32[yPeak][ix] / noise; + if (value < sigma_thresh ) break; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[yPeak][ix] |= 0x40; + xR = ix; + nCRpix ++; + if (VERBOSE) fprintf (stderr, "mark %d,%d (%d) : %d - %d\n", ix, yPeak, nCRpix, xL, xR); + } + // xL and xR mark the first and last valid pixel in the row + + // for each of the neighboring rows, mark the high pixels if they touch the range xL to xR + xLo = PS_MAX(xL - 1, 0); + xRo = PS_MIN(xR + 1, mymask->numCols); + + // first go down: + for (int iy = yPeak - 1; iy >= 0; iy--) { + + int xLn = -1; + int xRn = -1; + int newPix = 0; + + // mark the pixels in the good range + for (int ix = xLo; ix < xRo; ix++) { + if (!(mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & 0x01)) continue; // only use pixels in the footprint + float noise = binning * sqrt(noise_factor * myvar->data.F32[iy][ix]); + float value = edges->data.F32[iy][ix] / noise; + if (value < sigma_thresh ) continue; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= 0x40; + if (xLn == -1) xLn = ix; // first valid pixel in this row + xRn = ix; // last valid pixel in this row + nCRpix ++; + newPix ++; + if (VERBOSE) fprintf (stderr, "mark C %d,%d (%d) : %d - %d | %d - %d | %d - %d \n", ix, iy, nCRpix, xL, xR, xLo, xRo, xLn, xRn); + } + + // mark the pixels to the left of the good range + for (int ix = xLo; (ix >= 0) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & 0x01); ix--) { + float noise = binning * sqrt(noise_factor * myvar->data.F32[iy][ix]); + float value = edges->data.F32[iy][ix] / noise; + if (value < sigma_thresh ) break; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= 0x40; + if (xRn == -1) xRn = ix; // last valid pixel in this row + xLn = ix; + nCRpix ++; + newPix ++; + if (VERBOSE) fprintf (stderr, "mark L %d,%d (%d) : %d - %d | %d - %d | %d - %d \n", ix, iy, nCRpix, xL, xR, xLo, xRo, xLn, xRn); + } + + // mark the pixels to the right of the good range + for (int ix = xRo; (ix < mymask->numCols) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & 0x01); ix++) { + float noise = binning * sqrt(noise_factor * myvar->data.F32[iy][ix]); + float value = edges->data.F32[iy][ix] / noise; + if (value < sigma_thresh ) break; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= 0x40; + if (xLn == -1) xLn = ix; // first valid pixel in this row + xRn = ix; + nCRpix ++; + newPix ++; + if (VERBOSE) fprintf (stderr, "mark R %d,%d (%d) : %d - %d | %d - %d | %d - %d \n", ix, iy, nCRpix, xL, xR, xLo, xRo, xLn, xRn); + } + if (newPix == 0) break; + xLo = PS_MAX(xLn - 1, 0); + xRo = PS_MIN(xRn + 1, mymask->numCols); + } + + xLo = PS_MAX(xL - 1, 0); + xRo = PS_MIN(xR + 1, mymask->numCols); + + // next go up: + for (int iy = yPeak + 1; iy < mymask->numRows; iy++) { + + int xLn = -1; + int xRn = -1; + int newPix = 0; + + // mark the pixels in the good range + for (int ix = xLo; ix < xRo; ix++) { + if (!(mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & 0x01)) continue; // only use pixels in the footprint + float noise = binning * sqrt(noise_factor * myvar->data.F32[iy][ix]); + float value = edges->data.F32[iy][ix] / noise; + if (value < sigma_thresh ) continue; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= 0x40; + if (xLn == -1) xLn = ix; // first valid pixel in this row + xRn = ix; // last valid pixel in this row + nCRpix ++; + newPix ++; + if (VERBOSE) fprintf (stderr, "mark C %d,%d (%d) : %d - %d | %d - %d | %d - %d \n", ix, iy, nCRpix, xL, xR, xLo, xRo, xLn, xRn); + } + + // mark the pixels to the left of the good range + for (int ix = xLo; (ix >= 0) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & 0x01); ix--) { + float noise = binning * sqrt(noise_factor * myvar->data.F32[iy][ix]); + float value = edges->data.F32[iy][ix] / noise; + if (value < sigma_thresh ) break; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= 0x40; + if (xRn == -1) xRn = ix; // last valid pixel in this row + xLn = ix; + nCRpix ++; + newPix ++; + if (VERBOSE) fprintf (stderr, "mark L %d,%d (%d) : %d - %d | %d - %d | %d - %d \n", ix, iy, nCRpix, xL, xR, xLo, xRo, xLn, xRn); + } + + // mark the pixels to the right of the good range + for (int ix = xRo; (ix < mymask->numCols) && (mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & 0x01); ix++) { + float noise = binning * sqrt(noise_factor * myvar->data.F32[iy][ix]); + float value = edges->data.F32[iy][ix] / noise; + if (value < sigma_thresh ) break; + mymask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= 0x40; + if (xLn == -1) xLn = ix; // first valid pixel in this row + xRn = ix; + nCRpix ++; + newPix ++; + if (VERBOSE) fprintf (stderr, "mark R %d,%d (%d) : %d - %d | %d - %d | %d - %d \n", ix, iy, nCRpix, xL, xR, xLo, xRo, xLn, xRn); + } + if (newPix == 0) break; + xLo = PS_MAX(xLn - 1, 0); + xRo = PS_MIN(xRn + 1, mymask->numCols); + } + + return nCRpix; +} + +float psphotSourceSizeFindThreshold (psVector *value, psVector *mask, int maskValue, float minValue, float maxValue, float delta, float guess, float fraction) { + + // make a histogram of the sources with mKron < magLimit + int nValue = (int)((maxValue - minValue) / delta) + 1; + + psVector *histogram = psVectorAlloc(nValue, PS_TYPE_S32); + psVectorInit (histogram, 0); + + for (int i = 0; i < value->n; i++) { + if (mask->data.U8[i] & maskValue) continue; + int bin = (value->data.F32[i] - minValue) / delta; + if (bin < 0.0) continue; + if (bin > histogram->n - 1) continue; + histogram->data.S32[bin] ++; + } + + // find the peak of this histogram, but stop search at guess + int last = (guess - minValue) / delta; + int nPeak = 0; + int vPeak = histogram->data.S32[0]; + for (int i = 1; (i < last) && (i < histogram->n); i++) { + if (histogram->data.S32[i] < vPeak) continue; + nPeak = i; + vPeak = histogram->data.S32[i]; + } + + // start at the peak and find the valley between here and the end of the histogram + int nValley = nPeak; + int vValley = histogram->data.S32[nPeak]; + for (int i = nPeak + 1; i < histogram->n; i++) { + if (histogram->data.S32[i] > vValley) continue; + nValley = i; + vValley = histogram->data.S32[i]; + } + + psLogMsg ("psphot", PS_LOG_MINUTIA, "CR limits threshold : peak %d @ %f, valley %d @ %f (%f sigma)\n", + vPeak, delta * nPeak + minValue, vValley, delta * nValley + minValue, vPeak / PS_MAX(1.0, sqrt(vValley))); + + if (nValley == nPeak) { + psFree (histogram); + return NAN; + } + + if (vPeak < 3.0*sqrt(vValley)) { + psFree (histogram); + return NAN; + } + + /// search for the first bin after the peak with f < vValley + 0.05*(vPeak - vValley) + float vLimit = vValley + fraction*(vPeak - vValley); + int nLimit = nValley; + for (int i = nPeak; i < nValley; i++) { + if (histogram->data.S32[i] > vLimit) continue; + nLimit = i; + break; + } + float result = nLimit * delta + minValue; + + psFree (histogram); + + return result; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceStats.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceStats.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSourceStats.c (revision 41083) @@ -0,0 +1,651 @@ +# include "psphotInternal.h" +void pmSourceMomentsSetVerbose(bool state); + +// convert detections to sources and measure their basic properties (moments, local sky, sky +// variance) Note: this function only generates sources for the new peaks (peak->assigned). +// The new sources are added to any existing sources on detections->newSources. The sources +// on detections->allSources are ignored. +bool psphotSourceStats (pmConfig *config, const pmFPAview *view, const char *filerule, bool setWindow) +{ + bool status = true; + + fprintf (stdout, "\n"); + psLogMsg ("psphot", PS_LOG_INFO, "--- psphot Source Stats ---"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + // int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + // if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + // if (i == chisqNum) continue; // skip chisq image + if (!psphotSourceStatsReadout (config, view, filerule, i, recipe, setWindow)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to find initial detections for %s entry %d", filerule, i); + return false; + } + } + return true; +} + +bool psphotSourceStatsReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe, bool setWindow) { + + bool status = false; + psArray *sources = NULL; + + psTimerStart ("psphot.stats"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psTraceGetLevel("psphot") > 5) { + static int pass = 0; + char name[64]; + sprintf (name, "srstats.v%d.fits", pass); + psphotSaveImage(NULL, readout->image, name); + pass ++; + } + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // determine properties (sky, moments) of initial sources + float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + psAssert (status, "missing SKY_OUTER_RADIUS in recipe?"); + + // XXX this seems like an arbitrary number... + OUTER = PS_MAX(OUTER, 20.0); // XXX Guarantee that we can encompass the max moments radius + + char *breakPt = psMetadataLookupStr (&status, recipe, "BREAK_POINT"); + psAssert (status, "missing BREAK_POINT?"); + + float INNER = psMetadataLookupF32 (&status, recipe, "SKY_INNER_RADIUS"); psAssert (status, "missing SKY_INNER_RADIUS"); + float MIN_SN = psMetadataLookupF32 (&status, recipe, "MOMENTS_SN_MIN"); psAssert (status, "missing MOMENTS_SN_MIN"); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + psAssert (maskVal, "missing MASK.PSPHOT"); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + psAssert (markVal, "missing MARK.PSPHOT"); + + psArray *peaks = detections->peaks; + if (!peaks) { + psError(PS_ERR_UNEXPECTED_NULL, false, "No peaks found!"); + return false; + } + + // generate the array of sources, define the associated pixel + psAssert (!detections->newSources, "programming error"); + sources = detections->newSources = psArrayAllocEmpty (peaks->n); + + bool firstPass = (detections->allSources == NULL); + + // if there are no peaks, we save the empty source array and return + if (!peaks->n) { + return true; + } + + for (int i = 0; i < peaks->n; i++) { + + pmPeak *peak = peaks->data[i]; + if (peak->assigned) continue; + + // create a new source + pmSource *source = pmSourceAlloc(); + source->imageID = index; + + // add the peak + source->peak = psMemIncrRefCounter(peak); + + // psAssert (source->peak->footprint, "peak without footprint??"); + + // allocate space for moments + source->moments = pmMomentsAlloc(); + + if (source->mode & PM_SOURCE_MODE_MOMENTS_FAILURE) { + fprintf (stderr, "moment failure\n"); + } + + if (firstPass) { + source->mode2 |= PM_SOURCE_MODE2_PASS1_SRC; + } + + // allocate image, weight, mask arrays for each peak (square of radius OUTER) + pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); + + peak->assigned = true; + psArrayAdd (sources, 100, source); + psFree (source); + } + + if (!strcasecmp (breakPt, "PEAKS")) { + psLogMsg ("psphot", PS_LOG_INFO, "%ld sources : %f sec\n", sources->n, psTimerMark ("psphot.stats")); + psLogMsg ("psphot", PS_LOG_INFO, "break point PEAKS, skipping MOMENTS\n"); + psphotVisualShowMoments (sources); + return true; + } + + if (setWindow) { + if (!psphotSetMomentsWindow(recipe, readout->analysis, sources, maskVal)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Failed to determine Moments Window!"); + psFree(detections->newSources); + return false; + } + } + + // if we have measured the window, we will be saving the modified version of these recipe values on readout->analysis + float SIGMA = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_GAUSS_SIGMA"); + if (!status) { + SIGMA = psMetadataLookupF32 (&status, recipe, "MOMENTS_GAUSS_SIGMA"); + } + float RADIUS = psMetadataLookupF32 (&status, readout->analysis, "PSF_MOMENTS_RADIUS"); + if (!status) { + RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_MOMENTS_RADIUS"); + } + float MIN_KRON_RADIUS = psMetadataLookupF32 (&status, readout->analysis, "MOMENTS_MIN_KRON"); + if (!status) { + MIN_KRON_RADIUS = 0.75*SIGMA; + } + + // threaded measurement of the source magnitudes + int Nfail = 0; + int Nmoments = 0; + int Nfaint = 0; + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_SOURCE_STATS"); + + psArrayAdd(job->args, 1, cells->data[j]); // 0 : sources + + PS_ARRAY_ADD_SCALAR(job->args, INNER, PS_TYPE_F32); // 1 + PS_ARRAY_ADD_SCALAR(job->args, MIN_SN, PS_TYPE_F32); // 2 + PS_ARRAY_ADD_SCALAR(job->args, RADIUS, PS_TYPE_F32); // 3 + PS_ARRAY_ADD_SCALAR(job->args, SIGMA, PS_TYPE_F32); // 4 + PS_ARRAY_ADD_SCALAR(job->args, MIN_KRON_RADIUS, PS_TYPE_F32); // 5 + + PS_ARRAY_ADD_SCALAR(job->args, maskVal, PS_TYPE_IMAGE_MASK); // 6 + PS_ARRAY_ADD_SCALAR(job->args, markVal, PS_TYPE_IMAGE_MASK); // 7 + + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // 8 : this is used as a return value for Nmoments + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // 9 : this is used as a return value for Nfail + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // 10 : this is used as a return value for Nfaint + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to launch thread job PSPHOT_SOURCE_STATS"); + psFree(detections->newSources); + return false; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS"); + psFree(detections->newSources); + return false; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[8]; + Nmoments += scalar->data.S32; + scalar = job->args->data[9]; + Nfail += scalar->data.S32; + scalar = job->args->data[10]; + Nfaint += scalar->data.S32; + } + psFree(job); + } + } + + psFree (cellGroups); + + psLogMsg ("psphot", PS_LOG_WARN, "%ld sources, %d moments, %d faint, %d failed: %f sec\n", sources->n, Nmoments, Nfaint, Nfail, psTimerMark ("psphot.stats")); + + psphotVisualShowMoments (sources); + + return true; +} + +// this function is currently only called by psphotCheckExtSources +bool psphotSourceStatsUpdate (psArray *sources, pmConfig *config, pmReadout *readout) { + + bool status = false; + + psTimerStart ("psphot.stats"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // determine the number of allowed threads + int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads + if (!status) { + nThreads = 0; + } + + // determine properties (sky, moments) of initial sources + float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + if (!status) return false; + + OUTER = PS_MAX(OUTER, 20.0); // XXX Guarantee that we can encompass the max moments radius + + char *breakPt = psMetadataLookupStr (&status, recipe, "BREAK_POINT"); + if (!status) return false; + + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source->peak) continue; // XXX how can we have a peak-less source? + + // allocate space for moments + if (!source->moments) { + source->moments = pmMomentsAlloc(); + } + + // allocate image, weight, mask arrays for each peak (square of radius OUTER) + pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER); + source->peak->assigned = true; + } + + if (!strcasecmp (breakPt, "PEAKS")) { + psLogMsg ("psphot", PS_LOG_INFO, "%ld sources : %f sec\n", sources->n, psTimerMark ("psphot.stats")); + psLogMsg ("psphot", PS_LOG_INFO, "break point PEAKS, skipping MOMENTS\n"); + psphotVisualShowMoments (sources); + return sources; + } + + // XXX how else could we get the window size in? + if (!psphotSetMomentsWindow(recipe, readout->analysis, sources, 0)) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Failed to determine Moments Window!"); + return NULL; + } + + // threaded measurement of the source magnitudes + int Nfail = 0; + int Nmoments = 0; + int Nfaint = 0; + + // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts) + int Cx = 1, Cy = 1; + psphotChooseCellSizes (&Cx, &Cy, readout, nThreads); + + psArray *cellGroups = psphotAssignSources (Cx, Cy, sources); + + for (int i = 0; i < cellGroups->n; i++) { + + psArray *cells = cellGroups->data[i]; + + for (int j = 0; j < cells->n; j++) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("PSPHOT_SOURCE_STATS"); + + // XXX: this must match the above + psAbort ("this code is broken: must match psphotSourceStatsReadout above"); + psArrayAdd(job->args, 1, cells->data[j]); // sources + psArrayAdd(job->args, 1, recipe); + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nmoments + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail + PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfaint + + if (!psThreadJobAddPending(job)) { + psError(PS_ERR_UNKNOWN, false, "Unable to launch thread job PSPHOT_SOURCE_STATS"); + return NULL; + } + } + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (false, true)) { + psError(PS_ERR_UNKNOWN, false, "Failure in thread job PSPHOT_SOURCE_STATS"); + return NULL; + } + + // we have only supplied one type of job, so we can assume the types here + psThreadJob *job = NULL; + while ((job = psThreadJobGetDone()) != NULL) { + if (job->args->n < 1) { + fprintf (stderr, "error with job\n"); + } else { + psScalar *scalar = NULL; + scalar = job->args->data[2]; + Nmoments += scalar->data.S32; + scalar = job->args->data[3]; + Nfail += scalar->data.S32; + scalar = job->args->data[4]; + Nfaint += scalar->data.S32; + } + psFree(job); + } + } + + psFree (cellGroups); + + psLogMsg ("psphot", PS_LOG_INFO, "%ld sources, %d moments, %d faint, %d failed: %f sec\n", sources->n, Nmoments, Nfaint, Nfail, psTimerMark ("psphot.stats")); + + psphotVisualShowMoments (sources); + + return (sources); +} + +bool psphotSourceStats_Threaded (psThreadJob *job) { + + bool status = false; + psScalar *scalar = NULL; + + psArray *sources = job->args->data[0]; + + float INNER = PS_SCALAR_VALUE(job->args->data[1],F32); + float MIN_SN = PS_SCALAR_VALUE(job->args->data[2],F32); + float RADIUS = PS_SCALAR_VALUE(job->args->data[3],F32); + float SIGMA = PS_SCALAR_VALUE(job->args->data[4],F32); + float MIN_KRON_RADIUS = PS_SCALAR_VALUE(job->args->data[5],F32); + + psImageMaskType maskVal = PS_SCALAR_VALUE(job->args->data[6],PS_TYPE_IMAGE_MASK_DATA); + psImageMaskType markVal = PS_SCALAR_VALUE(job->args->data[7],PS_TYPE_IMAGE_MASK_DATA); + + // if no valid pixels, massive swing or very large Mrf, likely saturated source, + // try a much larger box + float BIG_RADIUS = 3.0*RADIUS; + float BIG_SIGMA = 3.0*SIGMA; + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // XXX test : pmSourceMomentsSetVerbose(true); + + // threaded measurement of the sources moments + int Nfail = 0; + int Nmoments = 0; + int Nfaint = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + if (source->tmpFlags & PM_SOURCE_TMPF_MOMENTS_MEASURED) continue; + source->tmpFlags |= PM_SOURCE_TMPF_MOMENTS_MEASURED; + + // skip faint sources for moments measurement + if (sqrt(source->peak->detValue) < MIN_SN) { + source->mode |= PM_SOURCE_MODE_BELOW_MOMENTS_SN; + Nfaint++; + continue; + } + + // measure a local sky value + // the local sky is now ignored; kept here for reference only + status = pmSourceLocalSky (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal); + if (!status) { + source->mode |= PM_SOURCE_MODE_SKY_FAILURE; + psErrorClear(); // XXX re-consider the errors raised here + Nfail ++; + continue; + } + + // measure the local sky variance (needed if noise is not sqrt(signal)) + // XXX EAM : this should use ROBUST not SAMPLE median, but it is broken + status = pmSourceLocalSkyVariance (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal); + if (!status) { + source->mode |= PM_SOURCE_MODE_SKYVAR_FAILURE; + Nfail ++; + psErrorClear(); + continue; + } + + // XXX how can this be set: this is raised in pmSourceRoughClass, called after this function? + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + fprintf (stderr, "satstar: %f,%f\n", source->peak->xf, source->peak->yf); + } + + // skip saturated stars modeled with a radial profile (this probably never happens, since it is set after) + if (source->mode2 & PM_SOURCE_MODE2_SATSTAR_PROFILE) continue; + + if (!(source->peak->type == PM_PEAK_SUSPECT_SATURATION)) { + // measure basic source moments (no S/N clipping on input pixels) + status = pmSourceMoments (source, RADIUS, SIGMA, 0.0, MIN_KRON_RADIUS, maskVal); + } else { + // For saturated stars, choose a much larger box NOTE this is slightly sleazy, but + // only slightly: pmSourceRedefinePixels uses the readout to pass the pointers to + // the parent image data. I guess the API could be simplified: we could recover + // this from the source in the function + + pmReadout tmpReadout; + tmpReadout.image = (psImage *)source->pixels->parent; + tmpReadout.mask = (psImage *)source->maskView->parent; + tmpReadout.variance = (psImage *)source->variance->parent; + + // re-allocate image, weight, mask arrays for each peak with box big enough to fit BIG_RADIUS + pmSourceRedefinePixels (source, &tmpReadout, source->peak->x, source->peak->y, BIG_RADIUS + 2); + + psTrace ("psphot", 4, "retrying moments for %d, %d\n", source->peak->x, source->peak->y); + status = pmSourceMoments (source, BIG_RADIUS, BIG_SIGMA, 0.0, MIN_KRON_RADIUS, maskVal); + source->mode |= PM_SOURCE_MODE_BIG_RADIUS; + } + if (!status) { + source->mode |= PM_SOURCE_MODE_MOMENTS_FAILURE; + Nfail ++; + psErrorClear(); + continue; + } + + // XXX test of masking neighbors when measureing moments (does this fail?) + // psEllipseMoments moments = {source->moments->Mxx, source->moments->Myy, source->moments->Mxy}; + // psEllipseAxes axes = psEllipseMomentsToAxes(moments, 20.0); + // psImageMaskCircle (source->maskView, source->peak->x, source->peak->y, 3.0*axes.major, "OR", markVal); + + Nmoments ++; + + // re-try big sources or not?? + // if (source->moments->Mrf < 2.0*SIGMA) + } + + // change the value of a scalar on the array (wrap this and put it in psArray.h) + scalar = job->args->data[8]; + scalar->data.S32 = Nmoments; + + scalar = job->args->data[9]; + scalar->data.S32 = Nfail; + + scalar = job->args->data[10]; + scalar->data.S32 = Nfaint; + + return true; +} + +// this function attempts to iteratively determine the best value for sigma of the moments weighting Gaussian +// this function modifies the recipe values MOMENTS_SX_MAX, MOMENTS_SY_MAX, and PSF_CLUMP_GRID_SCALE, used by pmSourcePSFClump +bool psphotSetMomentsWindow (psMetadata *recipe, psMetadata *analysis, psArray *sources, psImageMaskType maskVal) { + + bool status; + + float MIN_SN = psMetadataLookupF32 (&status, recipe, "MOMENTS_SN_MIN"); psAssert (status, "missing MOMENTS_SN_MIN"); + psF32 MOMENTS_AR_MAX = psMetadataLookupF32(&status, recipe, "MOMENTS_AR_MAX"); psAssert (status, "missing MOMENTS_AR_MAX"); + + float MOMENTS_SX_MIN = psMetadataLookupF32(&status, recipe, "MOMENTS_SX_MIN"); + if (!status) { + MOMENTS_SX_MIN = 0.5; + } + float MOMENTS_SY_MIN = psMetadataLookupF32(&status, recipe, "MOMENTS_SY_MIN"); + if (!status) { + MOMENTS_SY_MIN = 0.5; + } + + // when we set the window, we are not attempting to measure spatial variations; we can use a somewhat higher S/N limit + // since we are using all sources (true?) + float PSF_SN_LIM = 2.0*psMetadataLookupF32(&status, recipe, "PSF_SN_LIM"); psAssert (status, "missing PSF_SN_LIM"); + + // XXX this will cause an error in the vector length is > 8 + # define NSIGMA 8 + // moved to config file + psVector *sigmavec = psMetadataLookupPtr (&status, recipe, "PSF.SIGMA.VALUES"); + + float sigma[NSIGMA] = {0,0,0,0,0,0,0}; + float Sout[NSIGMA] = {0,0,0,0,0,0,0}; + float Rmin[NSIGMA] = {0,0,0,0,0,0,0}; + int Nout[NSIGMA]; // number of stars found in clump : use this to control the number of regions measured by psphotRoughClass + + int nsigma; + if (sigmavec) { + psAssert(sigmavec->n <= NSIGMA, "too many sigma values in recipe %ld maximum is %d", sigmavec->n, NSIGMA); + // copy the data from vector to local array to keep the code below easier to read + for (int i = 0 ; i < sigmavec->n; i++) { + sigma[i] = sigmavec->data.F32[i]; + } + assert (sigmavec->n <= 8); + nsigma = sigmavec->n; + } else { + // requiring this causes updates to pop an assertion + // psAssert(status, "missing PSF.SIGMA.VALUES"); + float defaultsigma[NSIGMA] = {1.0, 2.0, 3.0, 4.5, 6.0, 9.0, 12.0, 18.0}; + for (int i = 0 ; i < NSIGMA; i++) { + sigma[i] = defaultsigma[i]; + } + nsigma = NSIGMA; + } + + // this sorts by peak->rawFlux + sources = psArraySort (sources, pmSourceSortByFlux); + + // loop over radii: + for (int i = 0; i < nsigma; i++) { + + // XXX move max source number to config + for (int j = 0; (j < sources->n) && (j < 400); j++) { + + pmSource *source = sources->data[j]; + psAssert (source->moments, "force moments to exist"); + source->moments->nPixels = 0; + + // skip faint sources for moments measurement + if (sqrt(source->peak->detValue) < MIN_SN) { + continue; + } + + // measure basic source moments (no S/N clipping on input pixels) + // sources with (mode & MODE_EXTERNAL) or (mode2 & MODE2_MATCHED) use the + // supplied Mx,My value for the centroid (not recalculated) + status = pmSourceMoments (source, 4*sigma[i], sigma[i], 0.0, 0.0, maskVal); + } + +#if (0) + if (psTraceGetLevel("psphot.moments.save")) { + char name[64]; + sprintf (name, "moments.v%d.dat", i); + FILE *fout = fopen (name, "w"); + for (int j = 0; j < sources->n; j++) { + pmSource *source = sources->data[j]; + psAssert (source->moments, "force moments to exist"); + source->moments->nPixels = 0; + status = pmSourceMoments (source, 20, sigma[i], 0.0, 0.0, maskVal); + fprintf (fout, "%f %f | %f %f %f | %f\n", source->moments->Mx, source->moments->My, source->moments->Mxx, source->moments->Mxy, source->moments->Myy, source->moments->Sum); + } + fclose (fout); + } +#endif + // choose a grid scale that is a fixed fraction of the psf sigma^2 + float PSF_CLUMP_GRID_SCALE = 0.1*PS_SQR(sigma[i]); + float MOMENTS_SX_MAX = 2.0*PS_SQR(sigma[i]); + float MOMENTS_SY_MAX = 2.0*PS_SQR(sigma[i]); + + // determine the PSF parameters from the source moment values + pmPSFClump psfClump = pmSourcePSFClump (NULL, NULL, sources, PSF_SN_LIM, PSF_CLUMP_GRID_SCALE, MOMENTS_SX_MAX, MOMENTS_SY_MAX, MOMENTS_SX_MIN, MOMENTS_SY_MIN, MOMENTS_AR_MAX); + psLogMsg ("psphot", 3, "sigma guess (pix) %.1f, nStars: %d of %d in clump, nSigma: %5.2f, X, Y: %f, %f (%f, %f)\n", sigma[i], psfClump.nStars, psfClump.nTotal, psfClump.nSigma, psfClump.X, psfClump.Y, sqrt(psfClump.X) / sigma[i], sqrt(psfClump.Y) / sigma[i]); + + Rmin[i] = pmSourceMinKronRadius(sources, PSF_SN_LIM); + +#if 0 + // Modifying clump parameters without restoring! + psMetadataAddS32 (analysis, PS_LIST_TAIL, "PSF.CLUMP.NREGIONS", PS_META_REPLACE, "psf clump regions", 1); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, "PSF.CLUMP.REGION.000"); + if (!regionMD) { + regionMD = psMetadataAlloc(); + psMetadataAddMetadata (analysis, PS_LIST_TAIL, "PSF.CLUMP.REGION.000", PS_META_REPLACE, "psf clump region", regionMD); + psFree (regionMD); + } + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.X", PS_META_REPLACE, "psf clump center", psfClump.X); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.Y", PS_META_REPLACE, "psf clump center", psfClump.Y); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DX", PS_META_REPLACE, "psf clump center", psfClump.dX); + psMetadataAddF32 (regionMD, PS_LIST_TAIL, "PSF.CLUMP.DY", PS_META_REPLACE, "psf clump center", psfClump.dY); + if (pmVisualTestLevel("psphot.moments.full", 2)) { + psphotVisualPlotMoments (recipe, analysis, sources); + } +#endif + + Sout[i] = sqrt(0.5*(psfClump.X + psfClump.Y)) / sigma[i]; + Nout[i] = psfClump.nStars; + } + + // we are looking for sigma for which Sout = 0.65 (or some other value) + int Nstars = 0; + float minKronRadius = NAN; + float Sigma = NAN; + float minS = Sout[0]; + float maxS = Sout[0]; + for (int i = 0; i < nsigma; i++) { + minS = PS_MIN(Sout[i], minS); + maxS = PS_MAX(Sout[i], maxS); + } + if (minS > 0.65) Sigma = sigma[nsigma-1]; + if (maxS < 0.65) Sigma = sigma[0]; + + for (int i = 0; i < nsigma - 1 && isnan(Sigma); i++) { + if (!isfinite(Sout[i]) || !isfinite(Sout[i+1])) continue; + if ((Sout[i] > 0.65) && (Sout[i+1] > 0.65)) continue; + if ((Sout[i] < 0.65) && (Sout[i+1] < 0.65)) continue; + Sigma = sigma[i] + (0.65 - Sout[i])*(sigma[i+1] - sigma[i])/(Sout[i+1] - Sout[i]); + Nstars = 0.5*(Nout[i] + Nout[i+1]); + minKronRadius = Rmin[i] + (0.65 - Sout[i])*(Rmin[i+1] - Rmin[i])/(Sout[i+1] - Sout[i]); + } + psAssert (isfinite(Sigma), "did we miss a case?"); + if (!isfinite(minKronRadius)) { + minKronRadius = Sigma; + } + + // choose a grid scale that is a fixed fraction of the psf sigma^2 + psMetadataAddF32(analysis, PS_LIST_TAIL, "PSF_CLUMP_GRID_SCALE", PS_META_REPLACE, "clump grid", 0.1*PS_SQR(Sigma)); + psMetadataAddF32(analysis, PS_LIST_TAIL, "MOMENTS_SX_MAX", PS_META_REPLACE, "moments limit", 2.0*PS_SQR(Sigma)); + psMetadataAddF32(analysis, PS_LIST_TAIL, "MOMENTS_SY_MAX", PS_META_REPLACE, "moments limit", 2.0*PS_SQR(Sigma)); + psMetadataAddF32(analysis, PS_LIST_TAIL, "MOMENTS_GAUSS_SIGMA", PS_META_REPLACE, "moments limit", Sigma); + psMetadataAddF32(analysis, PS_LIST_TAIL, "PSF_MOMENTS_RADIUS", PS_META_REPLACE, "moments limit", 4.0*Sigma); + psMetadataAddF32(analysis, PS_LIST_TAIL, "PSF_CLUMP_NSTARS", PS_META_REPLACE, "number of stars in clump", Nstars); + psMetadataAddF32(analysis, PS_LIST_TAIL, "MOMENTS_MIN_KRON", PS_META_REPLACE, "minimum Kron Radius", minKronRadius); + + psLogMsg ("psphot", 3, "using window function with sigma = %f (Min Kron Radius = %f, Nstars = %d)\n", Sigma, minKronRadius, Nstars); + return true; +} + +// if we use the footprints, the output peaks list contains both old and new peaks, +// otherwise it only contains the new peaks. + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStack.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStack.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStack.c (revision 41083) @@ -0,0 +1,44 @@ +# include "psphotStandAlone.h" + +int main (int argc, char **argv) { + + // Set flag to tell certain library functions that we are in the psphotStack program. + // (This is defined in psphotDefineFiles.c) + psphotINpsphotStack = true; + + // uncomment to turn on memory dumps (move this to an option) + // psMemDumpSetState(true); + + psTimerStart ("complete"); + pmErrorRegister(); // register psModule's error codes/messages + psphotInit(); + + // load command-line arguments, options, and system config data + pmConfig *config = psphotStackArguments (argc, argv); + assert(config); + + psphotVersionPrint(); + + psMemDump("start"); + + // load input data (config and images (signal, noise, mask) + if (!psphotStackParseCamera (config)) { + psErrorStackPrint(stderr, "Error setting up the camera\n"); + exit (psphotGetExitStatus()); + } + + // call psphot for each readout + if (!psphotStackImageLoop (config)) { + psErrorStackPrint(stderr, "Error in the psphot image loop\n"); + exit (psphotGetExitStatus()); + } + + psLogMsg ("psphot", 3, "complete psphot run: %f sec\n", psTimerMark ("complete")); + + psErrorCode exit_status = psphotGetExitStatus(); + psphotCleanup (config); + exit (exit_status); +} + +// all functions which return to this level must raise one of the top-level error codes if they +// exit with an error. these error codes are used to specify the program exit status Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackAllocateOutput.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackAllocateOutput.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackAllocateOutput.c (revision 41083) @@ -0,0 +1,31 @@ +# include "psphotInternal.h" + +// allocate readout and images for PSPHOT.STACK.INPUT.CNV +// This is called only if we are skiping the psf matching +bool psphotStackAllocateOutput (const pmConfig *config, pmFPAview *view, psMetadata *recipe) { + bool status = false; + + bool useRaw = psMetadataLookupBool (&status, recipe, "PSPHOT.STACK.USE.RAW"); + char *fileruleSrc = useRaw ? "PSPHOT.STACK.INPUT.RAW" : "PSPHOT.STACK.INPUT.CNV"; + + int num = psphotFileruleCount(config, fileruleSrc); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + // find hte source readout + pmFPAfile *fileSrc = pmFPAfileSelectSingle(config->files, fileruleSrc, i); + psAssert (fileSrc, "missing file?"); + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.OUTPUT.IMAGE", i); + psAssert (fileOut, "missing output file?"); + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + if (readoutOut == NULL) { + readoutOut = pmFPAGenerateReadout(config, view, "PSPHOT.STACK.OUTPUT.IMAGE", fileSrc->fpa, NULL, i); + psAssert (readoutOut, "missing readout?"); + } + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackArguments.c (revision 41083) @@ -0,0 +1,199 @@ +# include "psphotStandAlone.h" + +static void dumpTemplate(void); +static void usage(pmConfig *config, int exitCode); +static void writeHelpInfo(FILE* ofile); + +pmConfig *psphotStackArguments(int argc, char **argv) { + + int N; + + // print help info + if (psArgumentGet(argc, argv, "-help")) writeHelpInfo(stdout); + if (psArgumentGet(argc, argv, "-h")) writeHelpInfo(stdout); + + if (psArgumentGet(argc, argv, "-template")) dumpTemplate(); + + // load config data from default locations + pmConfig *config = pmConfigRead(&argc, argv, PSPHOT_RECIPE); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't read site configuration"); + exit(PS_EXIT_CONFIG_ERROR); + } + + // generic arguments (version, dumpconfig) + PS_ARGUMENTS_GENERIC( psphot, config, argc, argv ); + + // thread arguments + PS_ARGUMENTS_THREADS( psphot, config, argc, argv ) + + // save the following additional recipe values based on command-line options + // these options override the PSPHOT recipe values loaded from recipe files + psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE); + + bool updateMode = false; + if ((N = psArgumentGet (argc, argv, "-updatemode"))) { + psArgumentRemove (N, &argc, argv); + updateMode = true; + } + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.UPDATEMODE", 0, "update mode flag", updateMode); + + // visual : interactive display mode + if ((N = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (N, &argc, argv); + pmVisualSetVisual(true); + } + + // memdump : enable memory spot checks + if ((N = psArgumentGet (argc, argv, "-memdump"))) { + psArgumentRemove (N, &argc, argv); + psMemDumpSetState(true); + } + + // break : used from recipe throughout psphotReadout to stop processing early + if ((N = psArgumentGet (argc, argv, "-break"))) { + if (argc <= N+1) { + psErrorStackPrint(stderr, "Expected to see an argument for -break"); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove (N, &argc, argv); + psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]); + psArgumentRemove (N, &argc, argv); + } + if ((N = psArgumentGet (argc, argv, "-ds9regions"))) { + psArgumentRemove (N, &argc, argv); + pmSubtractionRegions(true); + } + + // find the input data file (an mdc file) + if ((N = psArgumentGet(argc, argv, "-input"))) { + if (argc <= N+1) { + psErrorStackPrint(stderr, "Expected to see 1 more argument; saw %d", argc - 1); + exit(PS_EXIT_CONFIG_ERROR); + } + psArgumentRemove(N, &argc, argv); + + unsigned int numBad = 0; // Number of bad lines + psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[N], false); // Input file info + if (!inputs || numBad > 0) { + psErrorStackPrint(stderr, "Unable to cleanly read MDC file with inputs."); + exit(PS_EXIT_CONFIG_ERROR); + } + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "INPUTS", 0, "Metadata with input details", inputs); + psFree(inputs); + + psArgumentRemove(N, &argc, argv); + } + + if (argc != 2) { + psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1); + usage(config, PS_EXIT_CONFIG_ERROR); + } + + // output position is fixed + psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]); + + psTrace("psphot", 1, "Done with psphotArguments...\n"); + return (config); +} + +static void writeHelpInfo(FILE* ofile) +{ + fprintf(ofile, + "Usage: psphotStack -input (INPUTS.mdc) (OUTROOT)\n" + "\n" + "where INPUTS.mdc contains various METADATAs, each with:\n" + "\tIMAGE : Image filename\n" + "\tMASK : Mask filename\n" + "\tVARIANCE : Variance map filename\n" + "(use -template to generate a sample input.mdc file)\n" + "OUTROOT is the 'root name' for output files\n" + "\n" + "additional options:\n" + " -chip nn[,nn,...]\n" + " select detector chips to process; default is all.\n" + " Indices correspond to zero-based offset in the FPA metadata table.\n" + " -photcode PhotoCodeName\n" + " specify photocode\n" + " -region RegionString\n" + " specify analysis region. String is of form '[x0:x1,y0:y1]'\n" + " To use this option you must define a default in psphot.config\n" + " -visual\n" + " turns on interactive display mode\n" + " -dumpconfig CfgFileName\n" + " causes config info to be dumped to the named file.\n" + " -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n" + " choose a point at which to exit processing early\n" + " -nthreads n\n" + " set number of parallel threads of execution\n" + " -F OldFileRule ReplacementFileRule\n" + " change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n" + " -D name stringval\n" + " set a string-valued config parameter\n" + " -Di name intval\n" + " set an integer-valued config parameter\n" + " -Df name fval\n" + " set a float-valued config parameter\n" + " -Db name boolval\n" + " set a boolean-valued config parameter\n" + " -v, -vv, -vvv\n" + " set increasing levels of verbosity\n" + " -logfmt FormatString\n" + " set format string used for log messages\n" + " -trace Fac Lvl\n" + " set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n" + " -trace-levels\n" + " print current trace levels\n"); + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} + +static void usage(pmConfig *config, // Configuration + int exitCode + ) +{ + fprintf(stderr, + "Usage: psphotStack -input input.mdc outroot\n" + "Try 'psphotStack -help' for more options and explanation\n"); + + if (exitCode != PS_EXIT_SUCCESS) psErrorStackPrint(stderr, "Error reading arguments\n"); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(exitCode); +} + +static void dumpTemplate(void) { + + fprintf (stdout, "# this line is required for multiple INPUT blocks to be accepted\n"); + fprintf (stdout, "INPUT MULTI\n\n"); + + fprintf (stdout, "# copy and repeat the following block as needed (one per input image set)\n"); + fprintf (stdout, "# RAW (unconvolved) and CNV (convolved) input images are required\n"); + fprintf (stdout, "# RAW is used for detection and by default for measurements,\n"); + fprintf (stdout, "# CNV is convolved (further) to match target PSF and is used for radial aperture measurements\n"); + fprintf (stdout, "# and if PSPHOT.STACK.USE.RAW = F for measurements\n"); + fprintf (stdout, "# if MASK or VARIANCE images are not supplied, they will be generated\n"); + fprintf (stdout, "# EXPNUM images are optional. If supplied they are uesd to set the value of NFRAMES\n"); + fprintf (stdout, "# PSF may be supplied for the convolution target\n"); + fprintf (stdout, "INPUT METADATA\n"); + fprintf (stdout, " STACK_ID S64 123456789 # stack_id for the input image\n"); + fprintf (stdout, " RAW:IMAGE STR file.im.fits # signal image filename\n"); + fprintf (stdout, " RAW:MASK STR file.mk.fits # mask image filename\n"); + fprintf (stdout, " RAW:VARIANCE STR file.wt.fits # variance image filename\n"); + fprintf (stdout, " RAW:EXPNUM STR file.num.fits # exposure number image filename\n"); + fprintf (stdout, " RAW:PSF STR file.psf.fits # psf from input unconvolved image\n"); + + fprintf (stdout, " CNV:IMAGE STR file.im.fits # signal image filename\n"); + fprintf (stdout, " CNV:MASK STR file.mk.fits # mask image filename\n"); + fprintf (stdout, " CNV:VARIANCE STR file.wt.fits # variance image filename\n"); + fprintf (stdout, " CNV:EXPNUM STR file.num.fits # exposure number image filename\n"); + fprintf (stdout, " CNV:PSF STR file.psf.fits # psf from input convolved image\n"); + + fprintf (stdout, " SOURCES STR file.cmf # measured source positions\n"); + fprintf (stdout, "END\n"); + + psLibFinalize(); + exit(PS_EXIT_SUCCESS); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackChisqImage.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackChisqImage.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackChisqImage.c (revision 41083) @@ -0,0 +1,190 @@ +# include "psphotInternal.h" + +// Chi = sqrt(\sum_i im_i^2/var_i) + +static int npass = 0; + +// XXX supply filename or keep PSPHOT.INPUT fixed? +bool psphotStackChisqImage (pmConfig *config, const pmFPAview *view, const char *ruleDet, const char *ruleSrc) +{ + psTimerStart ("psphot.chisq.image"); + + pmFPAfile *chisqFile = pmFPAfileSelectSingle(config->files, "PSPHOT.CHISQ.IMAGE", 0); + psAssert (chisqFile, "missing chisq image FPA?"); + + // the readout containing the chisq image is generated in the first pass of this loop and + // used by the successive passes + pmReadout *chiReadout = NULL; + + int num = psphotFileruleCount(config, "PSPHOT.INPUT"); + + // loop over the available readouts + // generate the chisq image from the 'detection' images + for (int i = 0; i < num; i++) { + if (!psphotStackChisqImageAddReadout(config, view, ruleDet, &chiReadout, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to model background for %s entry %d", ruleDet, i); + return false; + } + } + + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "PSPHOT.CHISQ.NUM", PS_META_REPLACE, "", num); + + // we need to increment the counter for ruleDet and ruleSrc: + num++; + + // save the resulting image in the 'detection' set + if (!psMetadataAddPtr(config->files, PS_LIST_TAIL, ruleDet, PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "", chisqFile)) { + psError(PM_ERR_CONFIG, false, "could not add chisqFPA to config files"); + return false; + } + psphotFileruleCountSet(config, ruleDet, num); + + // also save the resulting image in the 'source' set (analysis set) + if (strcmp(ruleDet, ruleSrc)) { + if (!psMetadataAddPtr(config->files, PS_LIST_TAIL, ruleSrc, PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "", chisqFile)) { + psError(PM_ERR_CONFIG, false, "could not add chisqFPA to config files"); + return false; + } + psphotFileruleCountSet(config, ruleSrc, num); + } + + psLogMsg ("psphot", PS_LOG_INFO, "built chisq image: %f sec\n", psTimerMark ("psphot.chisq.image")); + + return true; +} + +bool psphotStackChisqImageAddReadout(const pmConfig *config, // Configuration + const pmFPAview *view, + const char *filerule, + pmReadout **chiReadout, + int index) +{ + bool status = true; + + // find the currently selected readout + pmFPAfile *input = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (input, "missing file?"); + + pmReadout *inReadout = pmFPAviewThisReadout(view, input->fpa); + psAssert (inReadout, "missing readout?"); + + psImage *inImage = inReadout->image; + psAssert (inImage, "missing image?"); + + psImage *inVariance = inReadout->variance; + psAssert (inVariance, "missing variance?"); + + psImage *inMask = inReadout->mask; + psAssert (inMask, "missing mask?"); + + if (*chiReadout == NULL) { + *chiReadout = pmFPAGenerateReadout(config, view, "PSPHOT.CHISQ.IMAGE", input->fpa, NULL, 0); + } + + psImage *chiImage = (*chiReadout)->image; + psAssert (chiImage, "missing chi image"); + + psImage *chiVariance = (*chiReadout)->variance; + psAssert (chiVariance, "missing chi variance"); + + psImage *chiMask = (*chiReadout)->mask; + psAssert (chiMask, "missing chi mask"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // generate the chisq image: + for (int iy = 0; iy < inImage->numRows; iy++) { + for (int ix = 0; ix < inImage->numCols; ix++) { + if (inMask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal) continue; + // XXX TEST chiImage->data.F32[iy][ix] += PS_SQR(inImage->data.F32[iy][ix]) / inVariance->data.F32[iy][ix]; + chiImage->data.F32[iy][ix] = 0.0; + chiVariance->data.F32[iy][ix] = 1.0; // ?? what is the right value? just init to this? + // XXX TEST chiMask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] = 0x00; // we have valid data so unmask this pixel + chiMask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] = 0x01; // we have valid data so unmask this pixel + } + } + + if (psTraceGetLevel("psphot") > 5) { + char name[256]; + sprintf (name, "chiImage.%02d.fits", npass); + psphotSaveImage (NULL, chiImage, name); + } + npass ++; + + return true; +} + +bool psphotStackRemoveChisqFromInputs (pmConfig *config, const char *filerule) { + + bool status = false; + + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + psAssert (status, "programming error: must define PSPHOT.CHISQ.NUM"); + + int inputNum = psphotFileruleCount(config, filerule); + + pmFPAfileRemoveSingle (config->files, filerule, chisqNum); + + inputNum --; + psphotFileruleCountSet(config, filerule, inputNum); + + return true; +} + +bool pmFPAfileRemoveSingle(psMetadata *files, const char *name, int num) +{ + PS_ASSERT_PTR_NON_NULL(files, NULL); + PS_ASSERT_INT_NONNEGATIVE(num, NULL); + + psList* mdList = files->list; + psHash* mdHash = files->hash; + + // Generate a REGEX to select only items that match 'name' + psString regex = NULL; // Regular expression + if (name) { + if (!psMetadataLookup(files, name)) { + // No files match the requested name + return false; + } + psStringAppend(®ex, "^%s$", name); + } + + psMetadataIterator *iter = psMetadataIteratorAlloc(files, PS_LIST_HEAD, regex); // Iterator + psFree(regex); + psMetadataItem *item; // Item from iteration + + bool found = false; + int i = 0; // Counter + for (i = 0; !found && (item = psMetadataGetAndIncrement(iter)); i++) { + if (i == num) found = true; + } + psFree(iter); + if (!found) { + return false; + } + + char *key = item->name; + + // look up the name via hash to see if we have a multi or not + psMetadataItem* hashItem = psHashLookup(mdHash, name); + if (hashItem == NULL) { + psError(PS_ERR_UNKNOWN, false, _("Failed to remove metadata item, %s, from metadata table."), key); + return false; + } + + if (hashItem->type == PS_DATA_METADATA_MULTI) { + // multiple entries with same key, remove just the specified one + psListRemoveData(hashItem->data.list, item); + } else { + psHashRemove(mdHash, key); + } + psListRemoveData(mdList, item); + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackImageLoop.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackImageLoop.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackImageLoop.c (revision 41083) @@ -0,0 +1,242 @@ +# include "psphotStandAlone.h" + +# define ESCAPE(MESSAGE) { \ + psError(PSPHOT_ERR_DATA, false, MESSAGE); \ + psFree (view); \ + return false; \ + } + +// XXX this implementation is not smart about multi-level astrometry headers +bool UpdateHeadersForFPA (pmConfig *config, pmFPAview *view); +bool UpdateHeadersForChip (pmConfig *config, pmFPAview *view); +bool UpdateHeadersForReadout (pmConfig *config, pmFPAview *view); + +bool psphotStackImageLoop (pmConfig *config) { + + bool status; + pmChip *chip; + pmCell *cell; + pmReadout *readout; + + psMemDump("startloop"); + + pmFPAview *view = pmFPAviewAlloc (0); + pmFPAfile *input = psMetadataLookupPtr (&status, config->files, "PSPHOT.STACK.INPUT.RAW"); + + if (!input) { + psError(PSPHOT_ERR_PROG, false, "Can't find input data!"); + return false; + } + + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + bool updateMode = psMetadataLookupBool(&status, config->arguments, "PSPHOT.STACK.UPDATEMODE"); + + // just load the full set of images up front except for EXPNUM which we defer + pmFPAfileActivate (config->files, false, "PSPHOT.STACK.EXPNUM.RAW"); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa in psphot."); + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip in psphotStack."); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Cell in psphotStack."); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 6, "Readout %d: %x %x\n", view->readout, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + psMemDump("load"); + + if (!psphotStackAllocateOutput (config, view, recipe)) { + psError(psErrorCodeLast(), false, "failure in psphotStackAllocateOutput for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + psMemDump("stackmatch"); + + // XXX for now, we assume there is only a single chip in the PHU: + if (!updateMode) { + if (!psphotStackReadout (config, view)) { + psError(psErrorCodeLast(), false, "failure in psphotStackReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + } else { + if (!psphotStackUpdateReadout (config, view)) { + psError(psErrorCodeLast(), false, "failure in psphotStackReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout); + psFree (view); + return false; + } + } + + UpdateHeadersForReadout(config, view); + + psMemDump("psphot"); + } + // drop all versions of the internal files + status = true; + status &= pmFPAfileDropInternal (config->files, psphotGetFilerule("PSPHOT.BACKMDL")); + status &= pmFPAfileDropInternal (config->files, psphotGetFilerule("PSPHOT.BACKMDL.STDEV")); + status &= pmFPAfileDropInternal (config->files, psphotGetFilerule("PSPHOT.BACKGND")); + if (!status) { + psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files"); + psFree (view); + return false; + } + } + + UpdateHeadersForChip(config, view); + + // Defer output until we have performed psphotSetNFrames() + pmFPAfileActivate (config->files, false, NULL); + + // Iterate up + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed pmFPAfileIOChecks for Chip in psphot."); + } + psMemDump("doneloop"); + + UpdateHeadersForFPA(config, view); + + // Iterate up output which is saved at the fpa level + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed ouput pmFPAfileIOChecks FPA in psphot."); + + // Load the appropriate EXPNUM image + pmFPAfileActivate (config->files, true, "PSPHOT.STACK.EXPNUM.RAW"); + + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for fpa EXPNUM in psphot."); + + // for psphot, we force data to be read at the chip level + while ((chip = pmFPAviewNextChip (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (! chip->process || ! chip->file_exists) { continue; } + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Chip EXPNUM in psphotStack."); + + // there is now only a single chip (multiple readouts?). loop over it and process + while ((cell = pmFPAviewNextCell (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE ("failed input for Cell in psphotStack."); + + // process each of the readouts + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + psLogMsg ("psphot", 6, "Readout %d: %x %x\n", view->readout, cell->file_exists, cell->process); + if (! readout->data_exists) { continue; } + + if (!psphotSetNFrames (config, view, input->name)) ESCAPE ("failed to setNFrames."); + } + } + // now activate all files to trigger final output + pmFPAfileActivate (config->files, true, NULL); + + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed output for Chip in psphot."); + } + if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE ("failed ouput for FPA in psphot."); + + // fail if we failed to handle an error + if (psErrorCodeLast() != PS_ERR_NONE) psAbort ("failed to handle an error!"); + + psFree (view); + + psMemDump("doneoutput"); + return true; +} + +bool UpdateHeadersForReadout (pmConfig *config, pmFPAview *view) { + + int num = psphotFileruleCount(config, "PSPHOT.INPUT"); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *output = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.OUTPUT.IMAGE", i); // File of interest + psAssert (output, "missing file?"); + + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.INPUT.RAW", i); // File of interest + psAssert (input, "missing input file"); + + // just copy the input headers to the output headers, then update version info + pmReadout *inReadout = pmFPAviewThisReadout(view, input->fpa); ///< Chip in the input + pmReadout *outReadout = pmFPAviewThisReadout(view, output->fpa); ///< Chip in the output + + psMetadata *header = psMetadataLookupPtr (NULL, inReadout->analysis, "PSPHOT.HEADER"); + psMetadataAdd (outReadout->analysis, PS_LIST_TAIL, "PSPHOT.HEADER", PS_DATA_METADATA, "header stats", header); + } + return true; +} + +bool UpdateHeadersForChip (pmConfig *config, pmFPAview *view) { + + int num = psphotFileruleCount(config, "PSPHOT.INPUT"); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *output = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.OUTPUT.IMAGE", i); // File of interest + psAssert (output, "missing file?"); + + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.INPUT.RAW", i); // File of interest + psAssert (input, "missing input file"); + + // just copy the input headers to the output headers, then update version info + pmChip *inChip = pmFPAviewThisChip(view, input->fpa); ///< Chip in the input + pmHDU *inHDU = pmFPAviewThisHDU (view, input->fpa); + + pmChip *outChip = pmFPAviewThisChip(view, output->fpa); ///< Chip in the output + pmHDU *outHDU = pmFPAviewThisHDU (view, output->fpa); + if (!outHDU) { + pmFPAAddSourceFromView(output->fpa, view, output->format); + outHDU = pmFPAviewThisHDU (view, output->fpa); + psAssert (outHDU, "failed to make HDU"); + } + if (!outHDU->header) { + outHDU->header = psMetadataCopy(NULL, inHDU->header); + } + psphotVersionHeaderFull(outHDU->header); + outChip->toFPA = psMemIncrRefCounter(inChip->toFPA); + outChip->fromFPA = psMemIncrRefCounter(inChip->fromFPA); + } + return true; +} + +bool UpdateHeadersForFPA (pmConfig *config, pmFPAview *view) { + + int num = psphotFileruleCount(config, "PSPHOT.INPUT"); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *output = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.OUTPUT.IMAGE", i); // File of interest + psAssert (output, "missing file?"); + + pmFPAfile *input = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.INPUT.RAW", i); // File of interest + psAssert (input, "missing input file"); + + output->fpa->toTPA = psMemIncrRefCounter(input->fpa->toTPA); + output->fpa->fromTPA = psMemIncrRefCounter(input->fpa->fromTPA); + output->fpa->toSky = psMemIncrRefCounter(input->fpa->toSky); + + pmConceptsCopyFPA(output->fpa, input->fpa, true, true); + + // XXX TEST + // pmFPASetFileStatus(output->fpa, true); + // pmFPASetDataStatus(output->fpa, true); + // pmChip *chip = output->fpa->chips->data[0]; + // pmCell *cell = chip->cells->data[0]; + // pmReadout *readout = cell->readouts->data[0]; + } + return true; +} + +/* + the easiest way to implement this is to assume we can pre-load the full set of images up front. + with 5 filters and 6000^2 (image, mask, var = 10 byte per pixel), we need 1.8GB, which is not too bad. +*/ + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFs.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFs.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFs.c (revision 41083) @@ -0,0 +1,145 @@ +# include "psphotInternal.h" + +bool psphotStackMatchPSFs (pmConfig *config, const pmFPAview *view) +{ + bool status = true; + + psLogMsg ("psphot", PS_LOG_INFO, "--- psphotStack Match PSFs ---"); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, "PSPHOT"); + psAssert(recipe, "We've thrown an error on this before."); + + int num = psphotFileruleCount(config, "PSPHOT.INPUT"); + + // 'options' carries info needed to perform the stack matching + psphotStackOptions *options = psphotStackOptionsAlloc(num); + + options->convolve = psMetadataLookupBool (&status, recipe, "PSPHOT.STACK.MATCH.PSF"); + psAssert (status, "PSPHOT.STACK.MATCH.PSF not in recipe"); + + if (options->convolve) { + char *convolveSource = psMetadataLookupStr (&status, recipe, "PSPHOT.STACK.MATCH.PSF.SOURCE"); + options->convolveSource = psphotStackConvolveSourceFromString (convolveSource); + if (options->convolveSource == PSPHOT_CNV_SRC_NONE) { + psError (PSPHOT_ERR_CONFIG, true, "stack convolution source not defined in recipe"); + return false; + } + } + + // loop over the available readouts + for (int i = 0; i < num; i++) { + // set up the PSF-matching parameters describing the input images + if (!psphotStackMatchPSFsPrepare (config, view, options, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to set PSF matching options for entry %d", i); + return false; + } + } + + // XXX convolve == false might not be valid at the moment + if (options->convolve) { + // Determine the 1st target PSF (either AUTO or defined by PSPHOT.STACK.TARGET.PSF.FWHM) + // NOTE: this also set the full list of target FWHMs (options->targetSeeing) + if (!psphotStackPSF(config, options)) { + psError(psErrorCodeLast(), false, "Unable to determine output PSF."); + return false; + } + } + + // loop over the available readouts (ignore chisq image) + for (int i = 0; i < num; i++) { + if (!psphotStackMatchPSFsReadout (config, view, options, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to find initial detections for PSPHOT.INPUT entry %d", i); + return false; + } + } + + psFree (options); + return true; +} + +// convolve the image to match desired PSF +// XXX is this code consistent with 'convolve' = false? XXX: No +bool psphotStackMatchPSFsReadout (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index) { + + psImageMaskType maskValue; + psImageMaskType markValue; + + psLogMsg("psphot", PS_LOG_DETAIL, "-- starting PSF matching for readout %d --", index); + + // get the PSPHOT.MASK value from the config + if (!pmConfigMaskSetBits (&maskValue, &markValue, config)) { + psError (PS_ERR_UNKNOWN, true, "Unable to define the mask bit values"); + return false; + } + + pmFPAfile *fileSrc = psphotStackGetConvolveSource(config, options, index); + if (!fileSrc) { + psError(PSPHOT_ERR_CONFIG, false, "desired convolution source is missing"); + return false; + } + + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.OUTPUT.IMAGE", index); // File of interest + psAssert (fileOut, "missing output file?"); + + pmReadout *readoutSrc = pmFPAviewThisReadout(view, fileSrc->fpa); + psAssert (readoutSrc, "missing readout?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + if (readoutOut == NULL) { + readoutOut = pmFPAGenerateReadout(config, view, "PSPHOT.STACK.OUTPUT.IMAGE", fileSrc->fpa, NULL, index); + psAssert (readoutOut, "missing readout?"); + } + + // set NAN pixels to 'SAT' + psImageMaskType maskSat = pmConfigMaskGet("SAT", config); + if (!pmReadoutMaskInvalid(readoutSrc, maskValue, maskSat)) { + psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels in readout."); + return false; + } + + // Image Matching (PSFs or just flux) + if (options->convolve) { + if (!matchKernel(config, readoutOut, readoutSrc, options, index)){ + psError(psErrorCodeLast(), false, "Unable to match image PSF in readout."); + return false; + } + saveMatchData(readoutOut, options, index); + } + bool notMatched = psMetadataLookupBool(NULL, readoutOut->analysis, "NOT.PSF.MATCHED"); + if (notMatched) { + psFree(readoutOut->image); + readoutOut->image = psImageCopy(NULL, readoutSrc->image, PS_TYPE_F32); + psFree(readoutOut->variance); + if (readoutSrc->variance) { + readoutOut->variance = psImageCopy(NULL, readoutSrc->variance, PS_TYPE_F32); + } + psFree(readoutOut->mask); + if (readoutSrc->mask) { + readoutOut->mask = psImageCopy(NULL, readoutSrc->mask, PS_TYPE_IMAGE_MASK); + } + } + + // renormalize the stack variances to have sigma = 1.0 + if (!psphotStackRenormaliseVariance(config, readoutOut)) { + psError(psErrorCodeLast(), false, "Unable to renormalise variance."); + return false; + } + + // save the output fwhm values in the readout->analysis. we may have / will have multiple output PSF sizes, + // so we save this in a vector. if the vector is not yet defined, create it + // Skip this if psf matching failed for this readout. For example if deconvolution fraction was over the limit. + // NOTE: fwhmValues as defined here has 1 + nMatched PSF : 0 == unmatched + psVector *fwhmValues = psVectorAllocEmpty(10, PS_TYPE_F32); + psVectorAppend(fwhmValues, NAN); // XXX this corresponds to the unmatched image set + + if (!notMatched) { + for (int i = 0; i < options->targetSeeing->n; i++) { + psVectorAppend(fwhmValues, options->targetSeeing->data.F32[i]); + } + } + psMetadataAddVector(readoutSrc->analysis, PS_LIST_TAIL, "STACK.PSF.FWHM.VALUES", PS_META_REPLACE, "PSF sizes", fwhmValues); + psMetadataAddVector(readoutOut->analysis, PS_LIST_TAIL, "STACK.PSF.FWHM.VALUES", PS_META_REPLACE, "PSF sizes", fwhmValues); + psFree(fwhmValues); // drops this function's reference + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsNext.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsNext.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsNext.c (revision 41083) @@ -0,0 +1,200 @@ +# include "psphotInternal.h" + +// NOTE : element 0 of fwhmValues if the unmatched image, + +int psphotStackMatchPSFsEntries (pmConfig *config, const pmFPAview *view, const char *filerule) { + + int nRadialEntries = 0; + + // find the numer of fwhmValues in the first non-skipped input + int num = psphotFileruleCount(config, filerule); + for (int i=0; ifiles, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + bool status = false; + if (!psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psVector *fwhmValues = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + if (fwhmValues) { + nRadialEntries = fwhmValues->n; + } else { + nRadialEntries = 1; + } + break; + } + } + return nRadialEntries; +} + +// smooth the input image to match the next target PSF +// this function assumes the image has already been smoothed to match the first value (array element 0), +// and that the smoothing can use a 1D Gaussian kernel of width sqrt(TARGET^2 - CURRENT^2) +// each subsequent call +bool psphotStackMatchPSFsNext(pmConfig *config, const pmFPAview *view, const char *filerule, int lastSize) +{ + int num = psphotFileruleCount(config, filerule); + + // smooth the image and variance map + bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading in psImageConvolve + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotStackMatchPSFsNextReadout (config, view, filerule, i, lastSize)) { + psLogMsg ("psphot", PS_LOG_INFO, "failed to smooth image %s (%d) to target PSF", filerule, i); + psImageConvolveSetThreads(oldThreads); + return false; + } + } + + psImageConvolveSetThreads(oldThreads); + return true; +} + +bool psphotStackMatchPSFsNextReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, int lastSize) { + + bool status = false; + + psTimerStart ("psphot.smooth"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + if (psMetadataLookupBool (&status, readout->analysis, "PSPHOT.SKIP.INPUT")) { + psLogMsg("psphot", PS_LOG_INFO, "skipping smooth %d to next psf", index); + return true; + } + + psLogMsg("psphot", PS_LOG_INFO, "smooth %d to next psf", index); + psphotVisualShowImage(readout); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "missing mask value?"); + psImageMaskType maskSat = psMetadataLookupImageMask(&status, recipe, "MASK.SAT"); // Mask value for bad pixels + psAssert (maskSat, "missing mask value?"); + + + float minGauss = psMetadataLookupF32(NULL, recipe, "PEAKS_MIN_GAUSS"); // Minimum valid fraction of kernel + if (!isfinite(minGauss)) { + psWarning("PEAKS_MIN_GAUSS is not set in recipe; using default value"); + minGauss = 0.5; + } + + psVector *fwhmValues = psMetadataLookupVector(&status, readout->analysis, "STACK.PSF.FWHM.VALUES"); + psAssert (fwhmValues, "need target PSFs"); + + if (lastSize + 1 >= fwhmValues->n) { + return true; + } + + float targetFWHM = fwhmValues->data.F32[lastSize + 1]; + float currentFWHM = fwhmValues->data.F32[lastSize]; + + if (targetFWHM <= currentFWHM) { + // psError (PSPHOT_ERR_CONFIG, true, "target FWHM cannot be smaller than current FWHM"); + psLogMsg ("psphot", PS_LOG_INFO, "target FWHM (%f) is smaller than current FWHM (%f), not smoothing\n", targetFWHM, currentFWHM); + fwhmValues->data.F32[lastSize + 1] = currentFWHM; + pmReadoutMaskInvalid(readout, maskVal, maskSat); + return false; + } + + float smoothFWHM = sqrt(PS_SQR(targetFWHM) - PS_SQR(currentFWHM)); + float SIGMA_SMTH = smoothFWHM / (2.0*sqrt(2.0*log(2.0))); + float NSIGMA_SMTH = 3.0; + + // record the actual smoothing sigma + // psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SIGMA_SMOOTH", PS_META_REPLACE, "Smoothing sigma for detections", SIGMA_SMTH); + + // smooth the image, applying the mask as we go + psImageSmoothMask_Threaded(readout->image, readout->image, readout->mask, maskVal, SIGMA_SMTH, NSIGMA_SMTH, minGauss); + psLogMsg("psphot", PS_LOG_MINUTIA, "smooth image: %f sec\n", psTimerMark("psphot.smooth")); + + // Smooth the variance, applying the mask as we go. The variance is smoothed by the PSF^2, + // renomalized to maintain the input level of the variance. We achieve this by smoothing + // with a Gaussian with sigma = SIGMA_SMTH/sqrt(2) with unity normalization. Note that + // this process yields a smoothed image with correlated errors. The pixel-to-pixel + // variations in smooth_im will be decreased by a factor of 4*pi*SIGMA_SMTH^2, but for + // measurements based on apertures comparable to or larger than the smoothing kernel, the + // effective per-pixel variance is maintained. + psImageSmoothMask_Threaded(readout->variance, readout->variance, readout->mask, maskVal, SIGMA_SMTH * M_SQRT1_2, NSIGMA_SMTH, minGauss); + psLogMsg("psphot", PS_LOG_MINUTIA, "smooth variance: %f sec\n", psTimerMark("psphot.smooth")); + + // Insure that invalid pixels are masked + // XXX: the smoothing seems to generate nan pixels in the variance image + // XXX: We may need to loop over the cached sources and redefine the maskObj images... + pmReadoutMaskInvalid(readout, maskVal, maskSat); + { + // Now go rebuild the sources' copies of the mask + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + + if (sources && sources->n) { + for (int i = 0 ; i < sources->n; i++) { + // XXX: move this to a function in pmSource.c + pmSource *source = sources->data[i]; + if (source->maskObj && source->maskView) { + psFree(source->maskObj); + source->maskObj = psImageCopy (source->maskObj, source->maskView, PS_TYPE_IMAGE_MASK); + } + } + } + } + + psLogMsg("psphot", PS_LOG_INFO, "smoothed"); + psphotVisualShowImage(readout); + + // optionally save example images under trace + if (psTraceGetLevel("psphot") > 5) { + static int pass = 0; + char name[64]; + sprintf (name, "stksm.v%d.fits", pass); + psphotSaveImage(NULL, readout->image, name); + sprintf (name, "stkwt.v%d.fits", pass); + psphotSaveImage(NULL, readout->variance, name); + pass ++; + } + + // XXX need to apply this to the radial apertures somehow. + // Calculate correction factor for the covariance produced by the (potentially multiple) smoothing + // psKernel *kernel = psImageSmoothKernel(SIGMA_SMTH, NSIGMA_SMTH); // Kernel used for smoothing + // double sum2 = 0.0; // Sum of kernel squared + // for (int y = kernel->yMin; y <= kernel->yMax; y++) { + // for (int x = kernel->xMin; x <= kernel->xMax; x++) { + // sum2 += PS_SQR(kernel->kernel[y][x]); + // } + // } + // float factor = 1.0 / (sum2 * psImageCovarianceCalculateFactor(kernel, readout->covariance)); + // psFree(kernel); + + // record the effective area and significance scaling factor + float effArea = 8.0 * M_PI * PS_SQR(SIGMA_SMTH); + psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "EFFECTIVE_AREA", PS_META_REPLACE, "Effective Area", effArea); + // psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "SIGNIFICANCE_SCALE_FACTOR", PS_META_REPLACE, "Signicance scale factor", factor); + + // do not generate a PSF if we already were supplied one + pmPSF *psfOld = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + if (psfOld) { + // save PSF on readout->analysis + char psfEntry[64]; + snprintf (psfEntry, 64, "PSPHOT.PSF.V%d", lastSize); + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, psfEntry, PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot psf model", psfOld)) { + psError (PSPHOT_ERR_UNKNOWN, false, "problem saving sources on readout"); + return false; + } + psMetadataRemoveKey(readout->analysis, "PSPHOT.PSF"); + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsPrepare.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsPrepare.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsPrepare.c (revision 41083) @@ -0,0 +1,114 @@ +# include "psphotInternal.h" + +pmPSF *selectPSF (pmFPAfile *fileSrc, const pmFPAview *view, psphotStackOptions *options, int index); +bool determineSeeing (pmPSF *psf, psphotStackOptions *options, int index); + +// set up the PSF-matching parameters describing the input images +bool psphotStackMatchPSFsPrepare (pmConfig *config, const pmFPAview *view, psphotStackOptions *options, int index) { + + if (!options->convolve) { + psLogMsg ("psphotStack", PS_LOG_INFO, "psf-matching NOT selected"); + return true; + } + + pmFPAfile *fileSrc = psphotStackGetConvolveSource(config, options, index); + if (!fileSrc) { + psError(PSPHOT_ERR_CONFIG, false, "desired convolution source is missing"); + return false; + } + + pmPSF *psf = selectPSF (fileSrc, view, options, index); + if (!psf) { + psError(PSPHOT_ERR_PSF, false, "Problem with PSF."); + return false; + } + + determineSeeing (psf, options, index); + + // load the sources (used to find reference sources for the kernel stamps) + { + pmFPAfile *inputSrc = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.SOURCES", index); // File of interest + pmReadout *readout = pmFPAviewThisReadout(view, inputSrc->fpa); // Readout with sources + pmDetections *detections = psMetadataLookupPtr(NULL, readout->analysis, "PSPHOT.DETECTIONS"); // Sources + if (!detections || !detections->allSources) { + // An input must have sources. If we got here without any the input file was likely corrupt + psError(PSPHOT_ERR_IO, true, "Input %d has no sources.\n\tCheck all instances of: %s", index, inputSrc->origname); + return false; + } + psAssert (detections->allSources, "missing sources?"); + options->sourceLists->data[index] = psMemIncrRefCounter(detections->allSources); + } + + return true; +} + +// select the corresponding input psf +pmPSF *selectPSF (pmFPAfile *fileSrc, const pmFPAview *view, psphotStackOptions *options, int index) { + + bool status; + + pmChip *chip = pmFPAviewThisChip(view, fileSrc->fpa); // The chip holds the PSF + pmPSF *psf = psMetadataLookupPtr(&status, chip->analysis, "PSPHOT.PSF"); // PSF + if (!psf) { + // XXX if we were not supplied a PSF, we should be able to generate one by calling psphot + psError(PSPHOT_ERR_PROG, true, "Unable to find PSF."); + return NULL; + } + options->psfs->data[index] = psMemIncrRefCounter(psf); + + // find the image size + pmCell *cell = pmFPAviewThisCell(view, fileSrc->fpa); // Cell of interest + pmHDU *hdu = pmHDUFromCell(cell); + assert(hdu && hdu->header); + int naxis1 = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); // Number of columns + int naxis2 = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); // Number of rows + psAssert ((naxis1 > 0) && (naxis2 > 0), "Unable to determine size of image from PSF."); + if (!options->numCols) { + options->numCols = naxis1; + } else { + if (options->numCols != naxis1) { + psError (PSPHOT_ERR_CONFIG, true, "mismatched sizes (NAXIS1) for input images"); + return NULL; + } + } + if (!options->numRows) { + options->numRows = naxis2; + } else { + if (options->numRows != naxis2) { + psError (PSPHOT_ERR_CONFIG, true, "mismatched sizes (NAXIS2) for input images"); + return NULL; + } + } + return psf; +} + +// determine the input seeing +bool determineSeeing (pmPSF *psf, psphotStackOptions *options, int index) { + + // XXX set this based on the mode (+1 for poly, +0 for map) + float xNum = PS_MAX(psf->trendNx, 1); + float yNum = PS_MAX(psf->trendNy, 1); // Number of realisations + float sumFWHM = 0.0; // FWHM for image + int numFWHM = 0; // Number of FWHM measurements + for (float y = 0; y < yNum; y += 1.0) { + float yPos = options->numRows * ((y + 0.5) / yNum); + for (float x = 0; x < xNum; x++) { + float xPos = options->numCols * ((x + 0.5) / xNum); + float fwhm = pmPSFtoFWHM(psf, xPos, yPos); // FWHM for image + if (isfinite(fwhm)) { + sumFWHM += fwhm; + numFWHM++; + } + } + } + if (numFWHM == 0) { + options->inputSeeing->data.F32[index] = NAN; + options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[index] = 0x02; + psLogMsg("ppStack", PS_LOG_INFO, "Unable to measure PSF FWHM for image %d --- rejected.", index); + } else { + options->inputSeeing->data.F32[index] = sumFWHM / (float)numFWHM; + } + psLogMsg ("psphotStack", PS_LOG_INFO, "Input Seeing for %d: %f\n", index, options->inputSeeing->data.F32[index]); + return true; +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsUtils.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsUtils.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackMatchPSFsUtils.c (revision 41083) @@ -0,0 +1,511 @@ +# include "psphotInternal.h" +# define ARRAY_BUFFER 16 // Number to add to array at a time + +psVector *SetOptWidths (bool *optimum, psMetadata *recipe); +pmReadout *makeFakeReadout(pmConfig *config, pmReadout *raw, psArray *sources, pmPSF *psf, psImageMaskType maskVal, int fullSize); +bool saveMatchData (pmReadout *readout, psphotStackOptions *options, int index); +bool matchKernel(pmConfig *config, pmReadout *cnv, pmReadout *raw, psphotStackOptions *options, int index); +bool dumpImageDiff(pmReadout *readoutConv, pmReadout *readoutFake, pmReadout *readoutRef, int index, char *rootname); +bool dumpImage(pmReadout *readoutOut, pmReadout *readoutRef, int index, char *rootname); + +// Get coordinates from a source +void coordsFromSource(float *x, float *y, const pmSource *source) +{ + assert(x && y); + assert(source); + + if (source->modelPSF) { + *x = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + *y = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + } else { + *x = source->peak->xf; + *y = source->peak->yf; + } + return; +} + +# define SN_MIN 50.0 +psArray *stackSourcesFilter(psArray *sources, // Source list to filter + int exclusion // Exclusion zone, pixels + ) +{ + psAssert(sources && sources->n > 0, "Require array of sources"); + if (exclusion <= 0) { + return psMemIncrRefCounter(sources); + } + + int num = sources->n; // Number of sources + psVector *x = psVectorAlloc(num, PS_TYPE_F32), *y = psVectorAlloc(num, PS_TYPE_F32); // Coordinates + int numGood = 0; // Number of good sources + for (int i = 0; i < num; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (!source) { + continue; + } + if (!source->peak) continue; + if (sqrt(source->peak->detValue) < SN_MIN) continue; + coordsFromSource(&x->data.F32[numGood], &y->data.F32[numGood], source); + numGood++; + } + x->n = y->n = numGood; + + psTree *tree = psTreePlant(2, 2, PS_TREE_EUCLIDEAN, x, y); // kd tree + + psArray *filtered = psArrayAllocEmpty(numGood); // Filtered list of sources + psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of source + int numFiltered = 0; // Number of filtered sources + for (int i = 0; i < num; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (!source) { + continue; + } + if (!source->peak) continue; + if (sqrt(source->peak->detValue) < SN_MIN) continue; + float xSource, ySource; // Coordinates of source + coordsFromSource(&xSource, &ySource, source); + + coords->data.F64[0] = xSource; + coords->data.F64[1] = ySource; + + long numWithin = psTreeWithin(tree, coords, exclusion); // Number within exclusion zone + psTrace("psphotStack", 9, "Source at %.0lf,%.0lf has %ld sources in exclusion zone", + coords->data.F64[0], coords->data.F64[1], numWithin); + if (numWithin == 1) { + // Only itself inside the exclusion zone + filtered = psArrayAdd(filtered, filtered->n, source); + } else { + numFiltered++; + } + } + psFree(coords); + psFree(tree); + psFree(x); + psFree(y); + + psLogMsg("psphotStack", PS_LOG_INFO, "Filtered out %d of %d sources", numFiltered, numGood); + + return filtered; +} + +// Add background into the fake image +// Based on ppSubBackground() +psImage *stackBackgroundModel(pmReadout *ro, // Readout for which to generate background model + const pmConfig *config // Configuration + ) +{ + psAssert(ro && ro->image, "Need readout image"); + psAssert(config, "Need configuration"); + + psImage *image = ro->image; // Image of interest + int numCols = image->numCols, numRows = image->numRows; // Size of image + + psMetadata *ppStackRecipe = psMetadataLookupPtr(NULL, config->recipes, "PPSTACK"); + psAssert(ppStackRecipe, "Need PPSTACK recipe"); + psMetadata *psphotRecipe = psMetadataLookupPtr(NULL, config->recipes, "PSPHOT"); + psAssert(psphotRecipe, "Need PSPHOT recipe"); + +#if (0) + // DON'T OVERWRITE PSPHOT recipe value + psString maskBadStr = psMetadataLookupStr(NULL, ppStackRecipe, "MASK.BAD");// Name of bits to mask for bad + psMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psMetadataAddImageMask(psphotRecipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "user-defined mask", maskBad); +#endif + + psImage *binned = psphotModelBackgroundReadoutNoFile(ro, config); // Binned background model + psImageBinning *binning = psMetadataLookupPtr(NULL, ro->analysis, + "PSPHOT.BACKGROUND.BINNING"); // Binning for model + psAssert(binning, "Need binning parameters"); + psImage *unbinned = psImageAlloc(numCols, numRows, PS_TYPE_F32); // Unbinned background model + if (!psImageUnbin(unbinned, binned, binning)) { + psError(PSPHOT_ERR_DATA, false, "Unable to unbin background model"); + psFree(binned); + psFree(unbinned); + return NULL; + } + psFree(binned); + + return unbinned; +} + +// Renormalise a readout's variance map +bool psphotStackRenormaliseVariance(const pmConfig *config, // Configuration + pmReadout *readout // Readout to renormalise + ) +{ + bool mdok; // Status of metadata lookups + + psMetadata *recipe = psMetadataLookupPtr(NULL, config->recipes, "PPSTACK"); // Recipe for ppStack + psAssert(recipe, "Need PPSTACK recipe"); + + if (!psMetadataLookupBool(&mdok, recipe, "RENORM")) return true; + + int num = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); + if (!mdok) { + psError(PSPHOT_ERR_CONFIG, true, "RENORM.NUM is not set in the recipe"); + return false; + } + float minValid = psMetadataLookupF32(&mdok, recipe, "RENORM.MIN"); + if (!mdok) { + psError(PSPHOT_ERR_CONFIG, true, "RENORM.MIN is not set in the recipe"); + return false; + } + float maxValid = psMetadataLookupF32(&mdok, recipe, "RENORM.MAX"); + if (!mdok) { + psError(PSPHOT_ERR_CONFIG, true, "RENORM.MAX is not set in the recipe"); + return false; + } + + psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask + + psImageCovarianceTransfer(readout->variance, readout->covariance); + return pmReadoutVarianceRenormalise(readout, maskBad, num, minValid, maxValid); +} + +bool dumpImage(pmReadout *readoutOut, pmReadout *readoutRef, int index, char *rootname) { + + pmHDU *hdu = pmHDUFromCell(readoutRef->parent); + psString name = NULL; + psStringAppend(&name, "%s_%03d.fits", rootname, index); + pmStackVisualPlotTestImage(readoutOut->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, hdu->header, readoutOut->image, 0, NULL); + psFitsClose(fits); + return true; +} + +bool dumpImageDiff(pmReadout *readoutConv, pmReadout *readoutFake, pmReadout *readoutRef, int index, char *rootname) { + + pmHDU *hdu = pmHDUFromCell(readoutRef->parent); + psString name = NULL; + psStringAppend(&name, "%s_%03d.fits", rootname, index); + pmStackVisualPlotTestImage(readoutFake->image, name); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psBinaryOp(readoutFake->image, readoutConv->image, "-", readoutFake->image); + psFitsWriteImage(fits, hdu->header, readoutFake->image, 0, NULL); + psFitsClose(fits); + return true; +} + +// perform the bulk of the PSF-matching +bool matchKernel(pmConfig *config, pmReadout *readoutOut, pmReadout *readoutSrc, psphotStackOptions *options, int index) { + + bool mdok; + + psAssert(options->psf, "Require target PSF"); + psAssert(options->sourceLists && options->sourceLists->data[index], "Require source list"); + + psMetadata *stackRecipe = psMetadataLookupMetadata(NULL, config->recipes, "PPSTACK"); // ppStack recipe + psAssert(stackRecipe, "We've thrown an error on this before."); + + // Look up appropriate values from the ppSub recipe + psMetadata *subRecipe = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + psAssert(subRecipe, "recipe missing"); + + psString maskValStr = psMetadataLookupStr(NULL, subRecipe, "MASK.VAL"); // Name of bits to mask going in + psString maskPoorStr = psMetadataLookupStr(NULL, stackRecipe, "MASK.POOR"); // Name of bits to mask for poor + psString maskBadStr = psMetadataLookupStr(NULL, stackRecipe, "MASK.BAD"); // Name of bits to mask for bad + + psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Bits to mask going in to pmSubtractionMatch + psImageMaskType maskPoor = pmConfigMaskGet(maskPoorStr, config); // Bits to mask for poor pixels + psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels + + float penalty = psMetadataLookupF32(NULL, subRecipe, "PENALTY"); // Penalty for wideness + int threads = psMetadataLookupS32(NULL, config->arguments, "NTHREADS"); // Number of threads + + int order = psMetadataLookupS32(NULL, subRecipe, "SPATIAL.ORDER"); // Spatial polynomial order + float regionSize = psMetadataLookupF32(NULL, subRecipe, "REGION.SIZE"); // Size of iso-kernel regs + float spacing = psMetadataLookupF32(NULL, subRecipe, "STAMP.SPACING"); // Typical stamp spacing + + int footprint = psMetadataLookupS32(NULL, subRecipe, "STAMP.FOOTPRINT"); // Stamp half-size + int size = psMetadataLookupS32(NULL, subRecipe, "KERNEL.SIZE"); // Kernel half-size + + float threshold = psMetadataLookupF32(NULL, subRecipe, "STAMP.THRESHOLD"); // Threshold for stmps + int stride = psMetadataLookupS32(NULL, subRecipe, "STRIDE"); // Size of convolution patches + int iter = psMetadataLookupS32(NULL, subRecipe, "ITER"); // Rejection iterations + float rej = psMetadataLookupF32(NULL, subRecipe, "REJ"); // Rejection threshold + float kernelError = psMetadataLookupF32(NULL, subRecipe, "KERNEL.ERR"); // Relative systematic error in kernel + float normFrac = psMetadataLookupF32(NULL, subRecipe, "NORM.FRAC"); // Fraction of window for normalisn windw + float sysError = psMetadataLookupF32(NULL, subRecipe, "SYS.ERR"); // Relative systematic error in images + float skyErr = psMetadataLookupF32(NULL, subRecipe, "SKY.ERR"); // Additional error in sky + float covarFrac = psMetadataLookupF32(NULL, subRecipe, "COVAR.FRAC"); // Fraction for covariance calculation + + const char *typeStr = psMetadataLookupStr(NULL, subRecipe, "KERNEL.TYPE"); // Kernel type + pmSubtractionKernelsType type = pmSubtractionKernelsTypeFromString(typeStr); // Kernel type + psVector *widths = psMetadataLookupPtr(NULL, subRecipe, "ISIS.WIDTHS"); // ISIS Gaussian widths + psVector *orders = psMetadataLookupPtr(NULL, subRecipe, "ISIS.ORDERS"); // ISIS Polynomial orders + int inner = psMetadataLookupS32(NULL, subRecipe, "INNER"); // Inner radius + int ringsOrder = psMetadataLookupS32(NULL, subRecipe, "RINGS.ORDER"); // RINGS polynomial order + int binning = psMetadataLookupS32(NULL, subRecipe, "SPAM.BINNING"); // Binning for SPAM kernel + float badFrac = psMetadataLookupF32(NULL, subRecipe, "BADFRAC"); // Maximum bad fraction + float optThresh = psMetadataLookupF32(&mdok, subRecipe, "OPTIMUM.TOL"); // Tolerance for search + int optOrder = psMetadataLookupS32(&mdok, subRecipe, "OPTIMUM.ORDER"); // Order for search + float poorFrac = psMetadataLookupF32(&mdok, subRecipe, "POOR.FRACTION"); // Fraction for "poor" + + bool scale = psMetadataLookupBool(NULL, subRecipe, "SCALE"); // Scale kernel parameters? + float scaleRef = psMetadataLookupF32(NULL, subRecipe, "SCALE.REF"); // Reference for scaling + float scaleMin = psMetadataLookupF32(NULL, subRecipe, "SCALE.MIN"); // Minimum for scaling + float scaleMax = psMetadataLookupF32(NULL, subRecipe, "SCALE.MAX"); // Maximum for scaling + if (!isfinite(scaleRef) || !isfinite(scaleMin) || !isfinite(scaleMax)) { + psError(PSPHOT_ERR_CONFIG, false, + "Scale parameters (SCALE.REF=%f, SCALE.MIN=%f, SCALE.MAX=%f) not set in PPSUB recipe.", + scaleRef, scaleMin, scaleMax); + return false; + } + + // These values are specified specifically for stacking + const char *stampsName = psMetadataLookupStr(&mdok, config->arguments, "STAMPS");// Stamps filename + + psVector *widthsCopy = NULL; + psVector *optWidths = NULL; + pmReadout *fake = NULL; + psArray *stampSources = NULL; + + bool optimum = false; + optWidths = SetOptWidths(&optimum, subRecipe); // Vector with FWHMs for optimum search + + // For the sake of stamps, remove nearby sources + stampSources = stackSourcesFilter(options->sourceLists->data[index], footprint); // Filtered list of sources + + fake = makeFakeReadout(config, readoutSrc, stampSources, options->psf, maskVal | maskBad, footprint + size); + if (!fake) goto escape; + + // dumpImage(fake, readoutSrc, index, "fake"); + // dumpImage(readoutSrc, readoutSrc, index, "real"); + + if (threads) pmSubtractionThreadsInit(); + + // Do the image matching + bool rejectReadout = false; + pmSubtractionKernels *kernel = psMetadataLookupPtr(&mdok, readoutSrc->analysis, PM_SUBTRACTION_ANALYSIS_KERNEL); // Conv kernel + if (kernel) { + if (!pmSubtractionMatchPrecalc(NULL, readoutOut, fake, readoutSrc, readoutSrc->analysis, stride, kernelError, covarFrac, maskVal, maskBad, maskPoor, poorFrac, badFrac)) { + psError(psErrorCodeLast(), false, "Unable to convolve images."); + goto escape; + } + } else { + // Scale the input parameters + widthsCopy = psVectorCopy(NULL, widths, PS_TYPE_F32); // Copy of kernel widths + + // we need to register the FWHM values for use by pmSubtraction code + pmSubtractionSetFWHMs(options->targetSeeing->data.F32[0], options->inputSeeing->data.F32[index]); + + pmSubtractionParamScaleOptions(scale, scaleRef, scaleMin, scaleMax); + + // if (scale && !pmSubtractionParamsScale(&size, &footprint, widthsCopy, scaleRef, scaleMin, scaleMax)) { + // psError(psErrorCodeLast(), false, "Unable to scale kernel parameters"); + // goto escape; + // } + + if (!pmSubtractionMatch(NULL, readoutOut, fake, readoutSrc, footprint, stride, regionSize, spacing, threshold, stampSources, stampsName, type, size, order, widthsCopy, orders, inner, ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter, rej, normFrac, sysError, skyErr, kernelError, covarFrac, maskVal, maskBad, maskPoor, poorFrac, badFrac, PM_SUBTRACTION_MODE_2)) { + int errorCode = psErrorCodeLast(); + if (errorCode == PM_ERR_SMALL_AREA || errorCode == PM_ERR_STAMPS) { + // failed to match but, don't fault. Just drop this input from measurements that need the + // matched readout. + psErrorClear(); + rejectReadout = true; + if (errorCode == PM_ERR_SMALL_AREA) { + psLogMsg("psphot", PS_LOG_WARN, "PSF match failed for input %d. Too few pixels.", index); + } else if (errorCode == PM_ERR_STAMPS) { + psLogMsg("psphot", PS_LOG_WARN, "PSF match failed for input %d. Failed to find stamps.", index); + } + } else { + psError(psErrorCodeLast(), false, "Unable to match images."); + goto escape; + } + } + } + + // If the maximum deconvolution fraction exceeds the limit, reject this input for analyses that require PSF matching + float deconvLimit = psMetadataLookupF32(NULL, stackRecipe, "DECONV.LIMIT"); // Limit on deconvolution fraction + float deconv = psMetadataLookupF32(NULL, readoutOut->analysis, PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Max deconvolution fraction + if (deconv > deconvLimit) { + rejectReadout = true; + psLogMsg("psphot", PS_LOG_WARN, "Maximum deconvolution fraction (%f) exceeds limit (%f) --- rejecting input %d for matched psf analysis\n", deconv, deconvLimit, index); + } + + if (rejectReadout) { + psMetadataAddBool(readoutSrc->analysis, PS_LIST_TAIL, "NOT.PSF.MATCHED", PS_META_REPLACE, "", true); + psMetadataAddBool(readoutOut->analysis, PS_LIST_TAIL, "NOT.PSF.MATCHED", PS_META_REPLACE, "", true); + } else { + psMetadataAddBool(readoutSrc->analysis, PS_LIST_TAIL, "NOT.PSF.MATCHED", PS_META_REPLACE, "", false); + psMetadataAddBool(readoutOut->analysis, PS_LIST_TAIL, "NOT.PSF.MATCHED", PS_META_REPLACE, "", false); + } + // save the PSF on the new readout->analysis: + // if (!psMetadataAddPtr (readoutOut->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_META_REPLACE | PS_DATA_UNKNOWN, "psphot psf model", options->psf)) { + // psError (PSPHOT_ERR_UNKNOWN, false, "problem saving sources on readout"); + // return false; + // } + + // dumpImage(readoutOut, readoutSrc, index, "conv"); + // dumpImageDiff(readoutOut, fake, readoutSrc, index, "diff"); + + psFree(fake); + psFree(optWidths); + psFree(stampSources); + psFree(widthsCopy); + pmSubtractionThreadsFinalize(); + return true; + +escape: + psFree(fake); + psFree(optWidths); + psFree(stampSources); + psFree(widthsCopy); + pmSubtractionThreadsFinalize(); + return false; +} + +// Extract the regions and solutions used in the image matching +// This stops them from being freed when we iterate back up the FPA +// Record the chi-square value +// XXX this function may not be needed for psphotStack +bool saveMatchData (pmReadout *readout, psphotStackOptions *options, int index) { + + psArray *regions = options->regions->data[index] = psArrayAllocEmpty(ARRAY_BUFFER); // Match regions + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_REGION); + psMetadataIterator *iter = psMetadataIteratorAlloc(readout->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_REGION); + regions = psArrayAdd(regions, ARRAY_BUFFER, item->data.V); + } + psFree(iter); + } + + psArray *kernels = options->kernels->data[index] = psArrayAllocEmpty(ARRAY_BUFFER); // Match kernels + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataIterator *iter = psMetadataIteratorAlloc(readout->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_UNKNOWN); + pmSubtractionKernels *kernel = item->data.V; // Kernel used in subtraction + kernels = psArrayAdd(kernels, ARRAY_BUFFER, kernel); + } + psFree(iter); + } + psAssert((regions)->n == (kernels)->n, "Number of match regions and kernels should match"); + + // Record chi^2 + { + double sum = 0.0; // Sum of chi^2 + int num = 0; // Number of measurements of chi^2 + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataIterator *iter = psMetadataIteratorAlloc(readout->analysis, PS_LIST_HEAD, regex); + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_UNKNOWN); + pmSubtractionKernels *kernels = item->data.V; // Convolution kernels + sum += kernels->mean; + num++; + } + psFree(iter); + options->matchChi2->data.F32[index] = sum / (psImageCovarianceFactor(readout->covariance) * num); + } + + return true; +} + +# define NOISE_FRACTION 0.01 // Set minimum flux to this fraction of noise +# define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_SATURATED | PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_EXT_LIMIT) // Mask to apply to input sources + +// generate a fake readout against which to PSF match +pmReadout *makeFakeReadout(pmConfig *config, pmReadout *readoutSrc, psArray *sources, pmPSF *psf, psImageMaskType maskVal, int fullSize) { + + pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout with target PSF + + psStats *bg = psStatsAlloc(PS_STAT_ROBUST_STDEV); // Statistics for background + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(bg, NULL, readoutSrc->image, readoutSrc->mask, maskVal, rng)) { + psError(PSPHOT_ERR_DATA, false, "Can't measure background for image."); + psFree(fake); + psFree(bg); + psFree(rng); + return NULL; + } + float minFlux = NOISE_FRACTION * bg->robustStdev; // Minimum flux level for fake image + psFree(rng); + psFree(bg); + + bool oldThreads = pmReadoutFakeThreads(true); // Old threading state + if (!pmReadoutFakeFromSources(fake, readoutSrc->image->numCols, readoutSrc->image->numRows, sources, SOURCE_MASK, NULL, NULL, psf, minFlux, fullSize, false, true)) { + psError(PSPHOT_ERR_DATA, false, "Unable to generate fake image with target PSF."); + psFree(fake); + return NULL; + } + pmReadoutFakeThreads(oldThreads); + + fake->mask = psImageCopy(NULL, readoutSrc->mask, PS_TYPE_IMAGE_MASK); + + // Add the background into the target image + psImage *bgImage = stackBackgroundModel(readoutSrc, config); // Image of background + psBinaryOp(fake->image, fake->image, "+", bgImage); + psFree(bgImage); + + return fake; +} + +// set the widths +psVector *SetOptWidths (bool *optimum, psMetadata *recipe) { + + bool status; + + *optimum = psMetadataLookupBool(&status, recipe, "OPTIMUM"); // Derive optimum parameters? + psAssert (status, "missing recipe value %s", "OPTIMUM"); + + psVector *optWidths = NULL; // Vector with FWHMs for optimum search + + if (*optimum) { + float optMin = psMetadataLookupF32(&status, recipe, "OPTIMUM.MIN"); // Minimum width for search + psAssert (status, "missing recipe value %s", "OPTIMUM.MIN"); + + float optMax = psMetadataLookupF32(&status, recipe, "OPTIMUM.MAX"); // Maximum width for search + psAssert (status, "missing recipe value %s", "OPTIMUM.MAX"); + + float optStep = psMetadataLookupF32(&status, recipe, "OPTIMUM.STEP"); // Step for search + psAssert (status, "missing recipe value %s", "OPTIMUM.STEP"); + + optWidths = psVectorCreate(optWidths, optMin, optMax, optStep, PS_TYPE_F32); + } + + return optWidths; +} + +// Set input to be skipped if the analysis reports that psf matching has failed for some reason. +// For exmaple if decovolution fraction was overlimit or the bad fraction was too high. +// +bool psphotStackSetInputsToSkip(pmConfig *config, const pmFPAview *view, const char *filerule, bool set) { + int num = psphotFileruleCount(config, filerule); + bool status; + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + for (int i = 0; i < num; i++) { + if (i == chisqNum) { + continue; + } + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + if (set) { + bool notMatched = psMetadataLookupBool(&status, readout->analysis, "NOT.PSF.MATCHED"); + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSPHOT.SKIP.INPUT", PS_META_REPLACE, "Skip analysis", notMatched); + } else { + psMetadataAddBool(readout->analysis, PS_LIST_TAIL, "PSPHOT.SKIP.INPUT", PS_META_REPLACE, "Skip analysis", false); + } + } + + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackObjects.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackObjects.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackObjects.c (revision 41083) @@ -0,0 +1,151 @@ +# include "psphotInternal.h" + +// set a consistent position +bool psphotStackObjectsUnifyPosition (psArray *objects) { + + // consistent position: AVERAGE or CHISQ? + for (int i = 0; i < objects->n; i++) { + pmPhotObj *object = objects->data[i]; + if (!object) continue; + if (!object->sources) continue; + + int npts = 0; + float x = 0.0; + float y = 0.0; + // measure the average position (weighted?) + for (int j = 0; j < object->sources->n; j++) { + + pmSource *source = object->sources->data[j]; + if (!source) continue; + if (!source->peak) continue; + + x += source->peak->xf; + y += source->peak->yf; + npts ++; + } + if (npts == 0) continue; + + x /= (float) npts; + y /= (float) npts; + + // set the positions + for (int j = 0; j < object->sources->n; j++) { + + pmSource *source = object->sources->data[j]; + if (!source) continue; + if (!source->peak) continue; + + source->peak->xf = x; + source->peak->yf = y; + npts ++; + } + object->x = x; + object->y = y; + } + + psLogMsg ("psphot", PS_LOG_INFO, "updated positions\n"); + return true; +} + +// mark good vs bad objects +bool psphotStackObjectsSelectForAnalysis (pmConfig *config, const pmFPAview *view, const char *filerule, psArray *objects) { + + bool status = false; + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, 0); // use the 0-index image to represent the image area + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + // option to limit analysis to a specific region + char *region = psMetadataLookupStr (&status, recipe, "ANALYSIS_REGION"); + psRegion AnalysisRegion = psRegionForImage (readout->image, psRegionFromString (region)); + if (psRegionIsNaN (AnalysisRegion)) psAbort("analysis region mis-defined"); + + // S/N limit to perform full non-linear fits + float SN_LIM_PETRO = psMetadataLookupF32 (&status, recipe, "EXTENDED_SOURCE_SN_LIM"); + float SN_LIM_RADIAL = psMetadataLookupF32 (&status, recipe, "RADIAL_APERTURES_SN_LIM"); + + bool doPetroStars = psMetadataLookupBool (&status, recipe, "PETROSIAN_FOR_STARS"); + + for (int i = 0; i < objects->n; i++) { + pmPhotObj *object = objects->data[i]; + if (!object) continue; + if (!object->sources) continue; + + // we check each source for an object and keep the object if any source is valid + + bool keepObjectRadial = false; + bool keepObjectPetro = false; + for (int j = 0; j < object->sources->n; j++) { + + pmSource *source = object->sources->data[j]; + if (!source) continue; + if (!source->peak) continue; + + // skip PSF-like and non-astronomical objects + if (source->type == PM_SOURCE_TYPE_DEFECT) continue; + if (source->type == PM_SOURCE_TYPE_SATURATED) continue; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue; + if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; + + // limit selection by analysis region (this automatically apply + if (source->peak->x < AnalysisRegion.x0) continue; + if (source->peak->y < AnalysisRegion.y0) continue; + if (source->peak->x > AnalysisRegion.x1) continue; + if (source->peak->y > AnalysisRegion.y1) continue; + + // SN limit tests for RADIAL APERTURES: + bool skipSourceRadial = false; + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + skipSourceRadial = (source->moments->KronFlux < SN_LIM_RADIAL * source->moments->KronFluxErr); + } else { + skipSourceRadial = (sqrt(source->peak->detValue) < SN_LIM_RADIAL); + } + if (!skipSourceRadial) keepObjectRadial = true; + + // SN limit tests for PETRO + bool skipSourcePetro = false; + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + skipSourcePetro = (source->moments->KronFlux < SN_LIM_PETRO * source->moments->KronFluxErr); + } else { + skipSourcePetro = doPetroStars ? (sqrt(source->peak->detValue) < SN_LIM_PETRO) : true; + } + if (!skipSourcePetro) keepObjectPetro = true; + } + + for (int j = 0; j < object->sources->n; j++) { + pmSource *source = object->sources->data[j]; + if (!source) continue; + if (!source->peak) continue; + + // we have to set a bit in either case to tell psphotExtendedSourceAnalysis to + // avoid the single-detection tests + + if (keepObjectPetro) { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO_KEEP; + source->tmpFlags &= ~PM_SOURCE_TMPF_PETRO_SKIP; + } else { + source->tmpFlags |= PM_SOURCE_TMPF_PETRO_SKIP; + source->tmpFlags &= ~PM_SOURCE_TMPF_PETRO_KEEP; + } + + if (keepObjectRadial) { + source->tmpFlags |= PM_SOURCE_TMPF_RADIAL_KEEP; + source->tmpFlags &= ~PM_SOURCE_TMPF_RADIAL_SKIP; + } else { + source->tmpFlags |= PM_SOURCE_TMPF_RADIAL_SKIP; + source->tmpFlags &= ~PM_SOURCE_TMPF_RADIAL_KEEP; + } + } + } + + psLogMsg ("psphot", PS_LOG_INFO, "marked good vs bad objects\n"); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackOptions.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackOptions.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackOptions.c (revision 41083) @@ -0,0 +1,96 @@ +# include "psphotInternal.h" + +static void psphotStackOptionsFree (psphotStackOptions *options) { + + if (options == NULL) return; + + // free the psf + psFree (options->psf); + + // free the array elements + psFree (options->psfs); + psFree (options->sourceLists); + psFree (options->kernels); + psFree (options->regions); + + // free the vector elements + psFree (options->inputMask); + psFree (options->inputSeeing); + psFree (options->norm); + psFree (options->matchChi2); + psFree (options->targetSeeing); + + return; +} + +psphotStackOptions *psphotStackOptionsAlloc (int num) { + + psphotStackOptions *options = (psphotStackOptions *) psAlloc(sizeof(psphotStackOptions)); + psMemSetDeallocator(options, (psFreeFunc) psphotStackOptionsFree); + + options->numCols = 0; + options->numRows = 0; + + options->num = num; + options->psf = NULL; + options->convolve = false; + options->convolveSource = PSPHOT_CNV_SRC_NONE; + options->targetSeeing = NULL; + + options->psfs = psArrayAlloc(num); + options->sourceLists = psArrayAlloc(num); // Individual lists of sources for matching + options->kernels = psArrayAlloc(num); + options->regions = psArrayAlloc(num); + + options->inputMask = psVectorAlloc(num, PS_TYPE_VECTOR_MASK); // Mask for inputs + options->inputSeeing = psVectorAlloc(num, PS_TYPE_F32); + options->norm = psVectorAlloc(num, PS_TYPE_F32); + options->matchChi2 = psVectorAlloc(num, PS_TYPE_F32); // chi^2 for stamps when matching + + psVectorInit(options->inputMask, 0); + psVectorInit(options->inputSeeing, NAN); + psVectorInit(options->norm, NAN); + psVectorInit(options->matchChi2, NAN); + + return options; +} + +psphotStackConvolveSource psphotStackConvolveSourceFromString (const char *string) { + + if (!strcasecmp(string, "AUTO")) return PSPHOT_CNV_SRC_AUTO; + if (!strcasecmp(string, "CNV")) return PSPHOT_CNV_SRC_CNV; + if (!strcasecmp(string, "RAW")) return PSPHOT_CNV_SRC_RAW; + return PSPHOT_CNV_SRC_NONE; +} + +pmFPAfile *psphotStackGetConvolveSource (pmConfig *config, psphotStackOptions *options, int index) { + + // which image do we want to convolve? RAW, CNV, AUTO? + // find the currently selected readout + pmFPAfile *fileRaw = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.INPUT.RAW", index); // File of interest + pmFPAfile *fileCnv = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.INPUT.CNV", index); // File of interest + + pmFPAfile *fileSrc = NULL; + + switch (options->convolveSource) { + case PSPHOT_CNV_SRC_AUTO: + fileSrc = fileCnv ? fileCnv : fileRaw; + break; + + case PSPHOT_CNV_SRC_RAW: + fileSrc = fileRaw; + break; + + case PSPHOT_CNV_SRC_CNV: + fileSrc = fileCnv; + break; + + default: + psAbort("impossible case"); + } + if (!fileSrc) { + psError(PSPHOT_ERR_CONFIG, true, "desired convolution source is missing (cnv : %p, raw : %p)", fileCnv, fileRaw); + } + + return fileSrc; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackPSF.c (revision 41083) @@ -0,0 +1,114 @@ +# include "psphotInternal.h" + +// determine the 1st target PSF (either AUTO or defined by PSPHOT.STACK.TARGET.PSF.FWHM) +bool psphotStackPSF(const pmConfig *config, psphotStackOptions *options) { + + bool mdok = false; + + int numCols = options->numCols; + int numRows = options->numRows; + psArray *psfs = options->psfs; + psVector *inputMask = options->inputMask; + + // Get the recipe values + psMetadata *psphotRecipe = psMetadataLookupMetadata(NULL, config->recipes, "PSPHOT"); // psphot recipe + psAssert(psphotRecipe, "We've thrown an error on this before."); + + bool autoPSF = psMetadataLookupBool (&mdok, psphotRecipe, "PSPHOT.STACK.TARGET.PSF.AUTO"); + + // Get the recipe values + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, "PPSTACK"); // ppStack recipe + psAssert(recipe, "We've thrown an error on this before."); + + char *psfModel = psMetadataLookupStr(NULL, recipe, "PSF.MODEL"); // Model for PSF + + options->targetSeeing = psVectorAllocEmpty(4, PS_TYPE_F32); + + if (autoPSF) { + int psfInstances = psMetadataLookupS32(NULL, recipe, "PSF.INSTANCES"); // Number of instances for PSF + float psfRadius = psMetadataLookupF32(NULL, recipe, "PSF.RADIUS"); // Radius for PSF + int psfOrder = psMetadataLookupS32(NULL, recipe, "PSF.ORDER"); // Spatial order for PSF + + psString maskValStr = psMetadataLookupStr(&mdok, recipe, "MASK.VAL"); // Name of bits to mask going in + if (!mdok || !maskValStr) { + psError(PSPHOT_ERR_CONFIG, false, "Unable to find MASK.VAL in recipe"); + return false; + } + psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Bits to mask + + for (int i = 0; i < psfs->n; i++) { + if (inputMask->data.U8[i]) { + psFree(psfs->data[i]); + psfs->data[i] = NULL; + } + } + + // Solve for the target PSF + options->psf = pmPSFEnvelope(numCols, numRows, psfs, psfInstances, psfRadius, psfModel, psfOrder, psfOrder, maskVal); + if (!options->psf) { + psError(PSPHOT_ERR_PSF, false, "Unable to determine output PSF."); + return false; + } + + psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "PSF.TARGET", PS_DATA_UNKNOWN, "Target PSF for stack", options->psf); + float targetSeeing = pmPSFtoFWHM(options->psf, 0.5 * options->numCols, 0.5 * options->numRows); // FWHM for target + psVectorAppend(options->targetSeeing, targetSeeing); + psLogMsg("psphotStack", PS_LOG_INFO, "Target seeing FWHM (auto-scaled): %f\n", targetSeeing); + return true; + } + + // externally-defined PSF + // XXX need to test for compatibility of target with inputs + + // is a single target FWHM specified, or a set of values? set up the vector options->targetSeeing and the local 1st value + float targetSeeing = psMetadataLookupF32 (&mdok, psphotRecipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); + if (!mdok) { + psVector *fwhmValues = psMetadataLookupVector(&mdok, psphotRecipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); // Magnitude offsets + psAssert (mdok, "missing psphot recipe value PSPHOT.STACK.TARGET.PSF.FWHM"); + for (int i = 0; i < fwhmValues->n; i++) { + psVectorAppend(options->targetSeeing, fwhmValues->data.F32[i]); + } + targetSeeing = fwhmValues->data.F32[0]; + } else { + psVectorAppend(options->targetSeeing, targetSeeing); + } + + // measured scale factors (fwhm = Sxx * 2.35 * scaleFactor / sqrt(2.0)) + // GAUSS : 1.000 + // PGAUSS : 1.006 + // QGAUSS : 1.151 + // RGAUSS : 0.883 + // PS1_V1 : 1.134 + + float scaleFactor = NAN; + if (!strcmp(psfModel, "PS_MODEL_GAUSS")) { + scaleFactor = 1.000; + } + if (!strcmp(psfModel, "PS_MODEL_PGAUSS")) { + scaleFactor = 1.0006; + } + if (!strcmp(psfModel, "PS_MODEL_QGAUSS")) { + scaleFactor = 1.151; + } + if (!strcmp(psfModel, "PS_MODEL_RGAUSS")) { + scaleFactor = 0.883; + } + if (!strcmp(psfModel, "PS_MODEL_PS1_V1")) { + scaleFactor = 1.134; + } + psAssert (isfinite(scaleFactor), "invalid model for PSF"); + + float Sxx = sqrt(2.0)*targetSeeing / 2.35 / scaleFactor; + + // XXX probably should make the model type (and par 7) optional from recipe + // psf = pmPSFBuildSimple(psfModel, Sxx, Sxx, 0.0, 1.0); + options->psf = pmPSFBuildSimple(psfModel, Sxx, Sxx, 0.0, 0.2); + if (!options->psf) { + psError(PSPHOT_ERR_PSF, false, "Unable to build dummy PSF."); + return false; + } + + psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "PSF.TARGET", PS_DATA_UNKNOWN, "Target PSF for stack", options->psf); + psLogMsg("psphotStack", PS_LOG_INFO, "Target seeing FWHM (1 of %ld): %f\n", options->targetSeeing->n, targetSeeing); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackParseCamera.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackParseCamera.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackParseCamera.c (revision 41083) @@ -0,0 +1,384 @@ +# include "psphotStandAlone.h" + +static pmFPAfile *defineFile(pmConfig *config, pmFPAfile *bind, const char *name, const char *filename, pmFPAfileType type); + +// define the needed / desired I/O files +bool psphotStackParseCamera (pmConfig *config) { + + bool status = false; + + // the input images are defined as a set of metadatas in the INPUTS metadata folder + psMetadata *inputs = psMetadataLookupMetadata(&status, config->arguments, "INPUTS"); // The inputs info + if (!inputs) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find inputs."); + return false; + } + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + bool savePSF = psMetadataLookupBool(&status, recipe, "SAVE.PSF"); + // XXX: bills: I don't think psphotStack will work if SAVE.BACKMDL is false + bool saveBackgroundModel = psMetadataLookupBool(&status, recipe, "SAVE.BACKMDL"); + bool saveBackground = psMetadataLookupBool(&status, recipe, "SAVE.BACKGND"); + bool saveBackSub = psMetadataLookupBool(&status, recipe, "SAVE.BACKSUB"); + bool saveResid = psMetadataLookupBool(&status, recipe, "SAVE.RESID"); + + bool saveCnv = psMetadataLookupBool(&status, recipe, "SAVE.CNV"); + bool saveChisq = psMetadataLookupBool(&status, recipe, "SAVE.CHISQ"); + bool useRaw = psMetadataLookupBool(&status, recipe, "PSPHOT.STACK.USE.RAW"); + + bool updateMode = psMetadataLookupBool(&status, config->arguments, "PSPHOT.STACK.UPDATEMODE"); + if (updateMode) { + // Tell the sources reader to save a copy of the header from the input sources file on + // readout->analysis so that we can copy some metadata from measurements that are skipped + // in update mode from there to the analysis structure. + psMetadataAddBool(recipe, PS_LIST_TAIL, "SAVE.INPUT.SOURCES.HEADER", PS_META_REPLACE, "", true); + } + + + int nRaw = 0; + int nCnv = 0; + int nInputs = inputs->list->n; + for (int i = 0; i < nInputs; i++) { + psMetadataItem *item = psMetadataGet(inputs, i); + if (item->type != PS_DATA_METADATA) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Component %s of the input metadata is not of type METADATA", item->name); + return false; + } + + psMetadata *input = item->data.md; // The input metadata of interest + + pmFPAfile *rawInputFile = NULL; + pmFPAfile *cnvInputFile = NULL; + + // RAW (unconvolved) input data (RAW:IMAGE, RAW:MASK, RAW:VARIANCE, RAW:PSF) + psString rawImage = psMetadataLookupStr(&status, input, "RAW:IMAGE"); // Name of image + if (rawImage && strlen(rawImage) > 0) { + rawInputFile = defineFile(config, NULL, "PSPHOT.STACK.INPUT.RAW", rawImage, PM_FPA_FILE_IMAGE); // File for image + if (!rawInputFile) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from image %d (%s)", i, rawImage); + return false; + } + psString mask = psMetadataLookupStr(&status, input, "RAW:MASK"); // Name of mask + if (mask && strlen(mask) > 0) { + if (!defineFile(config, rawInputFile, "PSPHOT.STACK.MASK.RAW", mask, PM_FPA_FILE_MASK)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from mask %d (%s)", i, mask); + return false; + } + } + psString variance = psMetadataLookupStr(&status, input, "RAW:VARIANCE"); // Name of variance map + if (variance && strlen(variance) > 0) { + if (!defineFile(config, rawInputFile, "PSPHOT.STACK.VARIANCE.RAW", variance, PM_FPA_FILE_VARIANCE)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from variance %d (%s)", i, variance); + return false; + } + } + psString psf = psMetadataLookupStr(&status, input, "RAW:PSF"); // Name of psf + if (psf && strlen(psf) > 0) { + // if (!defineFile(config, rawInputFile, "PSPHOT.STACK.PSF.RAW", psf, PM_FPA_FILE_PSF)) { + if (!defineFile(config, rawInputFile, "PSPHOT.PSF.LOAD", psf, PM_FPA_FILE_PSF)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from psf %d (%s)", i, psf); + return false; + } + } + psString backmdl = psMetadataLookupStr(&status, input, "RAW:BACKMDL"); // Name of background model + if (backmdl && strlen(backmdl) > 0) { + if (!defineFile(config, NULL, "PSPHOT.STACK.BACKMDL.RAW", backmdl, PM_FPA_FILE_IMAGE)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from backmdl %d (%s)", i, backmdl); + return false; + } + } + psString expnum = psMetadataLookupStr(&status, input, "RAW:EXPNUM"); // Name of expnum image + if (expnum && strlen(expnum) > 0) { + if (!defineFile(config, rawInputFile, "PSPHOT.STACK.EXPNUM.RAW", expnum, PM_FPA_FILE_EXPNUM)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from expnum %d (%s)", i, expnum); + return false; + } + } + nRaw ++; + } + + // CNV (convolved) input data (CNV:IMAGE, CNV:MASK, CNV:VARIANCE, CNV:PSF) + psString cnvImage = psMetadataLookupStr(&status, input, "CNV:IMAGE"); // Name of image + if (cnvImage && strlen(cnvImage) > 0) { + cnvInputFile = defineFile(config, NULL, "PSPHOT.STACK.INPUT.CNV", cnvImage, PM_FPA_FILE_IMAGE); // File for image + if (!cnvInputFile) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from image %d (%s)", i, cnvImage); + return false; + } + psString mask = psMetadataLookupStr(&status, input, "CNV:MASK"); // Name of mask + if (mask && strlen(mask) > 0) { + if (!defineFile(config, cnvInputFile, "PSPHOT.STACK.MASK.CNV", mask, PM_FPA_FILE_MASK)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from mask %d (%s)", i, mask); + return false; + } + } + psString variance = psMetadataLookupStr(&status, input, "CNV:VARIANCE"); // Name of variance map + if (variance && strlen(variance) > 0) { + if (!defineFile(config, cnvInputFile, "PSPHOT.STACK.VARIANCE.CNV", variance, PM_FPA_FILE_VARIANCE)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from variance %d (%s)", i, variance); + return false; + } + } + psString expnum = psMetadataLookupStr(&status, input, "CNV:EXPNUM"); // Name of EXPNUM image + if (expnum && strlen(expnum) > 0) { + if (!defineFile(config, cnvInputFile, "PSPHOT.STACK.EXPNUM.CNV", expnum, PM_FPA_FILE_EXPNUM)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from expnum %d (%s)", i, expnum); + return false; + } + } + psString psf = psMetadataLookupStr(&status, input, "CNV:PSF"); // Name of mask + if (psf && strlen(psf) > 0) { + if (!defineFile(config, cnvInputFile, "PSPHOT.STACK.PSF.CNV", psf, PM_FPA_FILE_PSF)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from psf %d (%s)", i, psf); + return false; + } + } + nCnv ++; + } + + if (!rawInputFile && !cnvInputFile) { + psError(PSPHOT_ERR_CONFIG, true, "Component %s (%d) lacks both RAW:IMAGE and CNV:IMAGE of type STR", item->name, i); + return false; + } + + // XXX what if they do not match in length + if (nCnv && nRaw) { + if (nCnv != nRaw) { + psError (PSPHOT_ERR_CONFIG, true, "if both RAW and CNV images are supplied, the number must match"); + return false; + } + } + pmFPAfile *inputTemplate; + if (useRaw) { + inputTemplate = rawInputFile; + } else { + inputTemplate = cnvInputFile; + } + if (!inputTemplate) { + psError(PS_ERR_UNKNOWN, true, "cannot determinte inputTemplate: USE.RAW: %d\n", useRaw); + return false; + } + + psString sources = psMetadataLookupStr(&status, input, "SOURCES"); // Name of sources + if (sources && strlen(sources) > 0) { + // input sources are not bound to fpa. + // XXX: bills: I believe that they are only required in -updatemode now. + if (!defineFile(config, NULL, "PSPHOT.STACK.SOURCES", sources, PM_FPA_FILE_CMF)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from sources %d (%s)", i, sources); + return false; + } + } + + psS64 stack_id = psMetadataLookupS64(&status, input, "STACK_ID"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to find STACK_ID from sources %d", i); + return false; + } + // generate an pmFPAimage for the output convolved image + { + // pmFPAfile *srcInputFile = rawInputFile ? rawInputFile : cnvInputFile; + pmFPAfile *outputImage = pmFPAfileDefineOutput(config, NULL, "PSPHOT.STACK.OUTPUT.IMAGE"); + if (!outputImage) { + psError(PSPHOT_ERR_CONFIG, false, "Trouble defining PSPHOT.STACK.OUTPUT.IMAGE"); + return false; + } + outputImage->save = saveCnv; + outputImage->fileID = stack_id; // this is used to generate output names + + pmFPAfile *outputMask = pmFPAfileDefineOutput(config, outputImage->fpa, "PSPHOT.STACK.OUTPUT.MASK"); + if (!outputMask) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PSPHOT.STACK.OUTPUT.MASK")); + return NULL; + } + if (outputMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PSPHOT.STACK.OUTPUT.MASK is not of type MASK"); + return NULL; + } + outputMask->save = saveCnv; + outputMask->fileID = stack_id; // this is used to generate output names + + pmFPAfile *outputVariance = pmFPAfileDefineOutput(config, outputImage->fpa, "PSPHOT.STACK.OUTPUT.VARIANCE"); + if (!outputVariance) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PSPHOT.STACK.OUTPUT.VARIANCE")); + return NULL; + } + if (outputVariance->type != PM_FPA_FILE_VARIANCE) { + psError(PS_ERR_IO, true, "PSPHOT.STACK.OUTPUT.VARIANCE is not of type VARIANCE"); + return NULL; + } + outputVariance->save = saveCnv; + outputVariance->fileID = stack_id; // this is used to generate output names + + // the output sources are carried on the outputImage->fpa structures + pmFPAfile *outsources = pmFPAfileDefineOutputFromFile (config, outputImage, "PSPHOT.STACK.OUTPUT"); + if (!outsources) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.STACK.OUTPUT"); + return false; + } + outsources->save = true; + outsources->fileID = stack_id; // this is used to generate output names + + if (savePSF) { + pmFPAfile *output = pmFPAfileDefineOutputFromFile (config, inputTemplate, "PSPHOT.STACK.PSF.SAVE"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.STACK.PSF.SAVE"); + return false; + } + // don't save psf in update mode? + output->save = !updateMode; + output->fileID = stack_id; + } + if (saveBackgroundModel) { + int DX = psMetadataLookupS32 (&status, recipe, "BACKGROUND.XBIN"); + int DY = psMetadataLookupS32 (&status, recipe, "BACKGROUND.YBIN"); + pmFPAfile *output = pmFPAfileDefineFromFile (config, inputTemplate, DX, DY, "PSPHOT.STACK.BACKMDL"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.STACK.BACKMDL"); + return false; + } + + // do not save the background model file when in update mode. We need to create the file above + // but we don't need save it. + output->save = !updateMode; + output->fileID = stack_id; + } + if (saveBackground) { + pmFPAfile *output = pmFPAfileDefineFromFile (config, inputTemplate, 1, 1, "PSPHOT.STACK.BACKGND"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.STACK.STACK.BACKGND"); + return false; + } + output->save = true; + output->fileID = stack_id; + } + if (saveBackSub) { + pmFPAfile *output = pmFPAfileDefineFromFile (config, inputTemplate, 1, 1, "PSPHOT.STACK.BACKSUB"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.STACK.STACK.BACKSUB"); + return false; + } + output->save = true; + output->fileID = stack_id; + } + if (saveResid) { + pmFPAfile *output = pmFPAfileDefineOutputFromFile (config, inputTemplate, "PSPHOT.STACK.RESID"); + if (!output) { + psError(PSPHOT_ERR_CONFIG, false, "Cannot find a rule for PSPHOT.STACK.RESID"); + return false; + } + output->save = true; + output->fileID = stack_id; + } + } + } + psMetadataRemoveKey(config->arguments, "FILENAMES"); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.INPUT.RAW.NUM", PS_META_REPLACE, "number of inputs", nRaw); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.INPUT.CNV.NUM", PS_META_REPLACE, "number of inputs", nCnv); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.STACK.OUTPUT.IMAGE.NUM", PS_META_REPLACE, "number of inputs", nInputs); + psMetadataAddS32 (config->arguments, PS_LIST_TAIL, "PSPHOT.INPUT.NUM", PS_META_REPLACE, "number of inputs", nInputs); + + if (!psphotSetMaskBits (config)) { + psError (PS_ERR_UNKNOWN, false, "failed to set mask bit values"); + return NULL; + } + +#ifdef MAKE_CHISQ_IMAGE + // generate an pmFPAimage for the chisqImage + { + pmFPAfile *chisqImage = pmFPAfileDefineOutput(config, NULL, "PSPHOT.CHISQ.IMAGE"); + if (!chisqImage) { + psError(PSPHOT_ERR_CONFIG, false, "Trouble defining PSPHOT.CHISQ.IMAGE"); + return false; + } + chisqImage->save = saveChisq; + + pmFPAfile *chisqMask = pmFPAfileDefineOutput(config, chisqImage->fpa, "PSPHOT.CHISQ.MASK"); + if (!chisqMask) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PSPHOT.CHISQ.MASK")); + return NULL; + } + if (chisqMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PSPHOT.CHISQ.MASK is not of type MASK"); + return NULL; + } + chisqMask->save = saveChisq; + + pmFPAfile *chisqVariance = pmFPAfileDefineOutput(config, chisqImage->fpa, "PSPHOT.CHISQ.VARIANCE"); + if (!chisqVariance) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PSPHOT.CHISQ.VARIANCE")); + return NULL; + } + if (chisqVariance->type != PM_FPA_FILE_VARIANCE) { + psError(PS_ERR_IO, true, "PSPHOT.CHISQ.VARIANCE is not of type VARIANCE"); + return NULL; + } + chisqVariance->save = saveChisq; + } +#else + (void) saveChisq; +#endif // MAKE_CHISQ_IMAGE + + psTrace("psphot", 1, "Done with psphotStackParseCamera...\n"); + + psErrorClear(); // some metadata lookup may have failed + return true; +} + +// Define a file +static pmFPAfile *defineFile(pmConfig *config, // Configuration + pmFPAfile *bind, // File to which to bind + const char *name, // Name of file rule + const char *filename, // Name of file + pmFPAfileType type // Type of file + ) +{ + + psArray *dummy = psArrayAlloc(1); // Dummy array of filenames for this FPA + dummy->data[0] = psStringCopy(filename); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "FILENAMES", PS_META_REPLACE, + "Filenames for file rule definition", dummy); + psFree(dummy); + + bool found = false; // Found the file? + pmFPAfile *file = bind ? pmFPAfileBindFromArgs(&found, bind, config, name, "FILENAMES") : + pmFPAfileDefineFromArgs(&found, config, name, "FILENAMES"); + if (!file || !found) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file %s from %s", name, filename); + return NULL; + } + if (file->type != type) { + psError(PS_ERR_IO, true, "%s is not of type %s", name, pmFPAfileStringFromType(type)); + return NULL; + } + + return file; +} + + +/*** + * + * psphotStack : + + * * inputs: + * * unconvolved images + * * raw convolved images + * * psfs (unconvolved or convolved?) + * * sources + + * optionally convolve the unconvolved or the raw inputs + * optionally perform no convolutions + * optionally save the psf-matched images + + */ + + +# if (0) + // define the additional input/output files associated with psphot + // XXX figure out which files are needed by psphotStack + if (false && !psphotDefineFiles (config, input)) { + psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files"); + return false; + } +# endif + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackReadout.c (revision 41083) @@ -0,0 +1,728 @@ +# include "psphotInternal.h" + +static void psphotLogMemoryStats(const char *heading); + +// relevant filesets: +# define STACK_RAW "PSPHOT.STACK.INPUT.RAW" +# define STACK_OUT "PSPHOT.STACK.OUTPUT.IMAGE" + +// XXX STACK_OUT currently is a copy of STACK_RAW, but should be a pointer to is as in psphot (single) + +// TEST CODE, can be removed +bool psphotDumpImages (pmConfig *config, const pmFPAview *view, const char *filerule, char *base) { + + // XXX do nothing + return true; + + int num = psphotFileruleCount(config, "PSPHOT.INPUT"); + + for (int i = 0; i < num; i++) { + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + char line[256]; + snprintf (line, 256, "%s.%d.im.fits", base, i); + psphotSaveImage (NULL, readout->image, line); + + snprintf (line, 256, "%s.%d.wt.fits", base, i); + psphotSaveImage (NULL, readout->variance, line); + + snprintf (line, 256, "%s.%d.mk.fits", base, i); + psphotSaveImage (NULL, readout->mask, line); + } + // psphotSaveImage leaves an error on the stack + psErrorClear(); + return true; +} + +bool psphotStackVisualFilerule(pmConfig *config, const pmFPAview *view, const char *filerule) { + + bool status = false; + + int num = psphotFileruleCount(config, filerule); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detections, "missing detections?"); + + psArray *sources = detections->allSources; + psAssert (sources, "missing sources?"); + + psphotVisualShowResidualImage (readout, true); + psphotVisualShowObjectRegions (readout, recipe, sources); + } + return true; +} + +bool psphotStackReadout (pmConfig *config, const pmFPAview *view) { + + psArray *objects = NULL; // used below after 'pass1finish' label + + // measure the total elapsed time in psphotReadout. dtime is the elapsed time used jointly + // by the multiple threads, not the total time used by all threads. + psTimerStart ("psphotReadout"); + + psphotLogMemoryStats("Start"); + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); // allow models to have ugly fits (eg, central cusp) + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + // optional break-point for processing + char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); + psAssert (breakPt, "configuration error: set BREAK_POINT"); + + // load WCS + if (!psphotStackLoadWCS(config, view, STACK_RAW)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble loading WCS for %s", STACK_RAW); + return false; + } + + // set the photcode for each image + if (!psphotAddPhotcode (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images (if not supplied) and set mask bits. + if (!psphotSetMaskAndVariance (config, view, STACK_RAW)) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "NOTHING")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // generate a background model (median, smoothed image) + if (!psphotModelBackground (config, view, STACK_RAW)) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!psphotSubtractBackground (config, view, STACK_RAW)) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "BACKMDL")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + +#ifdef MAKE_CHISQ_IMAGE + // also make the chisq detection image + if (!psphotStackChisqImage(config, view, STACK_RAW, STACK_RAW)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failure to generate chisq image"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "CHISQ")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } +#endif + + // find the detections (by peak and/or footprint) in the image. + // This finds the detections on Chisq image as well as the individuals + if (!psphotFindDetections (config, view, STACK_RAW, true)) { // pass 1 + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // construct sources and measure basic stats (saved on detections->newSources) + if (!psphotSourceStats (config, view, STACK_RAW, true)) { // pass 1 + psError(PSPHOT_ERR_UNKNOWN, false, "failure to generate sources"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "PEAKS")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + // psphotDumpTest (config, view, STACK_RAW); + psMemDump("sourcestats"); + psphotLogMemoryStats("sourcestats"); + + // classify sources based on moments, brightness + // only run this on detections from the input images, not chisq image + if (!psphotRoughClass (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to determine rough classifications"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // find and subtract radial profile models for saturated stars (XXX change name eventually) + if (!psphotDeblendSatstars (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed on satstar deblend analysis"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // if we were not supplied a PSF model, determine the IQ stats here (detections->newSources) + // only run this on detections from the input images, not chisq image + if (!psphotImageQuality (config, view, STACK_RAW)) { // pass 1 + psError (PSPHOT_ERR_UNKNOWN, false, "failed to measure image quality"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "MOMENTS")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // use bright stellar objects to measure PSF + if (!psphotChoosePSF (config, view, STACK_RAW, true)) { // pass 1 + psLogMsg ("psphot", 3, "failure to construct a psf model"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "PSFMODEL")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + psphotMergeSources (config, view, STACK_RAW); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, STACK_RAW); + + // linear PSF fit to source peaks, subtract the models from the image (in PSF mask) + psphotFitSourcesLinear (config, view, STACK_RAW, false, false); + psphotStackVisualFilerule(config, view, STACK_RAW); + + // measure the radial profiles to the sky + psphotRadialProfileWings (config, view, STACK_RAW); + + // re-measure the kron mags with models subtracted. this pass starts with a circular + // window of size PSF_MOMENTS_RADIUS (same window used to measure the psf-scale moments) + // but iterates to an appropriately larger size + psphotLogMemoryStats("before.kron.1"); + psphotKronIterate(config, view, STACK_RAW, 1); + psphotLogMemoryStats("after.kron.1"); + + // identify CRs and extended sources + psphotSourceSize (config, view, STACK_RAW, true); + + // non-linear PSF and EXT fit to brighter sources + // replace model flux, adjust mask as needed, fit, subtract the models (full stamp) + psphotBlendFit (config, view, STACK_RAW); // pass 1 (detections->allSources) + + // replace all sources (do NOT ignore subtraction state) + psphotReplaceAllSources (config, view, STACK_RAW, false); // pass 1 (detections->allSources) + + psphotLogMemoryStats("pass1"); + + // if we only do one pass, skip to extended source analysis + if (!strcasecmp (breakPt, "PASS1")) goto pass1finish; + + // linear fit to include all sources (subtract again) + // NOTE : apply to ALL sources (extended + psf) + // NOTE 2 : this function subtracts the models from the given filerule + psphotFitSourcesLinear (config, view, STACK_RAW, true, false); // pass 2 (detections->allSources) + + // NOTE: possibly re-measure background model here with objects subtracted / or masked + + // NOTE: this block performs the 2nd pass low-significance PSF detection stage + { + // add noise for subtracted objects + psphotAddNoise (config, view, STACK_RAW); // pass 1 (detections->allSources) + + // find fainter sources + // NOTE: finds new peaks and new footprints, OLD and FULL set are saved on detections + psphotFindDetections (config, view, STACK_RAW, false); // pass 2 (detections->peaks, detections->footprints) + + // remove noise for subtracted objects (ie, return to normal noise level) + // NOTE: this needs to operate only on the OLD sources + // NOTE: if fooprintsUseUnsubtracted, the noise has already been removed by psphotFindDetections + bool footprintsUseUnsubtracted = psMetadataLookupBool(NULL, recipe, "FOOTPRINT_USE_UNSUBTRACTED"); + if (!footprintsUseUnsubtracted) { + psphotSubNoise (config, view, STACK_RAW); // pass 1 (detections->allSources) + } + + // define new sources based on only the new peaks & measure moments + // NOTE: new sources are saved on detections->newSources + psphotSourceStats (config, view, STACK_RAW, false); // pass 2 (detections->newSources) + + // set source type + // NOTE: apply only to detections->newSources + if (!psphotRoughClass (config, view, STACK_RAW)) { // pass 2 (detections->newSources) + psLogMsg ("psphot", 3, "failed to find a valid PSF clump for image"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // replace all sources so fit below applies to all at once + // NOTE: apply only to OLD sources (which have been subtracted) + psphotReplaceAllSources (config, view, STACK_RAW, false); // pass 2 + + // merge the newly selected sources into the existing list + // NOTE: merge OLD and NEW + // XXX check on free of sources... + psphotMergeSources (config, view, STACK_RAW); // (detections->newSources + detections->allSources -> detections->allSources) + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, STACK_RAW); + } + + // gcc doesn't like the label to refer to a declaration so we declare this here + bool splitLinearFit; + +pass1finish: + + splitLinearFit = psMetadataLookupBool(NULL, recipe, "PSPHOT.STACK.SPLIT.LINEAR.FIT"); + if (splitLinearFit) { + psLogMsg ("psphot", 3, "splitting fit of detected and matched soures\n"); + // Fit the detected sources separately from matched ones that we are about to create. + // NOTE: apply to ALL sources but only include sources with postitive flux in the fit + psphotFitSourcesLinear (config, view, STACK_RAW, true, true); // pass 3 (detections->allSources) + } + + psphotLogMemoryStats("prematch"); + + // generate the objects (objects unify the sources from the different images) NOTE: could + // this just match the detections for the chisq image, and not bother measuring the source + // stats in that case...? + objects = psphotMatchSources (config, view, STACK_RAW); + psMemDump("matchsources"); + + // check the source density. If it too high change the number of radial bins + // in the recipe. + psphotLimitRadialApertures(recipe, objects->n); + + // Construct an initial model for each object, set the radius to fitRadius, set circular + // fit mask. NOTE: only applied to sources without guess models + psphotGuessModels (config, view, STACK_RAW); + + psphotStackObjectsUnifyPosition (objects); + + // psphotStackObjectsSelectForAnalysis (config, view, STACK_RAW, objects); + + // final linear fit. NOTE: if splitLinearFit is true above, this pass will only fit + // the unsubtracted (matched) sources (the sources that we fit above are subtracted) + psphotFitSourcesLinear (config, view, STACK_RAW, true, false); // pass 4 (detections->allSources) + + // measure the radial profiles to the sky (only measures new objects) + psphotRadialProfileWings (config, view, STACK_RAW); + + // re-measure the kron mags with models subtracted + // psphotKronMasked(config, view, STACK_SRC); + psphotLogMemoryStats("before.kron.2"); + psphotKronIterate(config, view, STACK_RAW, 2); + psphotLogMemoryStats("after.kron.2"); + + // measure source size for the remaining sources + // NOTE: applies only to NEW (unmeasured) sources + psphotSourceSize (config, view, STACK_RAW, false); // pass 2 (detections->allSources) + + psMemDump("psfstats"); + + // drop matched sources without any useful measurements and set kron radii for the ones + // we decide to keep + psphotFilterMatchedSources (config, view, STACK_RAW, objects); + + // measure kron fluxes for the matched sources only + psphotKronIterate(config, view, STACK_RAW, 3); + + // decide which source(s) are to be fitted with the extended source analysis code. + psphotChooseAnalysisOptionsByObject (config, view, STACK_RAW, objects); + + // measure elliptical apertures, petrosians (objects sorted by S/N) + // psphotExtendedSourceAnalysisByObject (config, objects, view, STACK_SRC); // pass 1 (detections->allSources) + psphotExtendedSourceAnalysis (config, view, STACK_RAW); // pass 1 (detections->allSources) + + // measure non-linear extended source models (exponential, deVaucouleur, Sersic) (sources sorted by S/N) + psphotExtendedSourceFits (config, view, STACK_RAW); // pass 1 (detections->allSources) + + // measure some parameters for galxy science + psphotGalaxyParams (config, view, STACK_RAW); // pass 1 (detections->allSources) + + // create source children for the OUT filerule (for radial aperture photometry and output) + // NOTE: The new source children have image arrays pointing to the readout associated with + // STACK_OUT. in psphotStackMatchPSFsetup, we copy the current pixel values from RAW to OUT, + // but keep the pointers the same so we do not break these source image references + // XXX NOTE : if we use the pre-20130914 psphotStackReadout code, we need to use 'false' for the + // sourcesSubtracted argument + + // do this here so that the variance image is available to reset the models + psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); + psArray *objectsOut = psphotSourceChildrenByObject (config, view, STACK_OUT, STACK_RAW, objects, true); + if (!objectsOut) { + psFree(objects); + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + // These arrays are no longer needed. The inputs' sources arrays have references to the data + psFree (objects); + psFree (objectsOut); + + bool radial_apertures = psMetadataLookupBool(NULL, recipe, "RADIAL_APERTURES"); + if (radial_apertures) { + // measure circular, radial apertures (objects sorted by S/N) + // this forces photometry on the undetected sources from other images + + // set up the FWHM vector + psphotDumpImages (config, view, STACK_RAW, "raw.t0"); + psphotDumpImages (config, view, STACK_OUT, "out.t0"); + + int nRadialEntries = psphotStackMatchPSFsEntries(config, view, STACK_OUT); + + for (int entry = 0; entry < nRadialEntries; entry++) { + // NOTE: entry 0 is the unmatched image set + + char line[256]; + + // measure circular, radial apertures (objects sorted by S/N) + psphotRadialApertures (config, view, STACK_OUT, entry); + snprintf (line, 256, "%s.%d", "out.t1", entry); + psphotDumpImages (config, view, STACK_OUT, line); + + // replace the flux in the image so it is returned to its original state + psphotReplaceAllSources (config, view, STACK_OUT, false); + snprintf (line, 256, "%s.%d", "out.t2", entry); + psphotDumpImages (config, view, STACK_OUT, line); + + if (entry < nRadialEntries - 1) { + // smooth to the next FWHM + // this function does nothing if the targetFWHM is smaller than the currentFWHM + psphotStackMatchPSFsNext(config, view, STACK_OUT, entry); + snprintf (line, 256, "%s.%d", "out.t3", entry); + psphotDumpImages (config, view, STACK_OUT, line); + psMemDump("matched"); + + // re-measure the PSF for the smoothed image (using entries in 'allSources') + if (!psphotChoosePSF (config, view, STACK_OUT, false)) { + psLogMsg ("psphot", 3, "failure to construct a psf model in radial aperture loop for entry :%d", entry); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // this is necessary to update the models based on the new PSF + psphotResetModels (config, view, STACK_OUT); + + // this is necessary to get the right normalization for the new models + // and to subtract the sources + psphotFitSourcesLinear (config, view, STACK_OUT, false, false); + snprintf (line, 256, "%s.%d", "out.t4", entry); + psphotDumpImages (config, view, STACK_OUT, line); + } + } + } + + // measure aperture photometry corrections + if (!psphotApResid (config, view, STACK_RAW)) { + psLogMsg ("psphot", 3, "failed on psphotApResid"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // calculate source magnitudes + psphotMagnitudes(config, view, STACK_RAW); + + // calculate lensing parameters + if (!psphotLensing(config, view, STACK_RAW)) { + psErrorStackPrint(stderr, "Unable to do lensing parameters."); + psErrorClear(); + } + + if (!psphotEfficiency(config, view, STACK_RAW)) { + psErrorStackPrint(stderr, "Unable to determine detection efficiencies from fake sources"); + psErrorClear(); + } + psphotCopyEfficiency (config, view, STACK_OUT, STACK_RAW); + + psphotLogMemoryStats("final"); +#if (0) + psphotSourceMemory(config, view, STACK_RAW); + psphotSourceMemory(config, view, STACK_OUT); +#endif + + // replace background in residual image + psphotSkyReplace (config, view, STACK_RAW); + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, STACK_RAW); + psphotSourceFreePixels (config, view, STACK_OUT); + +#ifdef MAKE_CHISQ_IMAGE + // remove chisq image from config->file:PSPHOT.INPUT + psphotStackRemoveChisqFromInputs(config, STACK_RAW); +#endif + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, STACK_RAW); +} + +bool psphotStackLoadWCS(pmConfig *config, const pmFPAview *view, const char *filerule) { + + int num = psphotFileruleCount(config, filerule); + + for (int i=0; ifiles, filerule, i); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmHDU *hdu = pmHDUFromReadout(readout); + psAssert (hdu, "input missing hdu?"); + + if (!pmAstromReadWCS(file->fpa, readout->parent->parent, hdu->header, 1.0)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to read WCS from header for input %d", i); + return false; + } + } + return true; +} + +// read the memory usage data from /proc and log them out +// This will only work on a system that has /proc (not MacOS for instance) but since this function just +// tries to open and read from a file it is safe +// XXX: refine this and move it to psLib +void psphotLogMemoryStats(const char *heading) { + + // file containing memory statistics for this process proc. See proc(5) + const char* statm_path = "/proc/self/statm"; + + FILE *f = fopen(statm_path,"r"); + if (!f) { + psLogMsg ("psphot", PS_LOG_WARN, "failed to open %s", statm_path); + return; + } + + unsigned long vmSize, resident, share, text, lib, data; + + int nread; + nread = fscanf(f,"%ld %ld %ld %ld %ld %ld", &vmSize, &resident, &share, &text, &lib, &data); + fclose(f); + if (nread != 6) { + psLogMsg ("psphot", PS_LOG_WARN, "failed to read 6 items from %s", statm_path); + return; + } + + // XXX: assuming 4 KB page size here +# define PAGES_TO_MB(_v) (_v * 4.096 / 1024.) + psLogMsg ("psphot", PS_LOG_INFO, "Memory usage at %20s: Total VmSize: %8.2f MB Resident %8.2f MB Data: %8.2f MB\n", + heading, PAGES_TO_MB(vmSize), PAGES_TO_MB(resident), PAGES_TO_MB(data)); +} + +/* here is the process: + + * we have two image sets: + * RAW : unconvolved image stacks + + * OUT : psf-matched output image (there may be more than one of + * these. we will generate the first matched image by selecting the + * target PSF and doing a full psf-maching process (as used by ppStack + * and ppSub). But, additional target output files should use a + * simple gaussian convolution kernel determind from therms of the + * current and the target). + + * the output should be / could be one of the matched images, but not + * all. should we ensure the first gets written out, and ot save the + * others (or only optionally). + + * by default, we probably only sve the cmf ffile outputs. + + * load the RAW image (unconvolved stacks) + * add photcode to the output headers / readout->analysis + * generate mask and variance image (this is probably never needed in + practice: we always load an input mask & var. + * generate & subtract a model background for ?? (RAW? CNV? OUT? all?) + * load a PSF (probably not yet working) + + * generate the CHISQ image from the RAW input images (why save on OUT?) + + * find detections on RAW + + * copy detections to OUT + + * generate source stats (moments) for OUT + + * match sources across inputs (on OUT?) + + * generate source stats for the new constructions + + * rough class (star, galaxy, cosmic, etc) + + * Image quality + + * generate PSF + + * guess models + + * merge sources (new -> old) + + * linear fit to the psf + + * find ApResid + + * assign common positions + + * radial apertures (** this should be on the PSF-matched images + + * extended analysis (elliptical profile & petrosian) + + * extended fits (sersic, etc) + + * psphot magnitudes + + + ****** + + the above is all wrong: first, we should be doing the full + morphology analysis (ExtendedAnalysis & ExtendedFits) on the CNV or + RAW image (as desired optionally), etc. + + In the discussion below, 'BST' (best) means optionally RAW or CNV + + * detection : RAW & CHISQ (of RAW) + * moments : used by psf analysis & classification (BST) + * rough class : uses moments, not pixels + * image quality : uses moments as well + * generate PSF : (BST) + * guess models (BST) + * linear fit (BST) + * find ApResid (BST) -- uses sources not pixels + * extended analysis (BST) + * extended fits (BST) + * detection efficiency (BST) + + * somehow need to copy the sources so they point at the pixels on the + * OUT image + + * foreach target PSF + * radial aperture + * convolve to next target PSF + + * somehow need to organize the output file to have the values from + * the different PSFs in separate tables (with header info to + * specify the size of that PSF) + + */ + + +// generate a vector fwhmValues where the first has the fwhm of the raw image and the +// successive entries have the target values + +bool psphotStackMatchPSFsetup (pmConfig *config, const pmFPAview *view, const char *filerule, const char *filerulePSF) { + + bool status; + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + + if (!psphotStackMatchPSFsetupReadout (config, view, filerule, filerulePSF, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to define target PSF sizes"); + return false; + } + } + + return true; +} + +float psphotPSFseeing (pmPSF *psf, pmReadout *readout, int index); + +// copy the pixels from RAW to OUT ( + +bool psphotStackMatchPSFsetupReadout (pmConfig *config, const pmFPAview *view, const char *fileruleOut, const char *fileruleRaw, int index) { + + bool status; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + + // find the currently selected readout + pmFPAfile *fileOut = pmFPAfileSelectSingle(config->files, fileruleOut, index); // File of interest + psAssert (fileOut, "missing file?"); + + pmReadout *readoutOut = pmFPAviewThisReadout(view, fileOut->fpa); + psAssert (readoutOut, "missing readout?"); + + // find the currently selected readout + pmFPAfile *fileRaw = pmFPAfileSelectSingle(config->files, fileruleRaw, index); // File of interest + psAssert (fileRaw, "missing file?"); + + pmReadout *readoutRaw = pmFPAviewThisReadout(view, fileRaw->fpa); + psAssert (readoutRaw, "missing readout?"); + + readoutOut->image = psImageCopy(readoutOut->image, readoutRaw->image, PS_TYPE_F32); + if (readoutRaw->variance) { + readoutOut->variance = psImageCopy(readoutOut->variance, readoutRaw->variance, PS_TYPE_F32); + } + if (readoutRaw->mask) { + readoutOut->mask = psImageCopy(readoutOut->mask, readoutRaw->mask, PS_TYPE_IMAGE_MASK); + } + + // pmChip *chipRaw = pmFPAviewThisChip(view, fileRaw->fpa); // The chip holds the PSF + // psAssert (chipRaw, "missing chip"); + + pmPSF *psf = psMetadataLookupPtr(&status, readoutRaw->analysis, "PSPHOT.PSF"); // PSF + if (!psf) { + // we should have a PSF by this point in psphot + psError(PSPHOT_ERR_PROG, true, "Unable to find PSF."); + return false; + } + + float fwhmRaw = psphotPSFseeing (psf, readoutRaw, index); + + psVector *fwhmValues = psVectorAllocEmpty(10, PS_TYPE_F32); + psVectorAppend(fwhmValues, fwhmRaw); + + // is a single target FWHM specified, or a set of values? set up the vector options->targetSeeing and the local 1st value + float targetSeeing = psMetadataLookupF32 (&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); + if (!status) { + psVector *targetSeeing = psMetadataLookupVector(&status, recipe, "PSPHOT.STACK.TARGET.PSF.FWHM"); // Magnitude offsets + psAssert (status, "missing psphot recipe value PSPHOT.STACK.TARGET.PSF.FWHM"); + for (int i = 0; i < targetSeeing->n; i++) { + psVectorAppend(fwhmValues, targetSeeing->data.F32[i]); + } + } else { + psVectorAppend(fwhmValues, targetSeeing); + } + + psMetadataAddVector(readoutOut->analysis, PS_LIST_TAIL, "STACK.PSF.FWHM.VALUES", PS_META_REPLACE, "PSF sizes", fwhmValues); + psFree (fwhmValues); + + return true; +} + +float psphotPSFseeing (pmPSF *psf, pmReadout *readout, int index) { + + psImage *image = readout->image; + + int Nx = image->numCols; + int Ny = image->numRows; + + float sumFWHM = 0.0; // FWHM for image + int numFWHM = 0; // Number of FWHM measurements + for (float x = 0; x < Nx; x += 0.25*Nx) { + for (float y = 0; y < Ny; y += 0.25*Ny) { + float fwhm = pmPSFtoFWHM(psf, x, y); + if (isfinite(fwhm)) { + sumFWHM += fwhm; + numFWHM++; + } + } + } + if (numFWHM == 0) { + psLogMsg("ppStack", PS_LOG_INFO, "Unable to measure PSF FWHM for image %d --- rejected.", index); + return NAN; + } + + float fwhm = sumFWHM / (float) numFWHM; + psLogMsg ("psphotStack", PS_LOG_INFO, "Input Seeing for %d: %f\n", index, fwhm); + + return fwhm; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackUpdateReadout.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackUpdateReadout.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStackUpdateReadout.c (revision 41083) @@ -0,0 +1,524 @@ +# include "psphotInternal.h" + +static bool psphotStackUpdateLoadSources (pmConfig *config, const pmFPAview *view, const char *filerule); +bool psphotDumpImages (pmConfig *config, const pmFPAview *view, const char *filerule, char *base); + +// relevant filesets: +# define STACK_RAW "PSPHOT.STACK.INPUT.RAW" +# define STACK_OUT "PSPHOT.STACK.OUTPUT.IMAGE" + +bool psphotStackUpdateReadout (pmConfig *config, const pmFPAview *view) { + + psArray *objects = NULL; + + // measure the total elapsed time in psphotReadout. dtime is the elapsed time used jointly + // by the multiple threads, not the total time used by all threads. + psTimerStart ("psphotReadout"); + + pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); // allow models to have ugly fits (eg, central cusp) + + // select the current recipe + psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); + if (!recipe) { + psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE); + return false; + } + // optional break-point for processing + char *breakPt = psMetadataLookupStr (NULL, recipe, "BREAK_POINT"); + psAssert (breakPt, "configuration error: set BREAK_POINT"); + + // load WCS + if (!psphotStackLoadWCS(config, view, STACK_RAW)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble loading WCS for %s", STACK_RAW); + return false; + } + + // set the photcode for each image + if (!psphotAddPhotcode (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_CONFIG, false, "trouble defining the photcode"); + return false; + } + + // Generate the mask and weight images (if not supplied) and set mask bits. + if (!psphotSetMaskAndVariance (config, view, STACK_RAW)) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "NOTHING")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + if (!psphotLoadBackgroundModel (config, view, STACK_RAW)) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!psphotSubtractBackground (config, view, STACK_RAW)) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "BACKMDL")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // Load the psf from the previous run. + // Note this must be done before we load the sources because it is used. + // I believe only to set the residuals + if (!psphotLoadPSF (config, view, STACK_RAW)) { + // this only happens if we had a programming error in psphotLoadPSF + psError (PSPHOT_ERR_UNKNOWN, false, "error loading psf model"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "PSFMODEL")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // Copy the sources loaded to the sourcesReadout to STACK_RAW, updating various + // measurements which are not recorded in the cmf files + if (!psphotStackUpdateLoadSources (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_UNKNOWN, false, "error finishing sources"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "UPDATELOADEDSOURCES")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + if (!psphotDeblendSatstars (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed on satstar deblend analysis"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + if (!psphotMergeSources (config, view, STACK_RAW)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to merge sources"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + if (!psphotRemoveAllSources (config, view, STACK_RAW, false)) { + psError (PSPHOT_ERR_UNKNOWN, false, "error subtracting sources"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "REMOVESOURCES")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + // Connect loaded sources back into Objects + objects = psphotLinkSources (config, view, STACK_RAW); + if (!objects) { + psError (PSPHOT_ERR_UNKNOWN, false, "error linking sources into objects"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + if (!strcasecmp (breakPt, "LINKSOURCES")) { + psFree(objects); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + // XXX: Do we need to choose analysis options? + // How about other tmpFlags? + + // create source children for the OUT filerule (for radial aperture photometry and output) + // NOTE: The new source children have image arrays pointing to the readout associated with + // STACK_OUT. in psphotStackMatchPSFsetup, we copy the current pixel values from RAW to OUT, + // but keep the pointers the same so we do not break these source image references + // XXX NOTE : if we use the pre-20130914 psphotStackReadout code, we need to use 'false' for the + // sourcesSubtracted argument + // XXX: do this here so that the variance is availble to rebuild the models + psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); + psArray *objectsOut = psphotSourceChildrenByObject (config, view, STACK_OUT, STACK_RAW, objects, true); + if (!objectsOut) { + psFree(objects); + psError (PSPHOT_ERR_UNKNOWN, false, "failure in peak analysis"); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + // These arrays are no longer needed. The inputs' sources arrays have references to the data + psFree (objects); + psFree (objectsOut); + + if (!strcasecmp (breakPt, "SOURCECHILDREN")) { + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + bool radial_apertures = psMetadataLookupBool(NULL, recipe, "RADIAL_APERTURES"); + if (radial_apertures) { + // measure circular, radial apertures (objects sorted by S/N) + // this forces photometry on the undetected sources from other images + +// Moved this up above +// psphotStackMatchPSFsetup (config, view, STACK_OUT, STACK_RAW); + psphotDumpImages (config, view, STACK_RAW, "raw.t0"); + psphotDumpImages (config, view, STACK_OUT, "out.t0"); + + int nRadialEntries = psphotStackMatchPSFsEntries(config, view, STACK_OUT); + + for (int entry = 0; entry < nRadialEntries; entry++) { + // NOTE: entry 0 is the unmatched image set + + char line[256]; + + // measure circular, radial apertures (objects sorted by S/N) + psphotRadialApertures (config, view, STACK_OUT, entry); + snprintf (line, 256, "%s.%d", "out.t1", entry); + psphotDumpImages (config, view, STACK_OUT, line); + + // replace the flux in the image so it is returned to its original state + psphotReplaceAllSources (config, view, STACK_OUT, false); + snprintf (line, 256, "%s.%d", "out.t2", entry); + psphotDumpImages (config, view, STACK_OUT, line); + + if (entry < nRadialEntries - 1) { + // smooth to the next FWHM + // this function does nothing if the targetFWHM is smaller than the currentFWHM + psphotStackMatchPSFsNext(config, view, STACK_OUT, entry); + snprintf (line, 256, "%s.%d", "out.t3", entry); + psphotDumpImages (config, view, STACK_OUT, line); + psMemDump("matched"); + + // re-measure the PSF for the smoothed image (using entries in 'allSources') + if (!psphotChoosePSF (config, view, STACK_OUT, false)) { + psLogMsg ("psphot", 3, "failure to construct a psf model in radial aperture loop for entry :%d", entry); + return psphotReadoutCleanup (config, view, STACK_RAW); + } + + // this is necessary to update the models based on the new PSF + psphotResetModels (config, view, STACK_OUT); + + // this is necessary to get the right normalization for the new models + // and to subtract the sources + psphotFitSourcesLinear (config, view, STACK_OUT, false, false); + +#ifdef notyet + psphotRemoveAllSources (config, view, STACK_OUT, false); +#endif + + snprintf (line, 256, "%s.%d", "out.t4", entry); + psphotDumpImages (config, view, STACK_OUT, line); + } + } + } + + psphotCopyEfficiency (config, view, STACK_OUT, STACK_RAW); + + // replace background in residual image + psphotSkyReplace (config, view, STACK_RAW); + + // drop the references to the image pixels held by each source + psphotSourceFreePixels (config, view, STACK_RAW); + psphotSourceFreePixels (config, view, STACK_OUT); + + // create the exported-metadata and free local data + return psphotReadoutCleanup (config, view, STACK_RAW); +} + +static void copyHeaderValues (psMetadata *analysis, psMetadata *sourcesHeader) { + // copy parameters from measurment of background model from input soources file header + // to the analysis. +#define COPY_PARAM(_key) \ + psMetadataAddF32(analysis, PS_LIST_TAIL, _key, PS_META_REPLACE, _key, \ + psMetadataLookupF32(NULL, sourcesHeader, _key) ) + + // These three aren't saved to the header. These values should be close. + // XXX: right? + + psMetadataAddF32(analysis, PS_LIST_TAIL, "SKY_MEAN", PS_META_REPLACE, "SKY_MEAN", + psMetadataLookupF32(NULL, sourcesHeader, "MSKY_MN" )); + psMetadataAddF32(analysis, PS_LIST_TAIL, "SKY_STDEV", PS_META_REPLACE, "SKY_STDEV", + psMetadataLookupF32(NULL, sourcesHeader, "MSKY_DEV" )); + psMetadataAddF32(analysis, PS_LIST_TAIL, "SKY_DQ", PS_META_REPLACE, "SKY_DQ", + psMetadataLookupF32(NULL, sourcesHeader, "MSKY_DQ" )); + + COPY_PARAM( "MSKY_MN"); + COPY_PARAM( "MSKY_SIG" ); + COPY_PARAM( "MSKY_DEV" ); + COPY_PARAM( "MSKY_DQ" ); + COPY_PARAM( "MSKY_MAX" ); + COPY_PARAM( "MSKY_MIN" ); + COPY_PARAM( "MSKY_NX" ); + COPY_PARAM( "MSKY_NY" ); + // COPY_PARAM( "CHIP_SEEING" ); +} + +static bool psphotStackUpdateLoadSourcesReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index) { + bool status = true; + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + psAssert (file, "missing file?"); + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); + psAssert (readout, "missing readout?"); + + pmFPAfile *sourcesFile = pmFPAfileSelectSingle(config->files, "PSPHOT.STACK.SOURCES", index); + pmReadout *sourcesReadout = pmFPAviewThisReadout(view, sourcesFile->fpa); + psAssert (sourcesReadout, "missing sources readout?"); + + pmDetections *detectionsIn = psMetadataLookupPtr (&status, sourcesReadout->analysis, "PSPHOT.DETECTIONS"); + psAssert (detectionsIn, "missing input detections?"); + + psArray *sources = detectionsIn->allSources; + psAssert (sources, "missing sources?"); + + pmDetections *detections = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.DETECTIONS"); + if (!detections) { + detections = pmDetectionsAlloc(); + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PSPHOT.DETECTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "psphot detections", + detections); + psFree(detections); + } else { + // What is this doing here? + psAssert (detections == NULL, "not expecting detections to exist already. watsup?"); + } + + // copy sources from input detections + detections->newSources = psMemIncrRefCounter (sources); + + if (!sources->n) { + psLogMsg ("psphot", PS_LOG_INFO, "no sources, skipping sources update"); + return true; + } + + pmDetEff *de = psMetadataLookupPtr(&status, sourcesReadout->analysis, PM_DETEFF_ANALYSIS); // Detection efficiency + if (de) { + if (!psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, PM_DETEFF_ANALYSIS, PS_META_REPLACE | PS_DATA_UNKNOWN, + "Detection efficiency", de)) { + psError (PSPHOT_ERR_CONFIG, false, "problem saving Detection efficiency on readout"); + return false; + } + } + + psMetadata *sourcesHeader = psMetadataLookupPtr(&status, sourcesReadout->analysis, "INPUT.SOURCES.HEADER"); + psAssert (sourcesHeader, "missing sourcesHeader?"); + copyHeaderValues(readout->analysis, sourcesHeader); + + pmPSF *psf = psMetadataLookupPtr (&status, readout->analysis, "PSPHOT.PSF"); + psAssert (psf, "missing psf?"); + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + assert (recipe); + + // bit-masks to test for good/bad pixels + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); + assert (maskVal); + + // bit-mask to mark pixels not used in analysis + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); + assert (markVal); + + // maskVal is used to test for rejected pixels, and must include markVal + maskVal |= markVal; + + // This code currently assumes that all extended source models are PCM. + // Examine the recipe and make sure that this is true. + psMetadata *allModels = psMetadataLookupMetadata (&status, recipe, "EXTENDED_SOURCE_MODELS"); + if (!status) { + psAssert (allModels, "failed to find list of extended source models\n"); + } + psMetadataIterator *iter = psMetadataIteratorAlloc (allModels, PS_LIST_HEAD, NULL); + psMetadataItem *item = NULL; + while ((item = psMetadataGetAndIncrement (iter)) != NULL) { + if (item->type != PS_DATA_METADATA) { + psAbort ("Invalid type for EXTENDED_SOURCE_MODEL entry %s, not a metadata folder", item->name); + } + psMetadata *model = (psMetadata *) item->data.md; + // char *modelName = psMetadataLookupStr (&status, model, "MODEL"); + + char *convolvedWord = psMetadataLookupStr (&status, model, "PSF_CONVOLVED"); + if (!status || (strcasecmp (convolvedWord, "true") && strcasecmp (convolvedWord, "false"))) { + psAbort ("PSF_CONVOLVED entry invalid or missing for EXTENDED_SOURCE_MODEL entry %s", item->name); + } + bool convolved = !strcasecmp (convolvedWord, "true"); + psAssert (convolved, "EXTENDED SOURCE model %s is not convolved. This code is not ready for that\n", item->name); + } + psFree (iter); + + // setup the PSF fit radius details + psphotInitRadiusPSF (recipe, readout); + // and for extended source fitting + psphotInitRadiusEXT (recipe, readout); + + // not really fitting but need to get some things set up to instantiate the pcm models + + pmSourceFitOptions *fitOptions = pmSourceFitOptionsAlloc(); + fitOptions->mode = PM_SOURCE_FIT_EXT_AND_SKY; + fitOptions->covarFactor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); // Covariance matrix + // XXX: change all of this status checking to asserts. All should be defined with the current recipe. + float fitNsigmaConv = psMetadataLookupF32 (&status, recipe, "EXT_FIT_NSIGMA_CONV"); // number of sigma for the convolution + if (!status || !isfinite(fitNsigmaConv) || fitNsigmaConv <= 0) { + fitNsigmaConv = 5.0; + } + fitOptions->nsigma = fitNsigmaConv; + // Poisson or Constant weight for chisq tests? + fitOptions->poissonErrors = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_FITS_POISSON"); + if (!status) fitOptions->poissonErrors = true; + + int psfSize = psMetadataLookupS32 (&status, recipe, "PCM_BOX_SIZE"); + assert (status); + + // Remember these data for later + psMetadataAddPtr (readout->analysis, PS_LIST_TAIL, "PCM_FIT_OPTIONS", PS_DATA_UNKNOWN | PS_META_REPLACE, "pcm fit options", fitOptions); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + source->imageID = index; // XXX: This is usually set in psphotSourceStatsReadout + pmModel *modelPSF = source->modelPSF; + // free any previous radial aperture measurements + psFree(source->radialAper); + + // Guess Models ususally does this + if (modelPSF) { + psphotCheckRadiusPSF (readout, source, modelPSF, markVal); + source->modelPSF->residuals = psf->residuals; + } + + if (!(source->mode & PM_SOURCE_MODE_PSFMODEL)) { + // The cmf loaders unconditionallly create psf models even if the source did not have one. + // fprintf(stderr, "source %d %5d should not have a psf model 0x%08X%08X\n", index, source->seq, source->mode2, source->mode); + psFree(source->modelPSF) ; + modelPSF = NULL; + // This bit was set once even though there was no model + source->mode &= ~(PM_SOURCE_MODE_PSFSTAR); + // At least once I saw a source with an extended model but no psf model + psFree(source->modelEXT); + source->modelEXT = NULL; + } + + // make sure that the window radius is large enough. + // Should we do this regardless of whether or not the source is extended? + float fitRadius = NAN; + float windowRadius = NAN; + if (!psphotSetRadiusMoments (&fitRadius, &windowRadius, readout, source, markVal)) { + psError (PSPHOT_ERR_UNKNOWN, false, "failed to set radius for ext source"); + return false; + } + // We are getting some sources near the edge of skycells which are marked as having + // been fit with saturated star model but have peak coordinates far outside the + // image. When this happens an error is left on the stack. + psErrorClear(); + + // If we find a good model we will cache it below. + bool goodModel = false; + + // the cmf readers do not set PM_PAR_I0 for the models so we calculate it here. + // We may want to put this code in the cmf readers but for now I don't want to change psphot operation + // for anything except psphotStack -updatemode. + if (modelPSF && isfinite(source->psfFlux)) { + // The cmf reader was incorrectly setting the sky parameter for psf models + modelPSF->params->data.F32[PM_PAR_SKY] = 0.0; + modelPSF->dparams->data.F32[PM_PAR_SKY] = 0.0; + + // Calculate flux for normalized model + modelPSF->params->data.F32[PM_PAR_I0] = 1.0; + float normFlux = modelPSF->class->modelFlux(modelPSF->params); + float psfFlux = source->psfFlux; + // back out the ApTrend + double apTrend = pmTrend2DEval (psf->ApTrend, (float)source->peak->x, (float)source->peak->y); + double adjustment = pow(10.0, -0.4*apTrend); + psAssert ( (isfinite(adjustment) && adjustment != 0.0), "Invalid adjustment value: %f", adjustment); + + psfFlux = psfFlux / adjustment; + float newI0 = psfFlux / normFlux; + + psAssert(isfinite (newI0) , "got infinite I0 for psf model"); + + // new I0 for psf model looks good. + modelPSF->params->data.F32[PM_PAR_I0] = newI0; + goodModel = true; + } + + // It is rather expensive setting the I0 all of the extended models. Lets only initialize + // the extended model if it will be used for subtraction. + + bool isPSF = false; + pmModel *model = pmSourceGetModel (&isPSF, source); + if (model && !isPSF) { + // selected model is extended + goodModel = false; + // calculate flux from the previously measured model magnitude. It will be nan if the + // flux was negative unfortunately so we lose those. + if (isfinite(model->mag)) { + float modelFlux = pow(10., -0.4 * model->mag); + + model->fitRadius = fitRadius; + + // XXX: We are assuming here that we only use PCM models. + // I Should be getting that information from the recipe. + pmPCMdata *pcm = pmPCMinit (source, fitOptions, model, maskVal, psfSize); + if (pcm) { + model->params->data.F32[PM_PAR_I0] = 1.0; + float normFlux = model->class->modelFlux(model->params); + float I0 = modelFlux / normFlux; + if (isfinite(I0)) { + model->params->data.F32[PM_PAR_I0] = I0; + goodModel = true; + // Usually this it is set when doing the fit. Since we are instantiating the model we + // set it explicitly here. + model->isPCM = true; + } + psFree(pcm); + } else { + // psError (PSPHOT_ERR_UNKNOWN, false, "failed to to initialize PCM for model"); + // return false; + // fall through + } + } + if (!goodModel) { + // We do not have a usable extended model for this source. + // The original flux was probably negative. + // This is somewhat rare. I saw it in about .2% of extended sources in my first test. + // Set source type to star to prevent source subtraction from crashing by trying to use + // an incomplete extended model. + // XXX: this may cause model psf to be used in places. + // XXX However we aren't going to cache it below so... + source->type = PM_SOURCE_TYPE_STAR; + psFree(source->modelEXT); + source->modelEXT = NULL; + model = NULL; + } + } + + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + // XXX: hacking here. Psf fitting can't find any stars unless moments->nPixels is above + // some minimum. We don't measure the moments and this value is not saved in the cmf. + // Is this a reasonable guess? + source->moments->nPixels = source->modelPSF->nPix; + source->tmpFlags |= PM_SOURCE_TMPF_CANDIDATE_PSFSTAR; + } + + // psphotDeblendSatstars needs a value for signal to noise. + psF32 kronFlux = source->moments->KronFlux; + source->moments->SN = 0; + if (isfinite(kronFlux) && isfinite(source->moments->KronFluxErr) && isfinite(source->psfMag)) { + // This is how we set the SN column in the CFF files... + source->moments->SN = kronFlux/source->moments->KronFluxErr; + } else if (isfinite(source->psfFlux) && isfinite(source->psfFluxErr)) { + // kron no good try this. It's just used for sorting + source->moments->SN = source->psfFlux/source->psfFluxErr; + } + + if (goodModel && model) { + // Yipee! cache the model + if (model->isPCM) { + pmPCMCacheModel (source, maskVal, psfSize, fitNsigmaConv); + } else { + pmSourceCacheModel (source, maskVal); + } + } + } + + psFree(fitOptions); + + return true; +} + +static bool psphotStackUpdateLoadSources (pmConfig *config, const pmFPAview *view, const char *filerule) { + bool status = true; + + int num = psphotFileruleCount(config, filerule); + + // skip the chisq image (optionally?) + int chisqNum = psMetadataLookupS32 (&status, config->arguments, "PSPHOT.CHISQ.NUM"); + if (!status) chisqNum = -1; + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (i == chisqNum) continue; // skip chisq image + if (!psphotStackUpdateLoadSourcesReadout (config, view, filerule, i)) { + psError (PSPHOT_ERR_CONFIG, false, "failed on to guess models for %s entry %d", filerule, i); + return false; + } + } + return true; + +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStandAlone.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStandAlone.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStandAlone.h (revision 41083) @@ -0,0 +1,21 @@ +# ifdef HAVE_CONFIG_H +# include +# endif + +#ifndef PSPHOT_STAND_ALONE_H +#define PSPHOT_STAND_ALONE_H + +#include +#include +#include +#include "psphot.h" +#include "psphotStatsFile.h" + +// Top level functions +pmConfig *psphotArguments (int argc, char **argv); +bool psphotParseCamera (pmConfig *config); +bool psphotMosaicChip(pmConfig *config, const pmFPAview *view, char *outFile, char *inFile); +void psphotCleanup (pmConfig *config); +psExit psphotGetExitStatus (void); + +#endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStatsFile.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStatsFile.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStatsFile.c (revision 41083) @@ -0,0 +1,111 @@ +/** @file psphotStatsFile.c + * + * @brief functions for managing the stats file (or return NULL, or exit with an error) + * @ingroup psphot + * + * @author IfA + * @version $Revision: 1.38 $ $Name: not supported by cvs2svn $ + * @date $Date: 2009-02-13 21:54:32 $ + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#include "psphotStandAlone.h" + +static psphotStatsFile *theStatsFile = NULL; + +static void psphotStatsFileFree (psphotStatsFile *statsFile) { + psFree (statsFile->name); + psFree (statsFile->md); + theStatsFile = NULL; +} + +psphotStatsFile *psphotStatsFileAlloc () { + + psphotStatsFile *statsFile = psAlloc(sizeof(psphotStatsFile)); + psMemSetDeallocator(statsFile, (psFreeFunc)psphotStatsFileFree); + + statsFile->f = NULL; + statsFile->name = NULL; + statsFile->md = NULL; + + return statsFile; +} + +// Open the statistics file +psphotStatsFile *psphotStatsFileOpen (pmConfig *config) { + + bool mdok; + + char *name = psMetadataLookupStr(&mdok, config->arguments, "STATS"); ///< Filename for statistics + + if (!name) return NULL; + + psphotStatsFile *statsFile = psphotStatsFileAlloc (); + statsFile->name = psMemIncrRefCounter(name); + + if (!mdok) return statsFile; // XXX this is probably a config error, but treat as ok for now + if (!name) return statsFile; + if (strlen(name) == 0) return statsFile; + + psString resolved = pmConfigConvertFilename(name, config, true, true); + + statsFile->f = fopen(resolved, "w"); + if (!statsFile->f) { + psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); + psFree(resolved); + } + psFree(resolved); + + statsFile->md = psMetadataAlloc(); + psMetadataAddS32(statsFile->md, PS_LIST_TAIL, "QUALITY", 0, "No problems", 0); + + // the global copy + theStatsFile = statsFile; + + return statsFile; +} + +bool psphotStatsFileSave (pmConfig *config, psphotStatsFile *statsFile) { + + if (!statsFile) return true; + if (!statsFile->md) return true; + + // Write out summary statistics +// psMetadataAddF32(statsFile->md, PS_LIST_TAIL, "DT_WARP", 0, "Time for warp completion", psTimerMark("psphot")); + + // convert the psMetadata to a string block + const char *statsMDC = psMetadataConfigFormat(statsFile->md); + if (!statsMDC) { + psError(psErrorCodeLast(), false, "Unable to get statistics file."); + return false; + } + + if (fprintf(statsFile->f, "%s", statsMDC) != strlen(statsMDC)) { + psError(PSPHOT_ERR_IO, true, "Unable to write statistics file."); + return false; + } + psFree(statsMDC); + + if (fclose(statsFile->f) == EOF) { + psError(PSPHOT_ERR_IO, true, "Unable to close statistics file."); + statsFile->f = NULL; + return false; + } + statsFile->f = NULL; + + pmConfigRunFilenameAddWrite(config, "STATS", statsFile->name); + psFree(statsFile); + + return true; +} + +psphotStatsFile *psphotStatsFileGet () { + return theStatsFile; +} + +void psphotStatsFileSetQuality (int quality) { + psphotStatsFile *statsFile = psphotStatsFileGet(); + if (!statsFile) return; + + psMetadataAddS32(statsFile->md, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "quality value", quality); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStatsFile.h =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStatsFile.h (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotStatsFile.h (revision 41083) @@ -0,0 +1,16 @@ +#ifndef PSPHOT_STATS_FILE_H +#define PSPHOT_STATS_FILE_H + + +typedef struct { + FILE *f; // File stream for statistics + char *name; // Filename for statistics + psMetadata *md; // Container for statistics +} psphotStatsFile; + +psphotStatsFile *psphotStatsFileOpen (pmConfig *config); +psphotStatsFile *psphotStatsFileGet (); +bool psphotStatsFileSave (pmConfig *config, psphotStatsFile *statsFile); +void psphotStatsFileSetQuality (int quality); + +#endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSubtractBackground.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSubtractBackground.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSubtractBackground.c (revision 41083) @@ -0,0 +1,135 @@ +# include "psphotInternal.h" +static int npass = 0; + +// generate the median in NxN boxes, clipping heavily +// linear interpolation to generate full-scale model +bool psphotSubtractBackgroundReadout (pmConfig *config, const pmFPAview *view, const char *filerule, int index, psMetadata *recipe) +{ + bool status = true; + pmReadout *background = NULL; + pmReadout *backSub = NULL; + + psTimerStart ("psphot.background"); + + // find the currently selected readout + pmFPAfile *file = pmFPAfileSelectSingle(config->files, filerule, index); // File of interest + + pmFPA *inFPA = file->fpa; + pmReadout *readout = pmFPAviewThisReadout (view, inFPA); + psImage *image = readout->image; + psImage *mask = readout->mask; + + // find the currently selected readout (XXX note that the model is saved on PSPHOT.BACKMDL regardless of 'filename' + pmFPAfile *modelFile = pmFPAfileSelectSingle(config->files, psphotGetFilerule("PSPHOT.BACKMDL"), index); // File of interest + assert (modelFile); + + pmReadout *model = READOUT_OR_INTERNAL(view, modelFile); + assert (model); + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + psAssert (maskVal, "MASK.PSPHOT missing from recipe"); + + psImageBinning *binning = psMetadataLookupPtr(&status, model->analysis, "PSPHOT.BACKGROUND.BINNING"); + assert (binning); + + // select background pixels, from output background file, or create + // XXX for now, we will only allow a single background image to be generated + file = psMetadataLookupPtr (&status, config->files, psphotGetFilerule("PSPHOT.BACKGND")); + if (file) { + // we are using PSPHOT.BACKGND as an I/O file: select readout or create + background = READOUT_OR_INTERNAL(view, file); + if (background == NULL) { + // readout does not yet exist: create from input + pmFPAfileCopyStructureView (file->fpa, inFPA, 1, 1, view); + background = pmFPAviewThisReadout (view, file->fpa); + if ((image->numCols != background->image->numCols) || (image->numRows != background->image->numRows)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for background dimensions"); + return false; + } + } + } else { + background = pmFPAfileDefineInternal (config->files, psphotGetFilerule("PSPHOT.BACKGND"), image->numCols, image->numRows, PS_TYPE_F32); + } + psF32 **backData = background->image->data.F32; + + // linear interpolation to full-scale + if (!psImageUnbin (background->image, model->image, binning)) { + psError (PSPHOT_ERR_PROG, true, "inconsistent sizes for unbinning"); + return false; + } + + psLogMsg ("psphot", PS_LOG_MINUTIA, "build resampled image: %f sec\n", psTimerMark ("psphot.background")); + + // back-sub image pixels, from output background file (don't create if not requested) + // XXX for now, we will only allow a single background-subtracted image to be generated + file = psMetadataLookupPtr (&status, config->files, psphotGetFilerule("PSPHOT.BACKSUB")); + if (file) { + // we are using PSPHOT.BACKSUB as an I/O file: select readout or create + backSub = pmFPAviewThisReadout (view, file->fpa); + if (backSub == NULL) { + // readout does not yet exist: create from input + pmFPAfileCopyStructureView (file->fpa, inFPA, 1, 1, view); + backSub = pmFPAviewThisReadout (view, file->fpa); + } + } + + if (psTraceGetLevel("psphot") > 5) { + char name[256]; + sprintf (name, "image.%02d.fits", npass); + psphotSaveImage (NULL, image, name); + sprintf (name, "back.%02d.fits", npass); + psphotSaveImage (NULL, background->image, name); + sprintf (name, "mask.%02d.fits", npass); + psphotSaveImage (NULL, mask, name); + sprintf (name, "backmdl.%02d.fits", npass); + psphotSaveImage (NULL, model->image, name); + } + + // subtract the background model (save in backSub, if requested) + // XXX if needed, multithread this (fairly trivial) + for (int j = 0; j < image->numRows; j++) { + for (int i = 0; i < image->numCols; i++) { + image->data.F32[j][i] -= backData[j][i]; + if (backSub) { + backSub->image->data.F32[j][i] = image->data.F32[j][i]; + } + } + } + + if (psTraceGetLevel("psphot") > 5) { + char name[256]; + sprintf (name, "backsub.%02d.fits", npass); + psphotSaveImage (NULL, image, name); + } + psLogMsg ("psphot", PS_LOG_WARN, "subtracted background model: %f sec\n", psTimerMark ("psphot.background")); + + // the pmReadout selected in this function are all view on entries in config->files + + // display the backsub and backgnd images + // move this inthe the subtract background loop + psphotVisualShowBackground (config, view, readout); + + npass ++; + return true; +} + +bool psphotSubtractBackground (pmConfig *config, const pmFPAview *view, const char *filerule) +{ + bool status = false; + + // select the appropriate recipe information + psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE); + psAssert (recipe, "missing recipe?"); + + int num = psphotFileruleCount(config, filerule); + + // loop over the available readouts + for (int i = 0; i < num; i++) { + if (!psphotSubtractBackgroundReadout (config, view, filerule, i, recipe)) { + psError (PSPHOT_ERR_CONFIG, false, "failed to subtract background for %s entry %d", filerule, i); + return false; + } + } + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSummaryPlots.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSummaryPlots.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotSummaryPlots.c (revision 41083) @@ -0,0 +1,280 @@ +# include "psphotInternal.h" + +// the top portion of this file defines plotting functions which use kapa for plotting. +// if kapa is not available, these functions are defined in the bottom portion as stubs +// which perform NOP and return false (XXX should this be true??) + +// this variable is defined in psmodules.h if ohana-config is found +# if (HAVE_KAPA) + +# include + +// plot the sx, sy moments plane (faint and bright sources) +bool psphotPlotMoments (pmConfig *config, pmFPAview *view, psArray *sources) { + + // select model pixels (from output background model file, or create internal file) + pmFPAfile *file = psMetadataLookupPtr (NULL, config->files, "PSPHOT.MOMENT.PLT"); + if (file == NULL) { + psLogMsg ("psphot", 3, "skipping moments plot"); + return false; + } + + // pmFPAfileOpen defers disk I/O for KAPA files: just get the correct name + pmFPAfileOpen (file, view, config); + + Graphdata graphdata; + + psLogMsg ("psphot", 3, "creating moments plot"); + + // XXX get the 'showWindow' option from the recipes somewhere + int kapa = pmKapaOpen (false); + if (kapa == -1) { + psError(PSPHOT_ERR_UNKNOWN, true, "failure to open kapa"); + return false; + } + + KapaResize (kapa, 500, 500); + KapaInitGraph (&graphdata); + + // examine sources to set data range + graphdata.xmin = -0.05; + graphdata.ymin = -0.05; + graphdata.xmax = +2.05; + graphdata.ymax = +2.05; + KapaSetLimits (kapa, &graphdata); + + KapaSetFont (kapa, "helvetica", 14); + KapaBox (kapa, &graphdata); + KapaSendLabel (kapa, "&ss&h_x| (pixels)", KAPA_LABEL_XM); + KapaSendLabel (kapa, "&ss&h_y| (pixels)", KAPA_LABEL_YM); + + psVector *xBright = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yBright = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *xFaint = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yFaint = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + // construct the vectors + int nB = 0; + int nF = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->moments == NULL) continue; + + xFaint->data.F32[nF] = source->moments->Mxx; + yFaint->data.F32[nF] = source->moments->Myy; + nF++; + + // XXX make this a user-defined cutoff + if (source->moments->SN < 25) continue; + + xBright->data.F32[nB] = source->moments->Mxx; + yBright->data.F32[nB] = source->moments->Myy; + nB++; + } + xFaint->n = nF; + yFaint->n = nF; + + xBright->n = nB; + yBright->n = nB; + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (kapa, nF, &graphdata); + KapaPlotVector (kapa, nF, xFaint->data.F32, "x"); + KapaPlotVector (kapa, nF, yFaint->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (kapa, nB, &graphdata); + KapaPlotVector (kapa, nB, xBright->data.F32, "x"); + KapaPlotVector (kapa, nB, yBright->data.F32, "y"); + + psLogMsg ("psphot", 3, "saving plot to %s", file->filename); + KapaPNG (kapa, file->filename); + + psFree (xBright); + psFree (yBright); + psFree (xFaint); + psFree (yFaint); + + return true; +} + +// plot the sx, sy, sxy as vector field, +// plot the PSF measured sx, sy, sxy as vector field +// pull the sources from the config / file? +bool psphotPlotPSFModel (pmConfig *config, pmFPAview *view, psArray *sources) { + + // select model pixels (from output background model file, or create internal file) + pmFPAfile *file = psMetadataLookupPtr (NULL, config->files, "PSPHOT.PSFMODEL.PLT"); + if (file == NULL) { + psLogMsg ("psphot", 3, "skipping psf model plot"); + return false; + } + + // pmFPAfileOpen defers disk I/O for KAPA files: just get the correct name + pmFPAfileOpen (file, view, config); + + Graphdata graphdata; + + psLogMsg ("psphot", 3, "creating psf model plot"); + + int kapa = pmKapaOpen (false); + if (kapa == -1) { + psError(PSPHOT_ERR_UNKNOWN, true, "failure to open kapa"); + return false; + } + + // XXX make the aspect-ratio match the image + KapaResize (kapa, 800, 800); + KapaInitGraph (&graphdata); + + psVector *xMNT = psVectorAllocEmpty (2*sources->n, PS_TYPE_F32); + psVector *yMNT = psVectorAllocEmpty (2*sources->n, PS_TYPE_F32); + psVector *xPSF = psVectorAllocEmpty (2*sources->n, PS_TYPE_F32); + psVector *yPSF = psVectorAllocEmpty (2*sources->n, PS_TYPE_F32); + psVector *xMIN = psVectorAllocEmpty (2*sources->n, PS_TYPE_F32); + psVector *yMIN = psVectorAllocEmpty (2*sources->n, PS_TYPE_F32); + + // construct the plot vectors + int nMNT = 0; + int nPSF = 0; + int nMIN = 0; + float dx = 0; + float dy = 0; + float scale = 10; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->moments == NULL) continue; + if (source->moments->SN < 25) continue; + if (source->type != PM_SOURCE_TYPE_STAR) continue; + + pmModel *model = source->modelPSF; + if (model == NULL) continue; + + psF32 *PAR = model->params->data.F32; + + psEllipseMoments moments; + moments.x2 = source->moments->Mxx; + moments.xy = source->moments->Mxy; + moments.y2 = source->moments->Myy; + + psEllipseShape shape; + shape.sx = PAR[PM_PAR_SXX] / sqrt(2.0); + shape.sy = PAR[PM_PAR_SYY] / sqrt(2.0); + shape.sxy = PAR[PM_PAR_SXY]; + + // force the axis ratio to be < 20.0 + psEllipseAxes axes_mnt = psEllipseMomentsToAxes (moments, 20.0); + psEllipseAxes axes_psf = psEllipseShapeToAxes (shape, 20.0); + + // moments major axis + dx = scale*axes_mnt.major*cos(axes_mnt.theta); + dy = scale*axes_mnt.major*sin(axes_mnt.theta); + xMNT->data.F32[nMNT] = PAR[PM_PAR_XPOS] - dx; + yMNT->data.F32[nMNT] = PAR[PM_PAR_YPOS] - dy; + nMNT++; + xMNT->data.F32[nMNT] = PAR[PM_PAR_XPOS] + dx; + yMNT->data.F32[nMNT] = PAR[PM_PAR_YPOS] + dy; + nMNT++; + + // psf major axis + dx = scale*axes_psf.major*cos(axes_psf.theta); + dy = scale*axes_psf.major*sin(axes_psf.theta); + xPSF->data.F32[nPSF] = PAR[PM_PAR_XPOS] - dx; + yPSF->data.F32[nPSF] = PAR[PM_PAR_YPOS] - dy; + nPSF++; + xPSF->data.F32[nPSF] = PAR[PM_PAR_XPOS] + dx; + yPSF->data.F32[nPSF] = PAR[PM_PAR_YPOS] + dy; + nPSF++; + + // minor axis (to show size) + dy = +scale*axes_psf.minor*cos(axes_psf.theta); + dx = -scale*axes_psf.minor*sin(axes_psf.theta); + xMIN->data.F32[nMIN] = PAR[PM_PAR_XPOS] - dx; + yMIN->data.F32[nMIN] = PAR[PM_PAR_YPOS] - dy; + nMIN++; + xMIN->data.F32[nMIN] = PAR[PM_PAR_XPOS] + dx; + yMIN->data.F32[nMIN] = PAR[PM_PAR_YPOS] + dy; + nMIN++; + + graphdata.xmin = PS_MIN(graphdata.xmin, PAR[PM_PAR_XPOS]); + graphdata.xmax = PS_MAX(graphdata.xmax, PAR[PM_PAR_XPOS]); + graphdata.ymin = PS_MIN(graphdata.ymin, PAR[PM_PAR_YPOS]); + graphdata.ymax = PS_MAX(graphdata.ymax, PAR[PM_PAR_YPOS]); + } + xMNT->n = yMNT->n = nMNT; + xPSF->n = yPSF->n = nPSF; + xMIN->n = yMIN->n = nMIN; + + float range; + range = graphdata.xmax - graphdata.xmin; + graphdata.xmax += 0.05*range; + graphdata.xmin -= 0.05*range; + range = graphdata.ymax - graphdata.ymin; + graphdata.ymax += 0.05*range; + graphdata.ymin -= 0.05*range; + + // XXX set the plot range to match the image + KapaSetLimits (kapa, &graphdata); + + KapaSetFont (kapa, "helvetica", 14); + KapaBox (kapa, &graphdata); + KapaSendLabel (kapa, "x (pixels)", KAPA_LABEL_XM); + KapaSendLabel (kapa, "y (pixels)", KAPA_LABEL_YM); + KapaSendLabel (kapa, "vector is major axis (scale by 20) : black are moments, blue are psf model, red is psf minor axis", KAPA_LABEL_XP); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 100; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (kapa, nMNT, &graphdata); + KapaPlotVector (kapa, nMNT, xMNT->data.F32, "x"); + KapaPlotVector (kapa, nMNT, yMNT->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 100; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (kapa, nPSF, &graphdata); + KapaPlotVector (kapa, nPSF, xPSF->data.F32, "x"); + KapaPlotVector (kapa, nPSF, yPSF->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 100; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (kapa, nMIN, &graphdata); + KapaPlotVector (kapa, nMIN, xMIN->data.F32, "x"); + KapaPlotVector (kapa, nMIN, yMIN->data.F32, "y"); + + psLogMsg ("psphot", 3, "saving plot to %s", file->filename); + KapaPNG (kapa, file->filename); + + psFree (xMNT); + psFree (yMNT); + psFree (xPSF); + psFree (yPSF); + psFree (xMIN); + psFree (yMIN); + + return true; +} + +# else + +bool psphotPlotMoments (pmConfig *config, pmFPAview *view, psArray *sources) { + psLogMsg ("psphot", 3, "skipping moments plot"); + return true; +} + +bool psphotPlotPSFModel (pmConfig *config, pmFPAview *view, psArray *sources) { + psLogMsg ("psphot", 3, "skipping psf model plot"); + return true; +} + +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTest.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTest.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTest.c (revision 41083) @@ -0,0 +1,186 @@ +# include "psphotInternal.h" + +bool FillImage_Threaded (psThreadJob *job); + +bool SetThreads () { + + psThreadTask *task = NULL; + + task = psThreadTaskAlloc("FILL_IMAGE", 6); + task->function = &FillImage_Threaded; + psThreadTaskAdd(task); + psFree(task); + + return true; +} + +bool FillImage (psImage *image, int xs, int ys, int dx, int dy, int value) { + + psRegion region = psRegionSet (xs, xs + dx, ys, ys + dy); + psImage *subset = psImageSubset (image, region); + psImageInit (subset, value); + psFree (subset); + return true; +} + +bool FillImage_Threaded (psThreadJob *job) { + + psImage *image = job->args->data[0]; + int xs = PS_SCALAR_VALUE(job->args->data[1],S32); + int ys = PS_SCALAR_VALUE(job->args->data[2],S32); + int dx = PS_SCALAR_VALUE(job->args->data[3],S32); + int dy = PS_SCALAR_VALUE(job->args->data[4],S32); + int value = PS_SCALAR_VALUE(job->args->data[5],S32); + + // we want the threads to be likely to interact. run lots of psImageSubsets + psRegion region = psRegionSet (xs, xs + dx, ys, ys + dy); + for (int i = 0; i < 100; i++) { + psImage *subset = psImageSubset (image, region); + psImageInit (subset, value + i); + psFree (subset); + } + return true; +} + +int main (int argc, char **argv) { + + if (argc != 3) { + fprintf (stderr, "USAGE: psphotTest (output.fits) (nThreads)\n"); + exit (2); + } + + (void) psTraceSetLevel ("psLib.sys.mutex", 3); + + int nThreads = atoi (argv[2]); + + // create the thread pool with number of desired threads, supplying our thread launcher function + psThreadPoolInit (nThreads); + + SetThreads(); + + psImage *image = psImageAlloc (1000, 1000, PS_TYPE_S32); + + for (int ix = 0; ix < 1000; ix += 100) { + for (int iy = 0; iy < 1000; iy += 100) { + + // allocate a job -- if threads are not defined, this just runs the job + psThreadJob *job = psThreadJobAlloc ("FILL_IMAGE"); + + psArrayAdd(job->args, 1, image); + PS_ARRAY_ADD_SCALAR(job->args, ix, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, iy, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, 100, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, 100, PS_TYPE_S32); + PS_ARRAY_ADD_SCALAR(job->args, ix + iy, PS_TYPE_S32); + + // FillImage (image, ix, iy, 100, 100, ix + iy); + + if (!psThreadJobAddPending(job)) { + fprintf (stderr, "failure to run FillImage(1)"); + exit (1); + } + } + } + + + // wait for the threads to finish and manage results + if (!psThreadPoolWait (true, true)) { + fprintf (stderr, "failure to run FillImage (2)"); + exit (1); + } + + psFits *fits = psFitsOpen (argv[1], "w"); + psFitsWriteImage (fits, NULL, image, 0, NULL); + psFitsClose (fits); + + psThreadPoolFinalize (); + psFree(image); + + fprintf (stderr, "found %d leaks\n", psMemCheckLeaks (0, NULL, stdout, false)); + exit (0); +} + +# if (0) + +psRegion region = psRegionSet (0,0,0,0); // a region representing the entire array + psphotTestArguments (&argc, argv); + + psFits *file = psFitsOpen (argv[1], "r"); + psMetadata *header = psFitsReadHeader (NULL, file); + psImage *image = psFitsReadImage (NULL, file, region, 0); + psFitsClose (file); + + psImageJpegColormap (argv[5]); + + // psImage *fimage = psImageCopy (NULL, image, PS_TYPE_F32); + + int binning = atof(argv[6]); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN); + psImage *fimage = psImageRebin (NULL, image, NULL, 0, binning, stats); + + float min = atof(argv[3]); + float max = atof(argv[4]); + + psImageJpeg (fimage, argv[2], min, max); + + psFree (header); + psFree (image); + +# endif + +# if (0) + + psMetadata *row; + psArray *table; + + psMetadataItem *mdi; + + psMetadataConfigWrite (header, argv[2], NULL); + + // attempt to write image with NAXIS = 0 + mdi = psMetadataLookup (header, "NAXIS"); + mdi->data.S32 = 0; + mdi->type = PS_DATA_S32; + + // create a test image + // psImage *tmpimage = psImageAlloc (10, 10, PS_DATA_F32); + + // create a test table + table = psArrayAllocEmpty (10); + + for (int i = 0; i < 10; i++) { + row = psMetadataAlloc (); + psMetadataAdd (row, PS_LIST_TAIL, "ROW", PS_DATA_S32, "", i); + psMetadataAdd (row, PS_LIST_TAIL, "FROW", PS_TYPE_F32, "", 0.1*i); + psMetadataAdd (row, PS_LIST_TAIL, "DUMMY", PS_DATA_STRING, "", "test line"); + + table->data[i] = row; + } + table->n = 10; + + psMetadata *theader = psMetadataAlloc (); + psMetadataAdd (theader, PS_LIST_HEAD, "EXTNAME", PS_DATA_STRING, "extension name", "SMPFILE"); + + psFits *fits = psFitsOpen (argv[3], "w"); + // psFitsWriteImage (fits, header, tmpimage, 0); + psFitsWriteHeader (header, fits); + psFitsWriteTable (fits, theader, table); + +# endif + +# if (0) + +void psExit (int status, char *process, char *format, ...) { + + va_list ap; + + va_start (ap, format); + fprintf (stderr, "exiting %s\n", process); + vfprintf (stderr, format, ap); + va_end (ap); + + exit (status); +} + +# endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestArguments.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestArguments.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestArguments.c (revision 41083) @@ -0,0 +1,19 @@ +# include "psphotInternal.h" +static int usage (); + +void psphotTestArguments (int *argc, char **argv) { + + // basic pslib options + psLogSetFormat ("M"); + psArgumentVerbosity (argc, argv); + + if (*argc != 7) usage (); + + return; +} + +static int usage () { + + fprintf (stderr, "USAGE: psphotTest (input.fits) (output.jpg) (zero) (scale) (colormap) (rebin)\n"); + exit (2); +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestPSF.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestPSF.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestPSF.c (revision 41083) @@ -0,0 +1,131 @@ +# include "psphotInternal.h" + +bool psphotTestPSF (pmReadout *readout, psArray *sources, psMetadata *recipe) { + + bool status; + char *modelName; + pmPSF *psf = NULL; + psArray *stars = NULL; + + psTimerStart ("psphot"); + + psphotSaveImage (NULL, readout->image, "image.fits"); + + // check if a PSF model is supplied by the user + psf = psMetadataLookupPtr (NULL, readout->analysis, "PSPHOT.PSF"); + if (psf != NULL) return psf; + + // examine PSF sources in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // array to store candidate PSF stars + int NSTARS = psMetadataLookupS32 (&status, recipe, "PSF_MAX_NSTARS"); + if (!status) { + NSTARS = PS_MIN (sources->n, 200); + psWarning("PSF_MAX_NSTARS is not set in the recipe --- defaulting to %d\n", NSTARS); + } + + // use poissonian errors or local-sky errors + bool POISSON_ERRORS = psMetadataLookupBool (&status, recipe, "POISSON_ERRORS"); + if (!status) { + POISSON_ERRORS = true; + psWarning("POISSON_ERRORS is not set in the recipe --- defaulting to true.\n"); + } + pmSourceFitModelInit (15, 0.1, 1.0, POISSON_ERRORS); + + // how to model the PSF variations across the field + // XXX make a default value? or not? + psMetadata *md = psMetadataLookupMetadata (&status, recipe, "PSF.TREND.MASK"); + psPolynomial2D *psfTrendMask; + if (!status || !md) { + psWarning("PSF.TREND.MASK is not set in the recipe --- defaulting to use zeroth order.\n"); + psfTrendMask = psPolynomial2DAlloc(PS_POLYNOMIAL_ORD, 0, 0); + } else { + psfTrendMask = psPolynomial2DfromMetadata (md); + if (!psfTrendMask) { + psError(PSPHOT_ERR_PSF, true, "Unable to construct polynomial from PSF.TREND.MASK in the recipe"); + return false; + } + } + + stars = psArrayAllocEmpty (sources->n); + + // select the candidate PSF stars (pointers to original sources) + for (int i = 0; (i < sources->n) && (stars->n < NSTARS); i++) { + pmSource *source = sources->data[i]; + // if (source->mode & PM_SOURCE_MODE_PSFSTAR) psArrayAdd (stars, 200, source); + psArrayAdd (stars, 200, source); + } + psLogMsg ("psphot.pspsf", 4, "selected candidate %ld PSF objects\n", stars->n); + + if (stars->n == 0) { + psError(PSPHOT_ERR_PSF, true, "Failed to find any PSF candidates"); + return NULL; + } + + // get the fixed PSF fit radius + // XXX EAM : check that PSF_FIT_RADIUS < SKY_OUTER_RADIUS + float RADIUS = psMetadataLookupF32 (&status, recipe, "PSF_FIT_RADIUS"); + if (!status) { + psWarning("PSF_FIT_RADIUS is not set in the recipe --- defaulting to 20.0\n"); + RADIUS = 20.0; + } + + // for this test, require a single model + psMetadataItem *mdi = psMetadataLookup (recipe, "PSF_MODEL"); + if (mdi == NULL) psAbort("missing PSF_MODEL selection"); + if (mdi->type != PS_DATA_STRING) psAbort("choose a single PSF_MODEL"); + modelName = mdi->data.V; + + pmPSFtestModel (stars, modelName, RADIUS, POISSON_ERRORS, psfTrendMask); + + psphotSaveImage (NULL, readout->image, "resid.fits"); + psphotSaveImage (NULL, readout->mask, "mask.fits"); + psphotSaveImage (NULL, readout->weight, "weight.fits"); + + return true; +} + +bool pmPSFtestModel (psArray *sources, char *modelName, float RADIUS, bool poissonErrors, psPolynomial2D *psfTrendMask) +{ + bool status; + float x; + float y; + + pmModelType type = pmModelSetType (modelName); + pmPSF *psf = pmPSFAlloc (type, poissonErrors, psfTrendMask); + if (psf == NULL) psAbort("unknown model"); + + FILE *f = fopen ("params.dat", "w"); + + // stage 1: fit an independent model (freeModel) to all sources + psTimerStart ("fit"); + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + pmModel *model = pmSourceModelGuess (source, psf->type); + x = source->peak->x; + y = source->peak->y; + + // set temporary object mask and fit object + // fit model as EXT, not PSF + psImageKeepCircle (source->mask, x, y, RADIUS, "OR", markVal); + status = pmSourceFitModel (source, model, PM_SOURCE_FIT_EXT); + psImageKeepCircle (source->mask, x, y, RADIUS, "AND", PS_NOT_IMAGE_MASK(markVal)); + + // write fitted parameters to file + fprintf (f, "%f ", model->params->data.F32[PM_PAR_XPOS]); + fprintf (f, "%f ", model->params->data.F32[PM_PAR_YPOS]); + + fprintf (f, "%f ", model->params->data.F32[PM_PAR_SXX]); + fprintf (f, "%f ", model->params->data.F32[PM_PAR_SYY]); + fprintf (f, "%f ", model->params->data.F32[PM_PAR_SXY]); + + fprintf (f, "%f %d\n", model->chisq, model->nIter); + + // subtract model flux + pmModelSub (source->pixels, source->mask, model, PM_MODEL_OP_FULL); + } + fclose (f); + psLogMsg ("psphot.psftest", 4, "fit ext: %f sec for %ld sources\n", psTimerMark ("fit"), sources->n); + return true; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestSourceOutput.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestSourceOutput.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotTestSourceOutput.c (revision 41083) @@ -0,0 +1,157 @@ +# include "psphotInternal.h" + +enum { + PSPHOT_ADD_NONE = 0, + PSPHOT_ADD_MODEL = 1, + PSPHOT_ADD_R0 = 2, + PSPHOT_ADD_R1 = 4, +}; + +bool psphotAddModel(psImage *image, + pmModel *model, + int mode + ) +{ + psTrace("psModules.objects", 3, "---- %s() begin ----\n", __func__); + + PS_ASSERT_PTR_NON_NULL(model, false); + PS_ASSERT_IMAGE_NON_NULL(image, false); + PS_ASSERT_IMAGE_TYPE(image, PS_TYPE_F32, false); + + psVector *x = psVectorAlloc(2, PS_TYPE_F32); + psVector *params = model->params; + psS32 imageCol; + psS32 imageRow; + psF32 skyValue = params->data.F32[0]; + psF32 pixelValue; + + float xCenter = model->params->data.F32[PM_PAR_XPOS]; + float yCenter = model->params->data.F32[PM_PAR_YPOS]; + float Io = model->params->data.F32[PM_PAR_I0]; + + int xBin = 1; + int yBin = 1; + float xResidCenter = 0.0; + float yResidCenter = 0.0; + + psImageInterpolateOptions *Ro = NULL; + psImageInterpolateOptions *Rx = NULL; + psImageInterpolateOptions *Ry = NULL; + if (model->residuals && (mode & (PSPHOT_ADD_R0 | PSPHOT_ADD_R1))) { + Ro = psImageInterpolateOptionsAlloc( + PS_INTERPOLATE_BILINEAR, + model->residuals->Ro, NULL, NULL, 0, 0.0, 0.0, 1, 0, 0.0); + Rx = psImageInterpolateOptionsAlloc( + PS_INTERPOLATE_BILINEAR, + model->residuals->Rx, NULL, NULL, 0, 0.0, 0.0, 1, 0, 0.0); + Ry = psImageInterpolateOptionsAlloc( + PS_INTERPOLATE_BILINEAR, + model->residuals->Ry, NULL, NULL, 0, 0.0, 0.0, 1, 0, 0.0); + + xBin = model->residuals->xBin; + yBin = model->residuals->yBin; + xResidCenter = model->residuals->xCenter; + yResidCenter = model->residuals->yCenter; + } + + for (psS32 iy = 0; iy < image->numRows; iy++) { + for (psS32 ix = 0; ix < image->numCols; ix++) { + + // Convert i/j to image coord space: + imageCol = ix + image->col0; + imageRow = iy + image->row0; + + x->data.F32[0] = (float) imageCol; + x->data.F32[1] = (float) imageRow; + + // set the appropriate pixel value for this coordinate + if (mode & PSPHOT_ADD_MODEL) { + pixelValue = model->modelFunc (NULL, params, x) - skyValue; + } else { + pixelValue = 0.0; + } + + // get the contribution from the residual model + // XXX for a test, do this for all sources and all pixels + if (Ro) { + // fractional image position + // this is wrong for the 'center' case + float ox = xBin*(ix + 0.5 + image->col0 - xCenter) + xResidCenter; + float oy = yBin*(iy + 0.5 + image->row0 - yCenter) + yResidCenter; + + psImageMaskType mflux = 0; + double Fo = 0.0; + double Fx = 0.0; + double Fy = 0.0; + psImageInterpolate (&Fo, NULL, &mflux, ox, oy, Ro); + psImageInterpolate (&Fx, NULL, &mflux, ox, oy, Rx); + psImageInterpolate (&Fy, NULL, &mflux, ox, oy, Ry); + + if (!mflux && isfinite(Fo) && isfinite(Fx) && isfinite(Fy)) { + if (mode & PSPHOT_ADD_R0) { + pixelValue += Io*Fo; + } + if (mode & PSPHOT_ADD_R1) { + pixelValue += Io*(xCenter*Fx + yCenter*Fy); + } + } + } + image->data.F32[iy][ix] += pixelValue; + } + } + psFree(x); + psFree(Ro); + psFree(Rx); + psFree(Ry); + psTrace("psModules.objects", 3, "---- %s(true) end ----\n", __func__); + return(true); +} + +// construct an initial PSF model for each object +bool psphotTestSourceOutput (pmReadout *readout, psArray *sources, psMetadata *recipe, pmPSF *psf) { + + psImage *imMo = psImageAlloc (readout->image->numCols, readout->image->numRows, PS_TYPE_F32); + psImage *imR0 = psImageAlloc (readout->image->numCols, readout->image->numRows, PS_TYPE_F32); + psImage *imR1 = psImageAlloc (readout->image->numCols, readout->image->numRows, PS_TYPE_F32); + + // create template model + pmModel *modelRef = pmModelAlloc(psf->type); + modelRef->params->data.F32[PM_PAR_SKY] = 0; + modelRef->params->data.F32[PM_PAR_I0] = 1000; + + int dx = 25; + int dy = 25; + + // generate a grid of fake sources with amplitude 1000 + for (int iy = 50; iy < imMo->numRows; iy += 100) { + for (int ix = 50; ix < imMo->numCols; ix += 100) { + + // assign the x and y coords to the image center + modelRef->params->data.F32[PM_PAR_XPOS] = ix; + modelRef->params->data.F32[PM_PAR_YPOS] = iy; + + // create modelPSF from this model + pmModel *model = pmModelFromPSF (modelRef, psf); + model->residuals = psf->residuals; + + // generate working image for this source + psRegion region = psRegionSet(ix - dx, ix + dx, iy - dy, iy + dy); + + psImage *vM = psImageSubset (imMo, region); + psImage *v0 = psImageSubset (imR0, region); + psImage *v1 = psImageSubset (imR1, region); + + // we want to make one image o + psphotAddModel (vM, model, PSPHOT_ADD_MODEL); + psphotAddModel (v0, model, PSPHOT_ADD_R0); + psphotAddModel (v1, model, PSPHOT_ADD_R1); + } + } + + psphotSaveImage (NULL, imMo, "grid.Mo.fits"); + psphotSaveImage (NULL, imR0, "grid.R0.fits"); + psphotSaveImage (NULL, imR1, "grid.R1.fits"); + + exit (0); +} + Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotThreadTools.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotThreadTools.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotThreadTools.c (revision 41083) @@ -0,0 +1,116 @@ +# include "psphotInternal.h" + +// the strategy here is to divide the image into 2x2 blocks of cells and cycle through +// the four discontiguous sets of cells, threading all within a set and blocking between +// sets + +// we divide the image region into 2*2 blocks of size Nx*Ny, the image will have +// Cx*Cy blocks so that (2Nx)Cx = numCols, (2Ny)Cy = numRows. We want to choose Cx and +// Cy so that (2Nx)Cx * (2Ny)Cy = 4 * NFILL * nThreads -- each of the four sets of cells +// has enough cells to allow NFILL cells for each thread (to better distribute heavy and +// light load cells + +// the array runs from readout->image->col0 to readout->image->col0 + readout->image->numCols + +// we save these for threaded analysis runs +static int Xo = 0; +static int Yo = 0; +static int Nx = 1; +static int Ny = 1; + +bool psphotChooseCellSizes (int *Cx, int *Cy, pmReadout *readout, int nThreads) { + + int nCells = nThreads * 2*2; // number of cells in a single set + int C = sqrt(nCells) + 0.5; + + // we need to assign Cx and Cy based on the dimensionality of the image + // crude way to find most evenly balanced factors of nCells: + for (int i = C; i >= 1; i--) { + int C1 = nCells / C; + int C2 = nCells / C1; + if (C1*C2 != nCells) continue; + + if (readout->image->numRows > readout->image->numCols) { + *Cx = PS_MAX (C1, C2); + *Cy = PS_MIN (C1, C2); + } else { + *Cx = PS_MAX (C1, C2); + *Cy = PS_MIN (C1, C2); + } + + Xo = readout->image->col0; + Yo = readout->image->row0; + Nx = readout->image->numCols / (*Cx*2); + Ny = readout->image->numRows / (*Cy*2); + + return true; + } + *Cx = 1; + *Cy = 1; + + Xo = readout->image->col0; + Yo = readout->image->row0; + Nx = readout->image->numCols / (*Cx*2); + Ny = readout->image->numRows / (*Cy*2); + + return true; +} + + +bool psphotCoordToCell (int *group, int *cell, float x, float y, int Cx, int Cy) { + + // XXX need to handle edges + int ix = (x - Xo)/(2*Nx); + ix = PS_MAX (0, PS_MIN (ix, Cx - 1)); + + int iy = (y - Yo)/(2*Ny); + iy = PS_MAX (0, PS_MIN (iy, Cy - 1)); + + int jx = (((int)(x - Xo))%(2*Nx))/Nx; + jx = PS_MAX (0, PS_MIN (jx, Nx - 1)); + + int jy = (((int)(y - Yo))%(2*Ny))/Ny; + jy = PS_MAX (0, PS_MIN (jy, Ny - 1)); + + *group = jx + 2*jy; + *cell = ix + Cx*iy; + + return true; +} + + +// we have 2x2 * Cx*Cy cells for the image. we need a function to convert an x,y +// coordinate pair into the index for these cells + +// first, how shall we number them? is there one index for all cells, or one set of +// indices for each of the 2x2 cell groups? + +psArray *psphotAssignSources (int Cx, int Cy, psArray *sources) { + + psArray *cellGroups = psArrayAlloc (4); + for (int i = 0; i < cellGroups->n; i++) { + psArray *cells = psArrayAlloc (Cx*Cy); + cellGroups->data[i] = cells; + for (int j = 0; j < cells->n; j++) { + psArray *cellSources = psArrayAllocEmpty (50); + cells->data[j] = cellSources; + } + } + + for (int i = 0; i < sources->n; i++) { + + int group = 0; + int cell = 0; + + pmSource *source = sources->data[i]; + + psphotCoordToCell (&group, &cell, source->peak->xf, source->peak->yf, Cx, Cy); + + psArray *cells = cellGroups->data[group]; + psArray *cellSources = cells->data[cell]; + + psArrayAdd (cellSources, 100, source); + } + + return cellGroups; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVersion.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVersion.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVersion.c (revision 41083) @@ -0,0 +1,128 @@ +#include "psphotInternal.h" +#include "psphotVersionDefinitions.h" + +#ifdef HAVE_KAPA +#include +#endif + +#ifndef PSPHOT_VERSION +#error "PSPHOT_VERSION is not set" +#endif +#ifndef PSPHOT_BRANCH +#error "PSPHOT_BRANCH is not set" +#endif +#ifndef PSPHOT_SOURCE +#error "PSPHOT_SOURCE is not set" +#endif + +psString psphotVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PSPHOT_BRANCH, PSPHOT_VERSION); + return value; +} + +psString psphotSource(void) +{ + return psStringCopy(PSPHOT_SOURCE); +} + +psString psphotVersionLong(void) +{ + psString version = psphotVersion(); // Version, to return + psString source = psphotSource(); // Source + + psStringPrepend(&version, "psphot "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + +#ifdef HAVE_KAPA +#if 0 + // XXX Need to get ohana and libdvo versions + psString ohanaVersion = psStringStripCVS(ohana_version(), "Name"); + psString libdvoVersion = psStringStripCVS(libdvo_version(), "Name"); + psStringAppend(&version, " with libkapa (ohana %s, libdvo: %s)", ohanaVersion, libdvoVersion); + psFree(ohanaVersion); + psFree(libdvoVersion); +#else + psStringAppend(&version, " with libkapa"); +#endif + +#else + psStringAppend (&version, " without libkapa"); +#endif + + return version; +} + +bool psphotVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psString version = psphotVersion(); // Software version + psString source = psphotSource(); // Software source + + psMetadataAddStr(header, PS_LIST_TAIL, "PHOT_V", PS_META_REPLACE, NULL, PSPHOT_VERSION); + + psStringPrepend(&version, "psphot version: "); + psStringPrepend(&source, "psphot source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + + return true; +} + + +bool psphotVersionHeaderFull(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "psphot at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + psModulesVersionHeader(header); + psphotVersionHeader(header); + + return true; +} + + +void psphotVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("psphot", PS_LOG_INFO, "psphot at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString psphot = psphotVersionLong(); // psphot version + + psLogMsg("psphot", PS_LOG_INFO, "%s", pslib); + psLogMsg("psphot", PS_LOG_INFO, "%s", psmodules); + psLogMsg("psphot", PS_LOG_INFO, "%s", psphot); + + psFree(pslib); + psFree(psmodules); + psFree(psphot); + + return; +} Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVersionDefinitions.h.in =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVersionDefinitions.h.in (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVersionDefinitions.h.in (revision 41083) @@ -0,0 +1,8 @@ +#ifndef PSPHOT_VERSION_DEFINITIONS_H +#define PSPHOT_VERSION_DEFINITIONS_H + +#define PSPHOT_VERSION @PSPHOT_VERSION@ // SVN version +#define PSPHOT_BRANCH @PSPHOT_BRANCH@ // SVN branch +#define PSPHOT_SOURCE @PSPHOT_SOURCE@ // SVN source + +#endif Index: /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVisual.c =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVisual.c (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/src/psphotVisual.c (revision 41083) @@ -0,0 +1,2918 @@ +# include "psphotInternal.h" + +// this function displays representative images as the psphot analysis progresses: +// 0 : image, 1 : variance +// 0 : backsub, 1 : variance, 2 : backgnd +// 0 : backsub, 1 : variance, 2 : signif +// (overlay peaks on images) +// (overlay footprints on images) +// (overlay moments on images) +// (overlay rough class on images) +// 0 : backsub, 1 : psfpos, 2: psfsub +// 0 : backsub, 1 : lin_resid, 2: psfsub + +# if (HAVE_KAPA) +# include + +# define DEBUG 0 + +bool pmVisualLimitsFromVectors (Graphdata *graphdata, psVector *xVec, psVector *yVec); + +// functions used to visualize the analysis as it goes +// these are invoked by the -visual options + +static int kapa1 = -1; +static int kapa2 = -1; +static int kapa3 = -1; + +/** destroy windows at the end of a run*/ +bool psphotVisualClose(void) +{ + if(kapa1 != -1) KapaClose(kapa1); + if(kapa2 != -1) KapaClose(kapa2); + if(kapa3 != -1) KapaClose(kapa3); + return true; +} + +int psphotKapaChannel (int channel) { + + switch (channel) { + case 1: + pmVisualInitWindow (&kapa1, "psphot:images"); + return kapa1; + case 2: + pmVisualInitWindow (&kapa2, "psphot:plots"); + return kapa2; + case 3: + pmVisualInitWindow (&kapa3, "psphot:stamps"); + return kapa3; + default: + psAbort ("unknown kapa channel"); + } + psAbort ("unknown kapa channel"); +} + +bool psphotVisualEraseOverlays (int channel, char *overlay) { + + int myKapa = psphotKapaChannel (channel); + if (myKapa == -1) return false; + + if (!(strcasecmp (overlay, "all"))) { + KiiEraseOverlay (myKapa, "red"); + KiiEraseOverlay (myKapa, "green"); + KiiEraseOverlay (myKapa, "blue"); + KiiEraseOverlay (myKapa, "yellow"); + return true; + } + KiiEraseOverlay (myKapa, overlay); + return true; +} + +bool psphotVisualShowMask (int kapaFD, psImage *inImage, const char *name, int channel) { + + KiiImage image; + KapaImageData data; + Coords coords; + + strcpy (coords.ctype, "RA---TAN"); + + image.Nx = inImage->numCols; + image.Ny = inImage->numRows; + + ALLOCATE (image.data2d, float *, image.Ny); + for (int iy = 0; iy < image.Ny; iy++) { + ALLOCATE (image.data2d[iy], float, image.Nx); + for (int ix = 0; ix < image.Nx; ix++) { + image.data2d[iy][ix] = inImage->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix]; + } + } + + strcpy (data.name, name); + strcpy (data.file, name); + data.zero = -1; + data.range = 32; + data.logflux = 0; + + KiiSetChannel (kapaFD, channel); + KiiNewPicture2D (kapaFD, &image, &data, &coords); + + for (int iy = 0; iy < image.Ny; iy++) { + free (image.data2d[iy]); + } + free (image.data2d); + + return true; +} + +bool psphotVisualShowObjectRegions (pmReadout *readout, psMetadata *recipe, psArray *sources) { + + KiiImage image; + KapaImageData data; + Coords coords; + + bool status = false; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + psImageMaskType markVal = psMetadataLookupImageMask(&status, recipe, "MARK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + maskVal |= markVal; + + if (!pmVisualTestLevel("psphot.image.objects", 2)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + strcpy (coords.ctype, "RA---TAN"); + + psImage *inImage = readout->image; + psImage *inMask = readout->mask; + image.Nx = inImage->numCols; + image.Ny = inImage->numRows; + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + if (!psImageBackground(stats, NULL, inImage, inMask, 0xffff, rng)) { + fprintf (stderr, "failed to get background values\n"); + return false; + } + + ALLOCATE (image.data2d, float *, image.Ny); + for (int iy = 0; iy < image.Ny; iy++) { + ALLOCATE (image.data2d[iy], float, image.Nx); + for (int ix = 0; ix < image.Nx; ix++) { + image.data2d[iy][ix] = 0; + } + } + + // loop over sources and set unmasked pixels to 0 + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (source == NULL) continue; + + psImage *mask = source->maskObj; + if (mask == NULL) continue; + + for (int iy = 0; iy < mask->numRows; iy++) { + int jy = iy + mask->row0; + if (jy < 0) continue; + if (jy >= inImage->numRows) continue; + for (int ix = 0; ix < mask->numCols; ix++) { + int jx = ix + mask->col0; + if (jx < 0) continue; + if (jx >= inImage->numCols) continue; + + if (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal) continue; + image.data2d[jy][jx] = 1; + } + } + } + + for (int iy = 0; iy < image.Ny; iy++) { + for (int ix = 0; ix < image.Nx; ix++) { + image.data2d[iy][ix] = (image.data2d[iy][ix] == 0.0) ? NAN : inImage->data.F32[iy][ix]; + } + } + + strcpy (data.name, "maskObj"); + strcpy (data.file, "maskObj"); + // data.zero = 0.0; + // data.range = 1.0; + data.zero = stats->robustMedian - stats->robustStdev; + data.range = 5*stats->robustStdev; + data.logflux = 0; + + KiiSetChannel (kapa, 2); + KiiNewPicture2D (kapa, &image, &data, &coords); + + for (int iy = 0; iy < image.Ny; iy++) { + free (image.data2d[iy]); + } + free (image.data2d); + + psFree (stats); + psFree (rng); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualScaleImage (int kapaFD, psImage *inImage, psImage *inMask, const char *name, float factor, int channel) { + + KiiImage image; + KapaImageData data; + Coords coords; + + strcpy (coords.ctype, "RA---TAN"); + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); + if (!psImageBackground(stats, NULL, inImage, inMask, 0xffff, rng)) { + fprintf (stderr, "failed to get background values\n"); + return false; + } + + image.data2d = inImage->data.F32; + image.Nx = inImage->numCols; + image.Ny = inImage->numRows; + + strcpy (data.name, name); + strcpy (data.file, name); + data.zero = stats->robustMedian - factor*stats->robustStdev; + + // XXX I we have a smoothed image, this make a much-too-tight display range + data.range = 5*factor*stats->robustStdev; + data.logflux = 0; + + KiiSetChannel (kapaFD, channel); + KiiNewPicture2D (kapaFD, &image, &data, &coords); + + psFree (stats); + psFree (rng); + + return true; +} + +bool psphotVisualRangeImage (int kapaFD, psImage *inImage, const char *name, int channel, float min, float max) { + + KiiImage image; + KapaImageData data; + Coords coords; + + strcpy (coords.ctype, "RA---TAN"); + + image.data2d = inImage->data.F32; + image.Nx = inImage->numCols; + image.Ny = inImage->numRows; + + strcpy (data.name, name); + strcpy (data.file, name); + data.zero = min; + data.range = max - min; + data.logflux = 0; + + KiiSetChannel (kapaFD, channel); + KiiNewPicture2D (kapaFD, &image, &data, &coords); + + return true; +} + +static psImage *posImage = NULL; +static psImage *delImage = NULL; + +bool psphotVisualShowImage (pmReadout *readout) { + + if (!pmVisualTestLevel("psphot.image", 1)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + float factor = 1.0; + if (readout->covariance) { + factor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); + } + + psphotVisualShowMask (kapa, readout->mask, "mask", 2); + psphotVisualScaleImage (kapa, readout->variance, readout->mask, "variance", 1.0, 1); + psphotVisualScaleImage (kapa, readout->image, readout->mask, "image", sqrt(factor), 0); + + if (posImage == NULL) { + posImage = psImageCopy (NULL, readout->image, PS_TYPE_F32); + } + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualShowBackground (pmConfig *config, const pmFPAview *view, pmReadout *readout) { + + if (!pmVisualTestLevel("psphot.image.backgnd", 2)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + bool status = false; + pmFPAfile *file = psMetadataLookupPtr (&status, config->files, "PSPHOT.BACKGND"); + + pmReadout *backgnd = READOUT_OR_INTERNAL(view, file); + + float factor = 1.0; + if (readout->covariance) { + factor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); + } + + psphotVisualScaleImage (kapa, backgnd->image, readout->mask, "backgnd", 1.0, 2); + psphotVisualScaleImage (kapa, readout->image, readout->mask, "backsub", sqrt(factor), 0); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualShowSignificance (psImage *image, float min, float max) { + + if (!pmVisualTestLevel("psphot.image.signif", 2)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + psphotVisualRangeImage (kapa, image, "signif", 2, min, max); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualShowLogSignificance (psImage *image, float min, float max) { + + if (!pmVisualTestLevel("psphot.image.logsignif", 3)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + psImage *lsig = (psImage *) psUnaryOp (NULL, image, "log"); + psphotVisualRangeImage (kapa, lsig, "log-signif", 2, min, max); + psFree (lsig); + + pmVisualAskUser(NULL); + return true; +} + +// requires psphotVisualShowImage +bool psphotVisualShowSources (psArray *sources) { + + int Noverlay; + KiiOverlay *overlay; + + if (!pmVisualTestLevel("psphot.objects.sources", 1)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + // note: this uses the Ohana allocation tools: + // ALLOCATE (overlay, KiiOverlay, 3*peaks->n + 1); + ALLOCATE (overlay, KiiOverlay, sources->n + 2); + + Noverlay = 0; + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!source) continue; + + pmPeak *peak = source->peak; + if (!peak) continue; + + overlay[Noverlay].type = KII_OVERLAY_BOX; + overlay[Noverlay].x = peak->xf; + overlay[Noverlay].y = peak->yf; + overlay[Noverlay].dx = 4.0; + overlay[Noverlay].dy = 4.0; + overlay[Noverlay].angle = 0.0; + overlay[Noverlay].text = NULL; + Noverlay ++; + } + + KiiLoadOverlay (kapa, overlay, Noverlay, "blue"); + FREE (overlay); + + pmVisualAskUser(NULL); + return true; +} + +// XXX : requires psphotVisualShowImage +bool psphotVisualShowPeaks (pmDetections *detections) { + + int Noverlay; + KiiOverlay *overlay; + + if (!pmVisualTestLevel("psphot.objects.peaks", 1)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + psArray *peaks = detections->peaks; + + // note: this uses the Ohana allocation tools: + // ALLOCATE (overlay, KiiOverlay, 3*peaks->n + 1); + ALLOCATE (overlay, KiiOverlay, peaks->n + 2); + + Noverlay = 0; + for (int i = 0; i < peaks->n; i++) { + + pmPeak *peak = peaks->data[i]; + if (peak == NULL) continue; + + overlay[Noverlay].type = KII_OVERLAY_BOX; + overlay[Noverlay].x = peak->xf; + overlay[Noverlay].y = peak->yf; + overlay[Noverlay].dx = 2.0; + overlay[Noverlay].dy = 2.0; + overlay[Noverlay].angle = 0.0; + overlay[Noverlay].text = NULL; + Noverlay ++; + } + + KiiLoadOverlay (kapa, overlay, Noverlay, "red"); + FREE (overlay); + + pmVisualAskUser(NULL); + return true; +} + +// XXX : requires psphotVisualShowImage +bool psphotVisualShowFootprints (pmDetections *detections) { + + int Noverlay; + KiiOverlay *overlay; + + if (!pmVisualTestLevel("psphot.objects.footprints", 3)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + psArray *footprints = detections->footprints; + if (!footprints) return true; + + // note: this uses the Ohana allocation tools: + int NOVERLAY = footprints->n; + ALLOCATE (overlay, KiiOverlay, NOVERLAY); + + Noverlay = 0; + for (int i = 0; i < footprints->n; i++) { + + pmSpan *span = NULL; + + pmFootprint *footprint = footprints->data[i]; + if (footprint == NULL) continue; + if (footprint->spans == NULL) continue; + if (footprint->spans->n < 1) continue; + + // draw the top + // XXX need to allow top (and bottom) to have more than one span + span = footprint->spans->data[0]; + overlay[Noverlay].type = KII_OVERLAY_LINE; + overlay[Noverlay].x = span->x0; + overlay[Noverlay].y = span->y; + overlay[Noverlay].dx = span->x1 - span->x0; + overlay[Noverlay].dy = 0; + overlay[Noverlay].angle = 0.0; + overlay[Noverlay].text = NULL; + Noverlay ++; + CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); + + int ys = span->y; + int x0s = span->x0; + int x1s = span->x1; + + // draw the outer span edges + for (int j = 1; j < footprint->spans->n; j++) { + pmSpan *span1 = footprint->spans->data[j]; + + int ye = span1->y; + int x0e = span1->x0; + int x1e = span1->x1; + + // we cannot have two discontinuous spans on the top or bottom, right? (no, probably not right) + // find all of the spans in this row and generate x0e, x01: + for (int k = j + 1; k < footprint->spans->n; k++) { + pmSpan *span2 = footprint->spans->data[k]; + if (span2->y > span1->y) break; + x0e = PS_MIN (x0e, span2->x0); + x1e = PS_MAX (x1e, span2->x1); + j++; + } + + overlay[Noverlay].type = KII_OVERLAY_LINE; + overlay[Noverlay].x = x0s; + overlay[Noverlay].y = ys; + overlay[Noverlay].dx = x0e - x0s; + overlay[Noverlay].dy = ye - ys; + overlay[Noverlay].angle = 0.0; + overlay[Noverlay].text = NULL; + Noverlay ++; + CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); + + overlay[Noverlay].type = KII_OVERLAY_LINE; + overlay[Noverlay].x = x1s; + overlay[Noverlay].y = ys; + overlay[Noverlay].dx = x1e - x1s; + overlay[Noverlay].dy = ye - ys; + overlay[Noverlay].angle = 0.0; + overlay[Noverlay].text = NULL; + Noverlay ++; + CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); + + ys = ye; + x0s = x0e; + x1s = x1e; + } + + // draw the bottom + span = footprint->spans->data[footprint->spans->n - 1]; + overlay[Noverlay].type = KII_OVERLAY_LINE; + overlay[Noverlay].x = span->x0; + overlay[Noverlay].y = span->y; + overlay[Noverlay].dx = span->x1 - span->x0; + overlay[Noverlay].dy = 0; + overlay[Noverlay].angle = 0.0; + overlay[Noverlay].text = NULL; + Noverlay ++; + CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); + } + + KiiLoadOverlay (kapa, overlay, Noverlay, "blue"); + FREE (overlay); + + pmVisualAskUser(NULL); + return true; +} + +// XXX : requires psphotVisualShowImage +bool psphotVisualShowMoments (psArray *sources) { + + int Noverlay; + KiiOverlay *overlay; + + psEllipseMoments emoments; + psEllipseAxes axes; + + if (!pmVisualTestLevel("psphot.objects.moments", 2)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + // XXX mark the different source classes with different color/shape dots + // XXX are moments S/N and peak S/N consistent? + + // note: this uses the Ohana allocation tools: + ALLOCATE (overlay, KiiOverlay, sources->n); + + Noverlay = 0; + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (source == NULL) continue; + + pmMoments *moments = source->moments; + if (moments == NULL) continue; + + overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + overlay[Noverlay].x = moments->Mx; + overlay[Noverlay].y = moments->My; + + emoments.x2 = moments->Mxx; + emoments.xy = moments->Mxy; + emoments.y2 = moments->Myy; + + axes = psEllipseMomentsToAxes (emoments, 20.0); + + overlay[Noverlay].dx = 2.0*axes.major; + overlay[Noverlay].dy = 2.0*axes.minor; + + overlay[Noverlay].angle = axes.theta * PS_DEG_RAD; + + overlay[Noverlay].text = NULL; + Noverlay ++; + } + + KiiLoadOverlay (kapa, overlay, Noverlay, "yellow"); + FREE (overlay); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualPlotMoments (psMetadata *recipe, psMetadata *analysis, psArray *sources) { + + bool status; + Graphdata graphdata; + KapaSection section; + + if (!pmVisualTestLevel("psphot.moments", 1)) return true; + + int myKapa = psphotKapaChannel (2); + if (myKapa == -1) return false; + + KapaClearSections (myKapa); + KapaInitGraph (&graphdata); + KapaSetFont (myKapa, "courier", 14); + + section.bg = KapaColorByName ("none"); // XXX probably should be 'none' + + float SN_LIM = psMetadataLookupF32(&status, recipe, "PSF_SN_LIM"); + + // select the max psfX,Y values for the plot limits + float Xmin = 1000.0, Xmax = 0.0; + float Ymin = 1000.0, Ymax = 0.0; + { + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + for (int n = 0; n < nRegions; n++) { + + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + float psfY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); + float psfdX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); + float psfdY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); + + float X0 = psfX - 4.0*psfdX; + float X1 = psfX + 4.0*psfdX; + float Y0 = psfY - 4.0*psfdY; + float Y1 = psfY + 4.0*psfdY; + + if (isfinite(X0)) { Xmin = PS_MIN(Xmin, X0); } + if (isfinite(X1)) { Xmax = PS_MAX(Xmax, X1); } + if (isfinite(Y0)) { Ymin = PS_MIN(Ymin, Y0); } + if (isfinite(Y1)) { Ymax = PS_MAX(Ymax, Y1); } + } + } + Xmin = PS_MAX(Xmin, -0.1); + Ymin = PS_MAX(Ymin, -0.1); + + // XXX test: hardwire plot limits + // Xmin = -0.1; Ymin = -0.1; + // Xmax = 20.1; Ymax = 20.1; + + // storage vectors for data to be plotted + psVector *xBright = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yBright = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mBright = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *xFaint = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yFaint = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mFaint = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + // construct the vectors + int nB = 0; + int nF = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->moments == NULL) + continue; + + xFaint->data.F32[nF] = source->moments->Mxx; + yFaint->data.F32[nF] = source->moments->Myy; + mFaint->data.F32[nF] = -2.5*log10(source->moments->Sum); + nF++; + + // XXX make this a user-defined cutoff + if (source->moments->SN < SN_LIM) + continue; + + xBright->data.F32[nB] = source->moments->Mxx; + yBright->data.F32[nB] = source->moments->Myy; + mBright->data.F32[nB] = -2.5*log10(source->moments->Sum); + nB++; + } + xFaint->n = nF; + yFaint->n = nF; + mFaint->n = nF; + + xBright->n = nB; + yBright->n = nB; + mBright->n = nB; + + // three sections: MxxMyy, MagMxx, MagMyy + + // first section: MxxMyy + section.dx = 0.75; + section.dy = 0.75; + section.x = 0.00; + section.y = 0.00; + section.name = psStringCopy ("MxxMyy"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = Xmin; + graphdata.ymin = Ymin; + graphdata.xmax = Xmax; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = NAN; + graphdata.padYm = NAN; + graphdata.padXp = 0.5; + graphdata.padYp = 0.5; + KapaBox (myKapa, &graphdata); + + KapaSendLabel (myKapa, "M_xx| (pixels)", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "M_yy| (pixels)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (myKapa, nF, &graphdata); + + KapaPlotVector (myKapa, nF, xFaint->data.F32, "x"); + KapaPlotVector (myKapa, nF, yFaint->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nB, &graphdata); + KapaPlotVector (myKapa, nB, xBright->data.F32, "x"); + KapaPlotVector (myKapa, nB, yBright->data.F32, "y"); + + // second section: MagMyy + section.dx = 0.75; + section.dy = 0.25; + section.x = 0.00; + section.y = 0.75; + section.name = psStringCopy ("MagMyy"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = -17.1; + graphdata.xmax = -7.9; + graphdata.ymin = Ymin; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = 0.5; + graphdata.padYm = NAN; + graphdata.padXp = NAN; + graphdata.padYp = 0.5; + strcpy (graphdata.labels, "0210"); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "inst mag", KAPA_LABEL_XP); + KapaSendLabel (myKapa, "M_yy| (pixels)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (myKapa, nF, &graphdata); + KapaPlotVector (myKapa, nF, mFaint->data.F32, "x"); + KapaPlotVector (myKapa, nF, yFaint->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nB, &graphdata); + KapaPlotVector (myKapa, nB, mBright->data.F32, "x"); + KapaPlotVector (myKapa, nB, yBright->data.F32, "y"); + + // third section: MagMxx + section.dx = 0.25; + section.dy = 0.75; + section.x = 0.75; + section.y = 0.00; + section.name = psStringCopy ("MagMxx"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = Xmin; + graphdata.xmax = Xmax; + graphdata.ymin = -7.9; + graphdata.ymax = -17.1; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = NAN; + graphdata.padYm = 0.5; + graphdata.padXp = 0.5; + graphdata.padYp = NAN; + strcpy (graphdata.labels, "2001"); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "M_xx| (pixels)", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "inst mag", KAPA_LABEL_YP); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (myKapa, nF, &graphdata); + KapaPlotVector (myKapa, nF, xFaint->data.F32, "x"); + KapaPlotVector (myKapa, nF, mFaint->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nB, &graphdata); + KapaPlotVector (myKapa, nB, xBright->data.F32, "x"); + KapaPlotVector (myKapa, nB, mBright->data.F32, "y"); + + // draw N circles to outline the clumps + { + KapaSelectSection (myKapa, "MxxMyy"); + + // draw a circle centered on psfX,Y with size of the psf limit + psVector *xLimit = psVectorAlloc (120, PS_TYPE_F32); + psVector *yLimit = psVectorAlloc (120, PS_TYPE_F32); + + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, recipe, "PSF_CLUMP_NSIGMA"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.style = 0; + + graphdata.xmin = Xmin; + graphdata.ymin = Ymin; + graphdata.xmax = Xmax; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); + + for (int n = 0; n < nRegions; n++) { + + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + float psfY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); + float psfdX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); + float psfdY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); + float Rx = psfdX * PSF_CLUMP_NSIGMA; + float Ry = psfdY * PSF_CLUMP_NSIGMA; + + for (int i = 0; i < xLimit->n; i++) { + xLimit->data.F32[i] = Rx*cos(i*2.0*M_PI/120.0) + psfX; + yLimit->data.F32[i] = Ry*sin(i*2.0*M_PI/120.0) + psfY; + } + KapaPrepPlot (myKapa, xLimit->n, &graphdata); + KapaPlotVector (myKapa, xLimit->n, xLimit->data.F32, "x"); + KapaPlotVector (myKapa, yLimit->n, yLimit->data.F32, "y"); + } + psFree (xLimit); + psFree (yLimit); + } + + psFree (xBright); + psFree (yBright); + psFree (mBright); + psFree (xFaint); + psFree (yFaint); + psFree (mFaint); + + pmVisualAskUser(NULL); + return true; +} + +// assumes 'kapa' value is checked and set +bool psphotVisualShowRoughClass_Single (int myKapa, psArray *sources, pmSourceType type, pmSourceMode mode, char *color) { + + int Noverlay; + KiiOverlay *overlay; + + psEllipseMoments emoments; + psEllipseAxes axes; + + // note: this uses the Ohana allocation tools: + ALLOCATE (overlay, KiiOverlay, sources->n); + + Noverlay = 0; + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (source == NULL) continue; + + if (source->type != type) continue; + + if (mode == PM_SOURCE_MODE_PSFSTAR) { + bool keep = false; + keep |= (source->tmpFlags & PM_SOURCE_TMPF_CANDIDATE_PSFSTAR); + keep |= (source->mode & PM_SOURCE_MODE_PSFSTAR); + if (!keep) continue; + } else { + if (mode && !(source->mode & mode)) continue; + } + + pmMoments *moments = source->moments; + if (moments == NULL) continue; + + overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + overlay[Noverlay].x = moments->Mx; + overlay[Noverlay].y = moments->My; + + emoments.x2 = moments->Mxx; + emoments.y2 = moments->Myy; + emoments.xy = moments->Mxy; + + axes = psEllipseMomentsToAxes (emoments, 20.0); + + overlay[Noverlay].dx = 2.0*axes.major; + overlay[Noverlay].dy = 2.0*axes.minor; + overlay[Noverlay].angle = axes.theta * PS_DEG_RAD; + overlay[Noverlay].text = NULL; + Noverlay ++; + } + + KiiLoadOverlay (myKapa, overlay, Noverlay, color); + FREE (overlay); + + return true; +} + +// XXX : requires psphotVisualShowImage +bool psphotVisualShowRoughClass (psArray *sources) { + + if (!pmVisualTestLevel("psphot.objects.size", 3)) return true; + + int myKapa = psphotKapaChannel (1); + if (myKapa == -1) return false; + + KiiEraseOverlay (myKapa, "yellow"); // moments + + psphotVisualShowRoughClass_Single (myKapa, sources, PM_SOURCE_TYPE_STAR, 0, "red"); + psphotVisualShowRoughClass_Single (myKapa, sources, PM_SOURCE_TYPE_EXTENDED, 0, "blue"); + psphotVisualShowRoughClass_Single (myKapa, sources, PM_SOURCE_TYPE_DEFECT, 0, "blue"); + psphotVisualShowRoughClass_Single (myKapa, sources, PM_SOURCE_TYPE_SATURATED, 0, "red"); + psphotVisualShowRoughClass_Single (myKapa, sources, PM_SOURCE_TYPE_STAR, PM_SOURCE_MODE_PSFSTAR, "yellow"); + psphotVisualShowRoughClass_Single (myKapa, sources, PM_SOURCE_TYPE_STAR, PM_SOURCE_MODE_SATSTAR, "green"); + + fprintf (stdout, "red: STAR or SAT AREA; blue: EXTENDED or DEFECT; green: SATSTAR; yellow: PSFSTAR\n"); + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualShowPSFModel (pmReadout *readout, pmPSF *psf) { + + if (!pmVisualTestLevel("psphot.psf.model", 1)) return true; + + int myKapa = psphotKapaChannel (3); + if (myKapa == -1) return false; + + int DX = 64; + int DY = 64; + + psImage *psfMosaic = psImageAlloc (5*DX, 5*DY, PS_TYPE_F32); + psImageInit (psfMosaic, 0.0); + + psImage *funMosaic = psImageAlloc (5*DX, 5*DY, PS_TYPE_F32); + psImageInit (funMosaic, 0.0); + + psImage *resMosaic = psImageAlloc (5*DX, 5*DY, PS_TYPE_F32); + psImageInit (resMosaic, 0.0); + + pmModel *modelRef = pmModelAlloc(psf->type); + + // generate a fake model at each of the 3x3 image grid positions + for (int x = -2; x <= +2; x ++) { + for (int y = -2; y <= +2; y ++) { + // use the center of the center pixel of the image + float xc = (int)((0.5 + 0.225*x)*readout->image->numCols) + readout->image->col0 + 0.5; + float yc = (int)((0.5 + 0.225*y)*readout->image->numRows) + readout->image->row0 + 0.5; + + // assign the x and y coords to the image center + // create an object with center intensity of 1000 + modelRef->params->data.F32[PM_PAR_SKY] = 0; + modelRef->params->data.F32[PM_PAR_I0] = 1000; + modelRef->params->data.F32[PM_PAR_XPOS] = xc; + modelRef->params->data.F32[PM_PAR_YPOS] = yc; + + // create modelPSF from this model + pmModel *model = pmModelFromPSF (modelRef, psf); + if (!model) continue; + + // place the reference object in the image center + // no need to mask the source here + // XXX should we measure this for the analytical model only or the full model? + pmModelAddWithOffset (psfMosaic, NULL, model, PM_MODEL_OP_FULL | PM_MODEL_OP_CENTER, 0, x*DX, y*DY); + pmModelAddWithOffset (funMosaic, NULL, model, PM_MODEL_OP_FUNC | PM_MODEL_OP_CENTER, 0, x*DX, y*DY); + pmModelAddWithOffset (resMosaic, NULL, model, PM_MODEL_OP_RES0 | PM_MODEL_OP_RES1 | PM_MODEL_OP_CENTER, 0, x*DX, y*DY); + psFree (model); + } + } + + psImage *psfLogFlux = (psImage *) psUnaryOp (NULL, psfMosaic, "log"); + psphotVisualRangeImage (myKapa, psfLogFlux, "psf_mosaic", 0, -2.0, 3.0); + psphotVisualRangeImage (myKapa, funMosaic, "psf_analytical", 1, -10.0, 100.0); + psphotVisualRangeImage (myKapa, resMosaic, "psf_residual", 2, -10.0, 100.0); + + psFree (psfMosaic); + psFree (funMosaic); + psFree (resMosaic); + psFree (psfLogFlux); + psFree (modelRef); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualShowPSFStars (psMetadata *recipe, pmPSF *psf, psArray *sources) { + + bool status; + + if (!pmVisualTestLevel("psphot.psf.stars", 2)) return true; + + int myKapa = psphotKapaChannel (3); + if (myKapa == -1) return false; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // the source images are written to an image 10x the size of a PSF object + // float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + // PS_ASSERT (status, false); + + int DX = 21; + int DY = 21; + + // examine PSF sources in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // counters to track the size of the image and area used in a row + int dX = 0; // starting corner of next box + int dY = 0; // height of row so far + int NX = 20*DX; // full width of output image + int NY = 0; // total height of output image + + // first, examine the PSF stars: + // - determine bounding boxes for summary image + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + bool keep = false; + keep |= (source->mode & PM_SOURCE_MODE_PSFSTAR); + if (!keep) continue; + + // how does this subimage get placed into the output image? + // DX = source->pixels->numCols + // DY = source->pixels->numRows + + if (dX + DX > NX) { + // too wide for the rest of this row + if (dX == 0) { + // alone on this row + NY += DY; + dX = 0; + dY = 0; + } else { + // start the next row + NY += dY; + dX = DX; + dY = DY; + } + } else { + // extend this row + dX += DX; + dY = PS_MAX (dY, DY); + } + } + NY += DY; + + // allocate output image + psImage *outpos = psImageAlloc (NX, NY, PS_TYPE_F32); + psImage *outsub = psImageAlloc (NX, NY, PS_TYPE_F32); + psImageInit (outpos, 0.0); + psImageInit (outsub, 0.0); + + int Xo = 0; // starting corner of next box + int Yo = 0; // starting corner of next box + dY = 0; // height of row so far + + int nPSF = 0; + + // next, examine the PSF stars: + // - create output image array + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + bool keep = false; + if (source->mode & PM_SOURCE_MODE_PSFSTAR) { + nPSF ++; + keep = true; + } + if (!keep) continue; + + if (Xo + DX > NX) { + // too wide for the rest of this row + if (Xo == 0) { + // place source alone on this row + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + Yo += DY; + Xo = 0; + dY = 0; + } else { + // start the next row + Yo += dY; + Xo = 0; + + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + Xo = DX; + dY = DY; + } + } else { + // extend this row + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outpos, source, Xo, Yo, DX, DY, true); + + pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsub, source, Xo, Yo, DX, DY, true); + if (!subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + Xo += DX; + dY = PS_MAX (dY, DY); + } + } + + psphotVisualRangeImage (myKapa, outpos, "psfpos", 0, -0.05, 0.95); + psphotVisualRangeImage (myKapa, outsub, "psfsub", 1, -0.05, 0.95); + + pmVisualAskUser(NULL); + psFree (outpos); + psFree (outsub); + + + // after displaying (as an image) the psf stars, we cycle throught them and display their + // radial profiles: + psphotVisualPlotRadialProfiles (recipe, sources, PM_SOURCE_MODE_PSFSTAR); + + return true; +} + +bool psphotVisualShowSatStars (psMetadata *recipe, pmPSF *psf, psArray *sources) { + + bool status; + + if (!pmVisualTestLevel("psphot.psf.sat", 3)) return true; + + int myKapa = psphotKapaChannel (3); + if (myKapa == -1) return false; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + // the source images are written to an image 10x the size of a PSF object + // float OUTER = psMetadataLookupF32 (&status, recipe, "SKY_OUTER_RADIUS"); + // PS_ASSERT (status, false); + + int DX = 41; + int DY = 41; + + // examine PSF sources in S/N order (brightest first) + sources = psArraySort (sources, pmSourceSortByFlux); + + // counters to track the size of the image and area used in a row + int dX = 0; // starting corner of next box + int dY = 0; // height of row so far + int NX = 10*DX; // full width of output image + int NY = 0; // total height of output image + + // first, examine the PSF and SAT stars: + // - determine bounding boxes for summary image + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // only show "real" saturated stars (not defects) + if (!(source->mode & PM_SOURCE_MODE_SATSTAR)) continue;; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue;; + + // how does this subimage get placed into the output image? + // DX = source->pixels->numCols + // DY = source->pixels->numRows + + if (dX + DX > NX) { + // too wide for the rest of this row + if (dX == 0) { + // alone on this row + NY += DY; + dX = 0; + dY = 0; + } else { + // start the next row + NY += dY; + dX = DX; + dY = DY; + } + } else { + // extend this row + dX += DX; + dY = PS_MAX (dY, DY); + } + } + NY += DY; + + // allocate output image + psImage *outsat = psImageAlloc (NX, NY, PS_TYPE_F32); + psImageInit (outsat, 0.0); + + int Xo = 0; // starting corner of next box + int Yo = 0; // starting corner of next box + dY = 0; // height of row so far + + int nSAT = 0; + + // next, examine the SAT stars: + // - create output image array + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // only show "real" saturated stars (not defects) + if (!(source->mode & PM_SOURCE_MODE_SATSTAR)) continue;; + if (source->mode & PM_SOURCE_MODE_DEFECT) continue;; + nSAT ++; + + if (Xo + DX > NX) { + // too wide for the rest of this row + if (Xo == 0) { + // place source alone on this row + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsat, source, Xo, Yo, DX, DY, false); + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + Yo += DY; + Xo = 0; + dY = 0; + } else { + // start the next row + Yo += dY; + Xo = 0; + + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsat, source, Xo, Yo, DX, DY, false); + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + Xo = DX; + dY = DY; + } + } else { + // extend this row + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + psphotMosaicSubimage (outsat, source, Xo, Yo, DX, DY, false); + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + Xo += DX; + dY = PS_MAX (dY, DY); + } + } + + psphotVisualScaleImage (myKapa, outsat, NULL, "satstar", 1.0, 2); + + pmVisualAskUser(NULL); + psFree (outsat); + return true; +} + +void plotline (int myKapa, Graphdata *graphdata, float x0, float y0, float x1, float y1) +{ + float x[2], y[2]; + x[0] = x0; + x[1] = x1; + y[0] = y0; + y[1] = y1; + KapaPrepPlot (myKapa, 2, graphdata); + KapaPlotVector (myKapa, 2, x, "x"); + KapaPlotVector (myKapa, 2, y, "y"); +} + +bool psphotVisualPlotRadialProfile (int myKapa, pmSource *source, psImageMaskType maskVal, pmSourceMode showmode) { + + Graphdata graphdata; + + float Rmax = (showmode & PM_SOURCE_MODE_SATSTAR) ? 100.0 : 30.0; + + bool subtracted = (source->tmpFlags & PM_SOURCE_TMPF_SUBTRACTED); + if (subtracted) pmSourceAdd (source, PM_MODEL_OP_FULL, maskVal); + + int nPts = source->pixels->numRows * source->pixels->numCols; + psVector *rg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *Rg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *fg = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *rb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *Rb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + psVector *fb = psVectorAllocEmpty (nPts, PS_TYPE_F32); + + int ng = 0; + int nb = 0; + + float Xo = NAN; + float Yo = NAN; + + if (source->modelPSF) { + Xo = source->modelPSF->params->data.F32[PM_PAR_XPOS] - source->pixels->col0; + Yo = source->modelPSF->params->data.F32[PM_PAR_YPOS] - source->pixels->row0; + } else { + Xo = source->moments->Mx - source->pixels->col0; + Yo = source->moments->My - source->pixels->row0; + } + + for (int iy = 0; iy < source->pixels->numRows; iy++) { + for (int ix = 0; ix < source->pixels->numCols; ix++) { + if ((source->maskObj->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskVal)) { + rb->data.F32[nb] = hypot (ix + 0.5 - Xo, iy + 0.5 - Yo) ; + // rb->data.F32[nb] = hypot (ix - Xo, iy - Yo) ; + Rb->data.F32[nb] = log10(rb->data.F32[nb]); + fb->data.F32[nb] = log10(source->pixels->data.F32[iy][ix]); + nb++; + } else { + rg->data.F32[ng] = hypot (ix + 0.5 - Xo, iy + 0.5 - Yo) ; + // rg->data.F32[ng] = hypot (ix - Xo, iy - Yo) ; + Rg->data.F32[ng] = log10(rg->data.F32[ng]); + fg->data.F32[ng] = log10(source->pixels->data.F32[iy][ix]); + ng++; + } + } + } + + KapaInitGraph (&graphdata); + + // ** linlog ** + KapaSelectSection (myKapa, "linlog"); + + // examine sources to set data range + graphdata.xmin = -0.05; + graphdata.xmax = Rmax + 0.05; + graphdata.ymin = -0.05; + graphdata.ymax = +8.05; + KapaSetLimits (myKapa, &graphdata); + + KapaSetFont (myKapa, "helvetica", 14); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "radius (pixels)", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "log flux (counts)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, ng, &graphdata); + KapaPlotVector (myKapa, ng, rg->data.F32, "x"); + KapaPlotVector (myKapa, ng, fg->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (myKapa, nb, &graphdata); + KapaPlotVector (myKapa, nb, rb->data.F32, "x"); + KapaPlotVector (myKapa, nb, fb->data.F32, "y"); + + // ** loglog ** + KapaSelectSection (myKapa, "loglog"); + + // examine sources to set data range + graphdata.xmin = -1.51; + graphdata.xmax = log10(Rmax) + 0.02; + graphdata.ymin = -0.05; + graphdata.ymax = +8.05; + graphdata.color = KapaColorByName ("black"); + KapaSetLimits (myKapa, &graphdata); + + KapaSetFont (myKapa, "helvetica", 14); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "log radius (pixels)", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "log flux (counts)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, ng, &graphdata); + KapaPlotVector (myKapa, ng, Rg->data.F32, "x"); + KapaPlotVector (myKapa, ng, fg->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.3; + graphdata.style = 2; + KapaPrepPlot (myKapa, nb, &graphdata); + KapaPlotVector (myKapa, nb, Rb->data.F32, "x"); + KapaPlotVector (myKapa, nb, fb->data.F32, "y"); + + if (source->modelPSF) { + // generate model profiles (major and minor axis): + // create a model with theta = 0.0 so major and minor axes are equiv to x and y: + psEllipseShape rawShape, rotShape; + + rawShape.sx = source->modelPSF->params->data.F32[PM_PAR_SXX] / M_SQRT2; + rawShape.sy = source->modelPSF->params->data.F32[PM_PAR_SYY] / M_SQRT2; + rawShape.sxy = source->modelPSF->params->data.F32[PM_PAR_SXY]; + + psEllipseAxes axes = psEllipseShapeToAxes (rawShape, 20.0); + + axes.theta = 0.0; + + rotShape = psEllipseAxesToShape (axes); + + psVector *params = psVectorAlloc(source->modelPSF->params->n, PS_TYPE_F32); + for (int i = 0; i < source->modelPSF->params->n; i++) { + params->data.F32[i] = source->modelPSF->params->data.F32[i]; + } + params->data.F32[PM_PAR_SXX] = rotShape.sx * M_SQRT2; + params->data.F32[PM_PAR_SYY] = rotShape.sy * M_SQRT2; + params->data.F32[PM_PAR_SXY] = rotShape.sxy; + params->data.F32[PM_PAR_XPOS] = 0.0; + params->data.F32[PM_PAR_YPOS] = 0.0; + + psVector *rmod = psVectorAlloc(Rmax*10, PS_TYPE_F32); + psVector *fmaj = psVectorAlloc(Rmax*10, PS_TYPE_F32); + psVector *fmin = psVectorAlloc(Rmax*10, PS_TYPE_F32); + + psVector *coord = psVectorAlloc(2, PS_TYPE_F32); + + float r = 0.0; + for (int i = 0; i < rmod->n; i++) { + r = i*0.1; + rmod->data.F32[i] = r; + + coord->data.F32[1] = r; + coord->data.F32[0] = 0.0; + fmaj->data.F32[i] = log10(source->modelPSF->class->modelFunc (NULL, params, coord)); + + coord->data.F32[0] = r; + coord->data.F32[1] = 0.0; + fmin->data.F32[i] = log10(source->modelPSF->class->modelFunc (NULL, params, coord)); + } + psFree (coord); + psFree (params); + + float FWHM_MAJOR = 2.0*source->modelPSF->class->modelRadius (source->modelPSF->params, 0.5*source->modelPSF->params->data.F32[PM_PAR_I0]); + float FWHM_MINOR = FWHM_MAJOR * (axes.minor / axes.major); + if (FWHM_MAJOR < FWHM_MINOR) PS_SWAP (FWHM_MAJOR, FWHM_MINOR); + + psEllipseMoments emoments; + emoments.x2 = source->moments->Mxx; + emoments.xy = source->moments->Mxy; + emoments.y2 = source->moments->Myy; + axes = psEllipseMomentsToAxes (emoments, 20.0); + float MOMENTS_MAJOR = 2.355*axes.major; + float MOMENTS_MINOR = 2.355*axes.minor; + + float logHM = log10(0.5*source->modelPSF->params->data.F32[PM_PAR_I0]); + + // reset source Add/Sub state to recorded + if (subtracted) pmSourceSub (source, PM_MODEL_OP_FULL, maskVal); + + // ** linlog ** + KapaSelectSection (myKapa, "linlog"); + KapaGetGraphData (myKapa, &graphdata); + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (myKapa, rmod->n, &graphdata); + KapaPlotVector (myKapa, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (myKapa, rmod->n, fmin->data.F32, "y"); + plotline (myKapa, &graphdata, graphdata.xmin, logHM, graphdata.xmax, logHM); + plotline (myKapa, &graphdata, 0.5*FWHM_MINOR, graphdata.ymin, 0.5*FWHM_MINOR, graphdata.ymax); + graphdata.ltype = 1; + plotline (myKapa, &graphdata, 0.5*MOMENTS_MINOR, graphdata.ymin, 0.5*MOMENTS_MINOR, graphdata.ymax); + graphdata.ltype = 0; + + graphdata.color = KapaColorByName ("green"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (myKapa, rmod->n, &graphdata); + KapaPlotVector (myKapa, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (myKapa, rmod->n, fmaj->data.F32, "y"); + plotline (myKapa, &graphdata, 0.5*FWHM_MAJOR, graphdata.ymin, 0.5*FWHM_MAJOR, graphdata.ymax); + graphdata.ltype = 1; + plotline (myKapa, &graphdata, 0.5*MOMENTS_MAJOR, graphdata.ymin, 0.5*MOMENTS_MAJOR, graphdata.ymax); + graphdata.ltype = 0; + + for (int i = 0; i < rmod->n; i++) { + rmod->data.F32[i] = log10(rmod->data.F32[i]); + } + + // ** loglog ** + KapaSelectSection (myKapa, "loglog"); + KapaGetGraphData (myKapa, &graphdata); + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (myKapa, rmod->n, &graphdata); + KapaPlotVector (myKapa, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (myKapa, rmod->n, fmin->data.F32, "y"); + + graphdata.color = KapaColorByName ("green"); + graphdata.ptype = 0; + graphdata.size = 0.0; + graphdata.style = 0; + KapaPrepPlot (myKapa, rmod->n, &graphdata); + KapaPlotVector (myKapa, rmod->n, rmod->data.F32, "x"); + KapaPlotVector (myKapa, rmod->n, fmaj->data.F32, "y"); + + psFree (rmod); + psFree (fmin); + psFree (fmaj); + } + + psFree (rg); + psFree (Rg); + psFree (fg); + psFree (rb); + psFree (Rb); + psFree (fb); + return true; +} + +bool psphotVisualPlotRadialProfiles (psMetadata *recipe, psArray *sources, pmSourceMode showmode) { + + KapaSection section; // put the positive profile in one and the residuals in another? + + if (!pmVisualTestLevel("psphot.profiles", 3)) return true; + + int myKapa = psphotKapaChannel (2); + if (myKapa == -1) return false; + + // user-defined masks to test for good/bad pixels (build from recipe list if not yet set) + bool status; + psImageMaskType maskVal = psMetadataLookupImageMask(&status, recipe, "MASK.PSPHOT"); // Mask value for bad pixels + assert (maskVal); + + section.bg = KapaColorByName ("none"); // XXX probably should be 'none' + + KapaClearSections (myKapa); + // first section : mag vs CR nSigma + section.dx = 1.0; + section.dy = 0.5; + section.x = 0.0; + section.y = 0.0; + section.name = NULL; + psStringAppend (§ion.name, "linlog"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + // first section : mag vs CR nSigma + section.dx = 1.0; + section.dy = 0.5; + section.x = 0.0; + section.y = 0.5; + section.name = NULL; + psStringAppend (§ion.name, "loglog"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + // loop over the PSF stars + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (!(source->mode & showmode)) continue; + + psphotVisualPlotRadialProfile (myKapa, source, maskVal, showmode); + + if (pmVisualTestLevel("psphot.image", 1)) { + int display = psphotKapaChannel (1); + KiiCenter (display, source->peak->xf, source->peak->yf, 1); + KiiOverlay overlay; + overlay.x = source->peak->xf; + overlay.y = source->peak->yf; + overlay.dx = 5; + overlay.dy = 5; + overlay.angle = 0.0; + overlay.type = KiiOverlayTypeByName ("circle"); + KiiLoadOverlay (display, &overlay, 1, "red"); + overlay.x = source->moments->Mx; + overlay.y = source->moments->My; + overlay.dx = 8; + overlay.dy = 8; + overlay.angle = 0.0; + overlay.type = KiiOverlayTypeByName ("circle"); + KiiLoadOverlay (display, &overlay, 1, "blue"); + } + + // pause and wait for user input: + // continue, save (provide name), ?? + char key[10]; + fprintf (stdout, "[e]rase and continue? [o]verplot and continue? [s]kip rest of stars? : "); + if (!fgets(key, 8, stdin)) { + psWarning("Unable to read option"); + } + if (key[0] == 'e') { + KapaClearPlots (myKapa); + } + if (key[0] == 's') { + break; + } + } + + return true; +} + +bool psphotVisualShowFlags (psArray *sources) { + + int NoverlayE, NOVERLAYE; + int NoverlayO, NOVERLAYO; + KiiOverlay *overlayE, *overlayO; + + psEllipseMoments emoments; + psEllipseAxes axes; + + // XXX skip this for now: it is not very clear + return true; + + if (!pmVisualTestLevel("psphot.objects.flags", 3)) return true; + + int myKapa = psphotKapaChannel (1); + if (myKapa == -1) return false; + + // note: this uses the Ohana allocation tools: + NoverlayE = 0; + NOVERLAYE = 100; + ALLOCATE (overlayE, KiiOverlay, NOVERLAYE); + + NoverlayO = 0; + NOVERLAYO = 100; + ALLOCATE (overlayO, KiiOverlay, NOVERLAYO); + + for (int i = 0; i < sources->n; i++) { + + float Xo, Yo, Rmaj, Rmin, cs, sn; + + pmSource *source = sources->data[i]; + if (source == NULL) continue; + + pmMoments *moments = source->moments; + if (0) { + emoments.x2 = moments->Mxx; + emoments.y2 = moments->Myy; + emoments.xy = moments->Mxy; + Xo = moments->Mx; + Yo = moments->My; + + axes = psEllipseMomentsToAxes (emoments, 20.0); + Rmaj = 2.0*axes.major; + Rmin = 2.0*axes.minor; + cs = cos(axes.theta); + sn = sin(axes.theta); + } else { + Rmaj = Rmin = 5.0; + cs = 1.0; + sn = 0.0; + Xo = source->peak->xf; + Yo = source->peak->yf; + } + + unsigned short int flagMask = 0x01; + for (int j = 0; j < 8; j++) { + if (source->mode & flagMask) { + overlayE[NoverlayE].type = KII_OVERLAY_LINE; + overlayE[NoverlayE].x = Xo; + overlayE[NoverlayE].y = Yo; + + float phi = j*M_PI/4.0; + overlayE[NoverlayE].dx = +Rmaj*cos(phi)*cs - Rmin*sin(phi)*sn; + overlayE[NoverlayE].dy = +Rmaj*cos(phi)*sn + Rmin*sin(phi)*cs; + overlayE[NoverlayE].angle = 0; + overlayE[NoverlayE].text = NULL; + NoverlayE ++; + CHECK_REALLOCATE (overlayE, KiiOverlay, NOVERLAYE, NoverlayE, 100); + } + flagMask <<= 1; + + if (source->mode & flagMask) { + overlayO[NoverlayO].type = KII_OVERLAY_LINE; + overlayO[NoverlayO].x = Xo + 1; + overlayO[NoverlayO].y = Yo; + + float phi = j*M_PI/4.0; + overlayO[NoverlayO].dx = +Rmaj*cos(phi)*cs - Rmin*sin(phi)*sn; + overlayO[NoverlayO].dy = +Rmaj*cos(phi)*sn + Rmin*sin(phi)*cs; + overlayO[NoverlayO].angle = 0; + overlayO[NoverlayO].text = NULL; + NoverlayO ++; + CHECK_REALLOCATE (overlayO, KiiOverlay, NOVERLAYO, NoverlayO, 100); + } + flagMask <<= 1; + } + } + + KiiLoadOverlay (myKapa, overlayE, NoverlayE, "red"); + KiiLoadOverlay (myKapa, overlayO, NoverlayO, "yellow"); + FREE (overlayE); + FREE (overlayO); + + fprintf (stdout, "even bits (0x0001, 0x0004, ... : red\n"); + fprintf (stdout, "odd bits (0x0002, 0x0008, ... : yellow\n"); + pmVisualAskUser(NULL); + + return true; +} + +bool psphotVisualShowSourceSize_Single (int myKapa, psArray *sources, pmSourceMode mode, bool keep, float scale, char *color) { + + int Noverlay; + KiiOverlay *overlay; + + psEllipseMoments emoments; + psEllipseAxes axes; + + // note: this uses the Ohana allocation tools: + ALLOCATE (overlay, KiiOverlay, sources->n); + + Noverlay = 0; + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + if (source == NULL) continue; + + if (mode) { + if (keep) { + if (!(source->mode & mode)) continue; + } else { + if (source->mode & mode) continue; + } + } + + pmMoments *moments = source->moments; + if (moments == NULL) continue; + + overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + overlay[Noverlay].x = moments->Mx; + overlay[Noverlay].y = moments->My; + + emoments.x2 = moments->Mxx; + emoments.y2 = moments->Myy; + emoments.xy = moments->Mxy; + + axes = psEllipseMomentsToAxes (emoments, 20.0); + + overlay[Noverlay].dx = scale*2.0*axes.major; + overlay[Noverlay].dy = scale*2.0*axes.minor; + overlay[Noverlay].angle = axes.theta * PS_DEG_RAD; + overlay[Noverlay].text = NULL; + Noverlay ++; + } + + KiiLoadOverlay (myKapa, overlay, Noverlay, color); + FREE (overlay); + + return true; +} + +bool psphotVisualShowSourceSize (pmReadout *readout, psArray *sources) { + + if (!pmVisualTestLevel("psphot.objects.size", 2)) return true; + + int myKapa = psphotKapaChannel (1); + if (myKapa == -1) return false; + + KiiEraseOverlay (myKapa, "red"); + KiiEraseOverlay (myKapa, "blue"); + KiiEraseOverlay (myKapa, "green"); + KiiEraseOverlay (myKapa, "yellow"); + + psphotVisualShowSourceSize_Single (myKapa, sources, PM_SOURCE_MODE_EXT_LIMIT | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_SATSTAR, 0, 1.0, "green"); + psphotVisualShowSourceSize_Single (myKapa, sources, PM_SOURCE_MODE_EXT_LIMIT, 1, 1.0, "blue"); + psphotVisualShowSourceSize_Single (myKapa, sources, PM_SOURCE_MODE_CR_LIMIT, 1, 1.0, "red"); + psphotVisualShowSourceSize_Single (myKapa, sources, PM_SOURCE_MODE_DEFECT, 1, 2.0, "red"); + psphotVisualShowSourceSize_Single (myKapa, sources, PM_SOURCE_MODE_SATSTAR, 1, 1.0, "yellow"); + + fprintf (stdout, "red: CR; blue: EXTENDED; green: PSF-like; yellow: SATSTAR\n"); + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualPlotSourceSize (psMetadata *recipe, psMetadata *analysis, psArray *sources) { + + bool status; + Graphdata graphdata; + KapaSection section; + + if (!pmVisualTestLevel("psphot.size", 2)) return true; + + int myKapa = psphotKapaChannel (2); + if (myKapa == -1) return false; + + KapaClearSections (myKapa); + KapaInitGraph (&graphdata); + KapaSetFont (myKapa, "courier", 14); + + section.bg = KapaColorByName ("none"); // XXX probably should be 'none' + + // select the max psfX,Y values for the plot limits + float Xmin = 1000.0, Xmax = 0.0; + float Ymin = 1000.0, Ymax = 0.0; + { + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + for (int n = 0; n < nRegions; n++) { + + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + float psfY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); + float psfdX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); + float psfdY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); + + float X0 = psfX - 10.0*psfdX; + float X1 = psfX + 10.0*psfdX; + float Y0 = psfY - 10.0*psfdY; + float Y1 = psfY + 10.0*psfdY; + + if (isfinite(X0)) { Xmin = PS_MIN(Xmin, X0); } + if (isfinite(X1)) { Xmax = PS_MAX(Xmax, X1); } + if (isfinite(Y0)) { Ymin = PS_MIN(Ymin, Y0); } + if (isfinite(Y1)) { Ymax = PS_MAX(Ymax, Y1); } + } + } + Xmin = PS_MAX(Xmin, -0.1); + Ymin = PS_MAX(Ymin, -0.1); + + // storage vectors for data to be plotted + psVector *xSAT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *ySAT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mSAT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *sSAT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *xPSF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yPSF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mPSF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *sPSF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *xEXT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yEXT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mEXT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *sEXT = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *xDEF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yDEF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mDEF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *sDEF = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *xLOW = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yLOW = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mLOW = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *sLOW = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *xCR = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *yCR = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *mCR = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *sCR = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + // construct the vectors + int nSAT = 0; + int nEXT = 0; + int nPSF = 0; + int nDEF = 0; + int nLOW = 0; + int nCR = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->moments == NULL) continue; + + // only plot the measured sources... + if (!(source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED)) continue; + + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) { + xCR->data.F32[nCR] = source->moments->Mxx; + yCR->data.F32[nCR] = source->moments->Myy; + mCR->data.F32[nCR] = -2.5*log10(source->moments->Sum); + sCR->data.F32[nCR] = source->extNsigma; + nCR++; + } + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + xSAT->data.F32[nSAT] = source->moments->Mxx; + ySAT->data.F32[nSAT] = source->moments->Myy; + mSAT->data.F32[nSAT] = -2.5*log10(source->moments->Sum); + sSAT->data.F32[nSAT] = source->extNsigma; + nSAT++; + } + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + xEXT->data.F32[nEXT] = source->moments->Mxx; + yEXT->data.F32[nEXT] = source->moments->Myy; + mEXT->data.F32[nEXT] = -2.5*log10(source->moments->Sum); + sEXT->data.F32[nEXT] = source->extNsigma; + nEXT++; + continue; + } + if (source->mode & PM_SOURCE_MODE_DEFECT) { + xDEF->data.F32[nDEF] = source->moments->Mxx; + yDEF->data.F32[nDEF] = source->moments->Myy; + mDEF->data.F32[nDEF] = -2.5*log10(source->moments->Sum); + sDEF->data.F32[nDEF] = source->extNsigma; + nDEF++; + continue; + } + if (source->psfMagErr > 0.1) { + xLOW->data.F32[nLOW] = source->moments->Mxx; + yLOW->data.F32[nLOW] = source->moments->Myy; + mLOW->data.F32[nLOW] = -2.5*log10(source->moments->Sum); + sLOW->data.F32[nLOW] = source->extNsigma; + nLOW++; + continue; + } + xPSF->data.F32[nPSF] = source->moments->Mxx; + yPSF->data.F32[nPSF] = source->moments->Myy; + mPSF->data.F32[nPSF] = -2.5*log10(source->moments->Sum); + sPSF->data.F32[nPSF] = source->extNsigma; + nPSF++; + } + + xSAT->n = nSAT; + ySAT->n = nSAT; + mSAT->n = nSAT; + sSAT->n = nSAT; + + xPSF->n = nPSF; + yPSF->n = nPSF; + mPSF->n = nPSF; + sPSF->n = nPSF; + + xEXT->n = nEXT; + yEXT->n = nEXT; + mEXT->n = nEXT; + sEXT->n = nEXT; + + xCR->n = nCR; + yCR->n = nCR; + mCR->n = nCR; + sCR->n = nCR; + + xDEF->n = nDEF; + yDEF->n = nDEF; + mDEF->n = nDEF; + sDEF->n = nDEF; + + xLOW->n = nLOW; + yLOW->n = nLOW; + mLOW->n = nLOW; + sLOW->n = nLOW; + + // four sections: MxxMyy, MagMxx, MagMyy, MagSigma + + // first section: MxxMyy + section.dx = 0.75; + section.dy = 0.60; + section.x = 0.00; + section.y = 0.00; + section.name = psStringCopy ("MxxMyy"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = Xmin; + graphdata.ymin = Ymin; + graphdata.xmax = Xmax; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = NAN; + graphdata.padYm = NAN; + graphdata.padXp = 0.5; + graphdata.padYp = 0.5; + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "M_xx| (pixels)", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "M_yy| (pixels)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nPSF, &graphdata); + KapaPlotVector (myKapa, nPSF, xPSF->data.F32, "x"); + KapaPlotVector (myKapa, nPSF, yPSF->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nEXT, &graphdata); + KapaPlotVector (myKapa, nEXT, xEXT->data.F32, "x"); + KapaPlotVector (myKapa, nEXT, yEXT->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nDEF, &graphdata); + KapaPlotVector (myKapa, nDEF, xDEF->data.F32, "x"); + KapaPlotVector (myKapa, nDEF, yDEF->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nCR, &graphdata); + KapaPlotVector (myKapa, nCR, xCR->data.F32, "x"); + KapaPlotVector (myKapa, nCR, yCR->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nSAT, &graphdata); + KapaPlotVector (myKapa, nSAT, xSAT->data.F32, "x"); + KapaPlotVector (myKapa, nSAT, ySAT->data.F32, "y"); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nLOW, &graphdata); + KapaPlotVector (myKapa, nLOW, xLOW->data.F32, "x"); + KapaPlotVector (myKapa, nLOW, yLOW->data.F32, "y"); + + // second section: MagMyy + section.dx = 0.75; + section.dy = 0.20; + section.x = 0.00; + section.y = 0.80; + section.name = psStringCopy ("MagMyy"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = -17.1; + graphdata.xmax = -6.9; + graphdata.ymin = Ymin; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = 0.5; + graphdata.padYm = NAN; + graphdata.padXp = NAN; + graphdata.padYp = 0.5; + strcpy (graphdata.labels, "0210"); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "inst mag", KAPA_LABEL_XP); + KapaSendLabel (myKapa, "M_yy| (pixels)", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nPSF, &graphdata); + KapaPlotVector (myKapa, nPSF, mPSF->data.F32, "x"); + KapaPlotVector (myKapa, nPSF, yPSF->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nEXT, &graphdata); + KapaPlotVector (myKapa, nEXT, mEXT->data.F32, "x"); + KapaPlotVector (myKapa, nEXT, yEXT->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nDEF, &graphdata); + KapaPlotVector (myKapa, nDEF, mDEF->data.F32, "x"); + KapaPlotVector (myKapa, nDEF, yDEF->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nCR, &graphdata); + KapaPlotVector (myKapa, nCR, mCR->data.F32, "x"); + KapaPlotVector (myKapa, nCR, yCR->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nSAT, &graphdata); + KapaPlotVector (myKapa, nSAT, mSAT->data.F32, "x"); + KapaPlotVector (myKapa, nSAT, ySAT->data.F32, "y"); + + // third section: MagMxx + section.dx = 0.25; + section.dy = 0.60; + section.x = 0.75; + section.y = 0.00; + section.name = psStringCopy ("MagMxx"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = Xmin; + graphdata.xmax = Xmax; + graphdata.ymin = -6.9; + graphdata.ymax = -17.1; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = NAN; + graphdata.padYm = 0.5; + graphdata.padXp = 0.5; + graphdata.padYp = NAN; + strcpy (graphdata.labels, "2001"); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "M_xx| (pixels)", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "inst mag", KAPA_LABEL_YP); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nPSF, &graphdata); + KapaPlotVector (myKapa, nPSF, xPSF->data.F32, "x"); + KapaPlotVector (myKapa, nPSF, mPSF->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nEXT, &graphdata); + KapaPlotVector (myKapa, nEXT, xEXT->data.F32, "x"); + KapaPlotVector (myKapa, nEXT, mEXT->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nDEF, &graphdata); + KapaPlotVector (myKapa, nDEF, xDEF->data.F32, "x"); + KapaPlotVector (myKapa, nDEF, mDEF->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nCR, &graphdata); + KapaPlotVector (myKapa, nCR, xCR->data.F32, "x"); + KapaPlotVector (myKapa, nCR, mCR->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nSAT, &graphdata); + KapaPlotVector (myKapa, nSAT, xSAT->data.F32, "x"); + KapaPlotVector (myKapa, nSAT, mSAT->data.F32, "y"); + + // fourth section: MagSigma + section.dx = 0.75; + section.dy = 0.20; + section.x = 0.00; + section.y = 0.60; + section.name = psStringCopy ("MagSigma"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmax = -6.9; + graphdata.xmin = -17.1; + graphdata.ymin = -20.1; + graphdata.ymax = +20.1; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = 0.5; + graphdata.padYm = NAN; + graphdata.padXp = 0.5; + graphdata.padYp = 0.5; + strcpy (graphdata.labels, "0100"); + KapaBox (myKapa, &graphdata); + // KapaSendLabel (myKapa, "inst mag", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "EXT&ss&c", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nPSF, &graphdata); + KapaPlotVector (myKapa, nPSF, mPSF->data.F32, "x"); + KapaPlotVector (myKapa, nPSF, sPSF->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nEXT, &graphdata); + KapaPlotVector (myKapa, nEXT, mEXT->data.F32, "x"); + KapaPlotVector (myKapa, nEXT, sEXT->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, nDEF, &graphdata); + KapaPlotVector (myKapa, nDEF, mDEF->data.F32, "x"); + KapaPlotVector (myKapa, nDEF, sDEF->data.F32, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nCR, &graphdata); + KapaPlotVector (myKapa, nCR, mCR->data.F32, "x"); + KapaPlotVector (myKapa, nCR, sCR->data.F32, "y"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 7; + graphdata.size = 1.0; + graphdata.style = 2; + KapaPrepPlot (myKapa, nSAT, &graphdata); + KapaPlotVector (myKapa, nSAT, mSAT->data.F32, "x"); + KapaPlotVector (myKapa, nSAT, sSAT->data.F32, "y"); + + // draw N circles to outline the clumps + { + KapaSelectSection (myKapa, "MxxMyy"); + + // draw a circle centered on psfX,Y with size of the psf limit + psVector *xLimit = psVectorAlloc (120, PS_TYPE_F32); + psVector *yLimit = psVectorAlloc (120, PS_TYPE_F32); + + int nRegions = psMetadataLookupS32 (&status, analysis, "PSF.CLUMP.NREGIONS"); + float PSF_CLUMP_NSIGMA = psMetadataLookupF32 (&status, recipe, "PSF_CLUMP_NSIGMA"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.style = 0; + + graphdata.xmin = Xmin; + graphdata.ymin = Ymin; + graphdata.xmax = Xmax; + graphdata.ymax = Ymax; + KapaSetLimits (myKapa, &graphdata); + + for (int n = 0; n < nRegions; n++) { + + char regionName[64]; + snprintf (regionName, 64, "PSF.CLUMP.REGION.%03d", n); + psMetadata *regionMD = psMetadataLookupPtr (&status, analysis, regionName); + + float psfX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.X"); + float psfY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.Y"); + float psfdX = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DX"); + float psfdY = psMetadataLookupF32 (&status, regionMD, "PSF.CLUMP.DY"); + float Rx = psfdX * PSF_CLUMP_NSIGMA; + float Ry = psfdY * PSF_CLUMP_NSIGMA; + + for (int i = 0; i < xLimit->n; i++) { + xLimit->data.F32[i] = Rx*cos(i*2.0*M_PI/120.0) + psfX; + yLimit->data.F32[i] = Ry*sin(i*2.0*M_PI/120.0) + psfY; + } + KapaPrepPlot (myKapa, xLimit->n, &graphdata); + KapaPlotVector (myKapa, xLimit->n, xLimit->data.F32, "x"); + KapaPlotVector (myKapa, yLimit->n, yLimit->data.F32, "y"); + } + psFree (xLimit); + psFree (yLimit); + } + + psFree (xSAT); + psFree (ySAT); + psFree (mSAT); + psFree (sSAT); + + psFree (xEXT); + psFree (yEXT); + psFree (mEXT); + psFree (sEXT); + + psFree (xPSF); + psFree (yPSF); + psFree (mPSF); + psFree (sPSF); + + psFree (xDEF); + psFree (yDEF); + psFree (mDEF); + psFree (sDEF); + + psFree (xLOW); + psFree (yLOW); + psFree (mLOW); + psFree (sLOW); + + psFree (xCR); + psFree (yCR); + psFree (mCR); + psFree (sCR); + + pmVisualAskUser(NULL); + return true; +} + +bool PlotSourceSizeAltSetVectors(psVector *m, psVector *k, psVector *v1, psVector *v2, psVector *v3, pmSource *source) { + + float Mxx = source->moments->Mxx; + float Myy = source->moments->Myy; + float Mxy = source->moments->Mxy; + float Mminor = 0.5*(Mxx + Myy) - 0.5*sqrt(PS_SQR(Mxx - Myy) + 4.0*PS_SQR(Mxy)); + float KronMag = -2.5*log10(source->moments->KronFlux); + + psVectorAppend(m, source->psfMag); + psVectorAppend(k, KronMag); + psVectorAppend(v1, Mminor); + psVectorAppend(v2, source->psfMag - KronMag); + psVectorAppend(v3, source->extNsigma); + return true; +} + +bool PlotSourceSizeAltAddPoints(Graphdata *graphdata, int myKapa, psVector *x, psVector *y, char *colorname, int ptype, float size) { + + graphdata->color = KapaColorByName (colorname); + graphdata->ptype = ptype; + graphdata->size = size; + graphdata->style = 2; + KapaPrepPlot (myKapa, x->n, graphdata); + KapaPlotVector (myKapa, x->n, x->data.F32, "x"); + KapaPlotVector (myKapa, x->n, y->data.F32, "y"); + return true; +} + +bool psphotVisualPlotSourceSizeAlt (psMetadata *recipe, psMetadata *analysis, psArray *sources) { + + Graphdata graphdata; + KapaSection section; + + if (!pmVisualTestLevel("psphot.size", 2)) return true; + + int myKapa = psphotKapaChannel (2); + if (myKapa == -1) return false; + + KapaClearSections (myKapa); + KapaInitGraph (&graphdata); + KapaSetFont (myKapa, "courier", 14); + + section.bg = KapaColorByName ("none"); // XXX probably should be 'none' + + // storage vectors for data to be plotted + psVector *SATm = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *SATk = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *SAT1 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *SAT2 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *SAT3 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *PSFm = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *PSFk = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *PSF1 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *PSF2 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *PSF3 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *EXTm = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *EXTk = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *EXT1 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *EXT2 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *EXT3 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *DEFm = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *DEFk = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *DEF1 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *DEF2 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *DEF3 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *BADm = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *BADk = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *BAD1 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *BAD2 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *BAD3 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + psVector *CRm = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *CRk = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *CR1 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *CR2 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *CR3 = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + // int notPSF = PM_SOURCE_MODE_SATSTAR | PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_EXT_LIMIT | PM_SOURCE_MODE_DEFECT; + int nSkip = 0; + + // construct the vectors + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->moments == NULL) { + nSkip ++; + continue; + } + + bool found = false; + + // only plot the measured sources... + if (!(source->tmpFlags & PM_SOURCE_TMPF_SIZE_MEASURED)) { + nSkip ++; + continue; + } + + // any sources missing a large fraction should just be treated as PSFs + if ((source->pixWeightNotBad < 0.9) || (source->pixWeightNotPoor < 0.9)) { + PlotSourceSizeAltSetVectors(BADm, BADk, BAD1, BAD2, BAD3, source); + } + if ((source->mode & PM_SOURCE_MODE_CR_LIMIT) || (source->tmpFlags & PM_SOURCE_TMPF_SIZE_CR_CANDIDATE)) { + PlotSourceSizeAltSetVectors(CRm, CRk, CR1, CR2, CR3, source); + found = true; + } + if (source->mode & PM_SOURCE_MODE_SATSTAR) { + PlotSourceSizeAltSetVectors(SATm, SATk, SAT1, SAT2, SAT3, source); + found = true; + } + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) { + PlotSourceSizeAltSetVectors(EXTm, EXTk, EXT1, EXT2, EXT3, source); + found = true; + } + if (source->mode & PM_SOURCE_MODE_DEFECT) { + PlotSourceSizeAltSetVectors(DEFm, DEFk, DEF1, DEF2, DEF3, source); + found = true; + } + if (!found) { + PlotSourceSizeAltSetVectors(PSFm, PSFk, PSF1, PSF2, PSF3, source); + } + // if (!(source->mode & notPSF) && !(source->tmpFlags & PM_SOURCE_TMPF_SIZE_CR_CANDIDATE)) { + // PlotSourceSizeAltSetVectors(PSFm, PSFk, PSF1, PSF2, PSF3, source); + // } + } + // three sections: kronMag vs Mminor, psfMag vs psfMag - KronMag, psfMag vs extNsigma + + // --- first section: kronMag vs Mminor --- + section.dx = 1.00; + section.dy = 0.33; + section.x = 0.00; + section.y = 0.00; + section.name = psStringCopy ("Mminor"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = -17.1; + graphdata.xmax = -6.9; + graphdata.ymin = -0.5; + graphdata.ymax = +7.1; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = NAN; + graphdata.padYm = 5.0; + graphdata.padXp = NAN; + graphdata.padYp = NAN; + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "kron mag", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "M_minor| (pixels^2|)", KAPA_LABEL_YM); + + PlotSourceSizeAltAddPoints(&graphdata, myKapa, PSFk, PSF1, "black", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, EXTk, EXT1, "blue", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, CRk, CR1, "red", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, DEFk, DEF1, "red", 7, 1.0); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, SATk, SAT1, "blue", 7, 1.2); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, BADk, BAD1, "green", 7, 1.4); + + // --- second section: dMag ---- + section.dx = 1.00; + section.dy = 0.33; + section.x = 0.00; + section.y = 0.33; + section.name = psStringCopy ("dMag"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = -17.1; + graphdata.xmax = -6.9; + graphdata.ymin = -0.75; + graphdata.ymax = +1.50; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = NAN; + graphdata.padYm = 5.0; + graphdata.padXp = 0.0; + graphdata.padYp = NAN; + strcpy (graphdata.labels, "0200"); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "dMag", KAPA_LABEL_YM); + + PlotSourceSizeAltAddPoints(&graphdata, myKapa, PSFm, PSF2, "black", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, EXTm, EXT2, "blue", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, CRm, CR2, "red", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, DEFm, DEF2, "red", 7, 1.0); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, SATm, SAT2, "blue", 7, 1.2); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, BADm, BAD2, "green", 7, 1.4); + + // --- third section: nSigma --- + section.dx = 1.00; + section.dy = 0.33; + section.x = 0.00; + section.y = 0.66; + section.name = psStringCopy ("nSigma"); + KapaSetSection (myKapa, §ion); + psFree (section.name); + + graphdata.color = KapaColorByName ("black"); + graphdata.xmin = -17.1; + graphdata.xmax = -6.9; + graphdata.ymin = -10.1; + graphdata.ymax = +10.1; + KapaSetLimits (myKapa, &graphdata); + + graphdata.padXm = 0.0; + graphdata.padYm = 5.0; + graphdata.padXp = NAN; + graphdata.padYp = NAN; + strcpy (graphdata.labels, "0210"); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "psf msg", KAPA_LABEL_XP); + KapaSendLabel (myKapa, "EXT nSigma", KAPA_LABEL_YM); + + PlotSourceSizeAltAddPoints(&graphdata, myKapa, PSFm, PSF3, "black", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, EXTm, EXT3, "blue", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, CRm, CR3, "red", 0, 0.5); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, DEFm, DEF3, "red", 7, 1.0); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, SATm, SAT3, "blue", 7, 1.0); + PlotSourceSizeAltAddPoints(&graphdata, myKapa, BADm, BAD3, "green", 7, 1.4); + + fprintf (stderr, "PSF: %ld, EXT: %ld, CR: %ld, DEF: %ld, SAT: %ld, TOTAL: %ld, nSkip: %d\n", + PSFm->n, EXTm->n, CRm->n, DEFm->n, SATm->n, + PSFm->n+ EXTm->n+ CRm->n+ DEFm->n+ SATm->n, nSkip); + + psFree (SATk); + psFree (SATm); + psFree (SAT1); + psFree (SAT2); + psFree (SAT3); + + psFree (EXTk); + psFree (EXTm); + psFree (EXT1); + psFree (EXT2); + psFree (EXT3); + + psFree (PSFk); + psFree (PSFm); + psFree (PSF1); + psFree (PSF2); + psFree (PSF3); + + psFree (DEFk); + psFree (DEFm); + psFree (DEF1); + psFree (DEF2); + psFree (DEF3); + + psFree (BADk); + psFree (BADm); + psFree (BAD1); + psFree (BAD2); + psFree (BAD3); + + psFree (CRk); + psFree (CRm); + psFree (CR1); + psFree (CR2); + psFree (CR3); + + pmVisualAskUser(NULL); + return true; +} + +// option to redo variance since in some cases we may have displayed a different image in the meanwhile +bool psphotVisualShowResidualImage (pmReadout *readout, bool reshow) { + + if (!pmVisualTestLevel("psphot.image.resid", 2)) return true; + + int myKapa = psphotKapaChannel (1); + if (myKapa == -1) return false; + + float factor = 1.0; + if (readout->covariance) { + factor = psImageCovarianceFactorForAperture(readout->covariance, 10.0); + } + + if (false && reshow) { + psphotVisualShowMask (myKapa, readout->mask, "mask", 2); + psphotVisualScaleImage (myKapa, readout->variance, readout->mask, "variance", 1.0, 1); + } + + if (posImage) { + delImage = (psImage *) psBinaryOp(delImage, posImage, "-", readout->image); + psphotVisualScaleImage (myKapa, posImage, readout->mask, "posimage", sqrt(factor), 0); + psphotVisualScaleImage (myKapa, delImage, readout->mask, "delimage", sqrt(factor), 2); + } + + psphotVisualScaleImage (myKapa, readout->image, readout->mask, "resid", sqrt(factor), 1); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualPlotApResid (psArray *sources, float mean, float error, bool useApMag) { + + Graphdata graphdata; + float lineX[2], lineY[2]; + + if (!pmVisualTestLevel("psphot.apresid", 1)) return true; + + int myKapa = psphotKapaChannel (2); + if (myKapa == -1) return false; + + KapaClearSections (myKapa); + KapaInitGraph (&graphdata); + + psVector *x = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *y = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *dy = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + graphdata.xmin = +32.0; + graphdata.xmax = -32.0; + graphdata.ymin = +32.0; + graphdata.ymax = -32.0; + + // construct the plot vectors + int n = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!source) continue; + if (source->type != PM_SOURCE_TYPE_STAR) continue; + if (!isfinite (source->apMag)) continue; + if (!isfinite (source->psfMag)) continue; + + float dMag; + if (useApMag) { + dMag = source->apMag - source->psfMag; + } else { + float kMag = -2.5*log10(source->moments->KronFlux); + dMag = source->psfMag - kMag; + } + + x->data.F32[n] = source->psfMag; + y->data.F32[n] = dMag; + dy->data.F32[n] = source->psfMagErr; + graphdata.xmin = PS_MIN(graphdata.xmin, x->data.F32[n]); + graphdata.xmax = PS_MAX(graphdata.xmax, x->data.F32[n]); + graphdata.ymin = PS_MIN(graphdata.ymin, y->data.F32[n]); + graphdata.ymax = PS_MAX(graphdata.ymax, y->data.F32[n]); + + n++; + } + x->n = y->n = dy->n = n; + + float range; + range = graphdata.xmax - graphdata.xmin; + graphdata.xmax += 0.05*range; + graphdata.xmin -= 0.05*range; + range = graphdata.ymax - graphdata.ymin; + graphdata.ymax += 0.05*range; + graphdata.ymin -= 0.05*range; + + // XXX test + graphdata.xmin = -17.0; + graphdata.xmax = -9.0; + graphdata.ymin = -0.31; + graphdata.ymax = +0.31; + + KapaSetLimits (myKapa, &graphdata); + + KapaSetFont (myKapa, "helvetica", 14); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "PSF Mag", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "Ap Mag - PSF Mag", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + graphdata.etype |= 0x01; + KapaPrepPlot (myKapa, n, &graphdata); + KapaPlotVector (myKapa, n, x->data.F32, "x"); + KapaPlotVector (myKapa, n, y->data.F32, "y"); + KapaPlotVector (myKapa, n, dy->data.F32, "dym"); + KapaPlotVector (myKapa, n, dy->data.F32, "dyp"); + + graphdata.color = KapaColorByName ("blue"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 0; + graphdata.etype = 0; + lineX[0] = graphdata.xmin; + lineX[1] = graphdata.xmax; + lineY[0] = lineY[1] = mean; + KapaPrepPlot (myKapa, 2, &graphdata); + KapaPlotVector (myKapa, 2, lineX, "x"); + KapaPlotVector (myKapa, 2, lineY, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 0; + graphdata.etype = 0; + lineX[0] = graphdata.xmin; + lineX[1] = graphdata.xmax; + lineY[0] = lineY[1] = mean + error; + KapaPrepPlot (myKapa, 2, &graphdata); + KapaPlotVector (myKapa, 2, lineX, "x"); + KapaPlotVector (myKapa, 2, lineY, "y"); + + graphdata.color = KapaColorByName ("red"); + graphdata.ptype = 0; + graphdata.size = 0.5; + graphdata.style = 0; + graphdata.etype = 0; + lineX[0] = graphdata.xmin; + lineX[1] = graphdata.xmax; + lineY[0] = lineY[1] = mean - error; + KapaPrepPlot (myKapa, 2, &graphdata); + KapaPlotVector (myKapa, 2, lineX, "x"); + KapaPlotVector (myKapa, 2, lineY, "y"); + + psFree (x); + psFree (y); + psFree (dy); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualPlotChisq (psArray *sources) { + + Graphdata graphdata; + + if (!pmVisualTestLevel("psphot.chisq", 1)) return true; + + int myKapa = psphotKapaChannel (2); + if (myKapa == -1) return false; + + KapaClearSections (myKapa); + KapaInitGraph (&graphdata); + + psVector *x = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + psVector *y = psVectorAllocEmpty (sources->n, PS_TYPE_F32); + + graphdata.xmin = +32.0; + graphdata.xmax = -32.0; + graphdata.ymin = +32.0; + graphdata.ymax = -32.0; + + // construct the plot vectors + int n = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (!source) continue; + if (source->type != PM_SOURCE_TYPE_STAR) continue; + if (!source->moments) continue; + if (!isfinite(source->moments->Sum)) continue; + if (!source->modelPSF) continue; + if (!isfinite(source->modelPSF->chisq)) continue; + + x->data.F32[n] = -2.5*log10(source->moments->Sum); + y->data.F32[n] = source->modelPSF->chisq / source->modelPSF->nDOF; + graphdata.xmin = PS_MIN(graphdata.xmin, x->data.F32[n]); + graphdata.xmax = PS_MAX(graphdata.xmax, x->data.F32[n]); + graphdata.ymin = PS_MIN(graphdata.ymin, y->data.F32[n]); + graphdata.ymax = PS_MAX(graphdata.ymax, y->data.F32[n]); + n++; + } + x->n = y->n = n; + + float range; + range = graphdata.xmax - graphdata.xmin; + graphdata.xmax += 0.05*range; + graphdata.xmin -= 0.05*range; + range = graphdata.ymax - graphdata.ymin; + graphdata.ymax += 0.05*range; + graphdata.ymin -= 0.05*range; + + // XXX test + graphdata.xmin = -17.0; + graphdata.xmax = -3.0; + graphdata.ymin = -0.1; + graphdata.ymax = +10.1; + + KapaSetLimits (myKapa, &graphdata); + + KapaSetFont (myKapa, "helvetica", 14); + KapaBox (myKapa, &graphdata); + KapaSendLabel (myKapa, "PSF Mag", KAPA_LABEL_XM); + KapaSendLabel (myKapa, "ChiSq", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName ("black"); + graphdata.ptype = 2; + graphdata.size = 0.5; + graphdata.style = 2; + KapaPrepPlot (myKapa, n, &graphdata); + KapaPlotVector (myKapa, n, x->data.F32, "x"); + KapaPlotVector (myKapa, n, y->data.F32, "y"); + + psFree (x); + psFree (y); + + pmVisualAskUser(NULL); + return true; +} + +bool psphotVisualShowPetrosians (psArray *sources) { + + int Noverlay, NOVERLAY; + KiiOverlay *overlay; + + if (!pmVisualTestLevel("psphot.objects.petro", 2)) return true; + + int kapa = psphotKapaChannel (1); + if (kapa == -1) return false; + + Noverlay = 0; + NOVERLAY = 100; + ALLOCATE (overlay, KiiOverlay, NOVERLAY); + + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + + if (!source) continue; + if (!source->extpars) continue; + if (!source->extpars->petProfile) continue; + + float petrosianRadius = source->extpars->petrosianRadius; + psEllipseAxes *axes = &source->extpars->axes; + + overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + overlay[Noverlay].x = source->peak->xf; + overlay[Noverlay].y = source->peak->yf; + overlay[Noverlay].dx = 1.0*petrosianRadius; + overlay[Noverlay].dy = 1.0*petrosianRadius*axes->minor/axes->major; + overlay[Noverlay].angle = axes->theta * PS_DEG_RAD; + overlay[Noverlay].text = NULL; + Noverlay ++; + CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); + + // overlay[Noverlay].type = KII_OVERLAY_CIRCLE; + // overlay[Noverlay].x = source->peak->xf; + // overlay[Noverlay].y = source->peak->yf; + // overlay[Noverlay].dx = 2.0*petrosianRadius; + // overlay[Noverlay].dy = 2.0*petrosianRadius*axes->minor/axes->major; + // overlay[Noverlay].angle = axes->theta * PS_DEG_RAD; + // overlay[Noverlay].text = NULL; + // Noverlay ++; + // CHECK_REALLOCATE (overlay, KiiOverlay, NOVERLAY, Noverlay, 100); + } + + KiiLoadOverlay (kapa, overlay, Noverlay, "red"); + FREE (overlay); + + pmVisualAskUser(NULL); + return true; +} + +# else + +bool psphotVisualShowImage (pmConfig *config, pmReadout *readout) { return true; } +bool psphotVisualShowBackground (pmConfig *config, const pmFPAview *view, pmReadout *readout) { return true; } +bool psphotVisualShowSignificance (psImage *image) { return true; } +bool psphotVisualShowPeaks (pmConfig *config, const pmFPAview *view, pmDetections *detections) { return true; } +bool psphotVisualShowFootprints (pmConfig *config, const pmFPAview *view, pmDetections *detections) { return true; } +bool psphotVisualShowMoments (pmConfig *config, const pmFPAview *view, psArray *sources) { return true; } +bool psphotVisualPlotMoments (pmConfig *config, const pmFPAview *view, psArray *sources) { return true; } +bool psphotVisualShowRoughClass (pmConfig *config, const pmFPAview *view, psArray *sources) { return true; } +bool psphotVisualShowPSFStars (pmConfig *config, const pmFPAview *view, pmPSF *psf, psArray *sources) { return true; } +bool psphotVisualShowSatStars (pmConfig *config, const pmFPAview *view, pmPSF *psf, psArray *sources) { return true; } +bool psphotVisualShowPSFModel (pmConfig *config, pmReadout *readout, pmPSF *psf) { return true; } +bool psphotVisualShowFlags (pmConfig *config, const pmFPAview *view, psArray *sources) { return true; } +bool psphotVisualSourceSize (pmConfig *config, const pmFPAview *view, psArray *sources) { return true; } +bool psphotVisualShowResidualImage (pmConfig *config, pmReadout *readout) { return true; } +bool psphotVisualPlotApResid (pmConfig *config, const pmFPAview *view, psArray *sources) { return true; } + +# endif + +# if (0) +// *** make a histogram of the source counts in the x and y directions +psHistogram *nX = psHistogramAlloc (graphdata.xmin, graphdata.xmax, 50.0); +psHistogram *nY = psHistogramAlloc (graphdata.ymin, graphdata.ymax, 50.0); +psVectorHistogram (nX, xFaint, NULL, NULL, 0); +psVectorHistogram (nY, yFaint, NULL, NULL, 0); +psVector *dX = psVectorAlloc (nX->nums->n, PS_TYPE_F32); +psVector *vX = psVectorAlloc (nX->nums->n, PS_TYPE_F32); +psVector *dY = psVectorAlloc (nY->nums->n, PS_TYPE_F32); +psVector *vY = psVectorAlloc (nY->nums->n, PS_TYPE_F32); +for (int i = 0; i < nX->nums->n; i++) { + dX->data.F32[i] = nX->nums->data.S32[i]; + vX->data.F32[i] = 0.5*(nX->bounds->data.F32[i] + nX->bounds->data.F32[i+1]); +} +for (int i = 0; i < nY->nums->n; i++) { + dY->data.F32[i] = nY->nums->data.S32[i]; + vY->data.F32[i] = 0.5*(nY->bounds->data.F32[i] + nY->bounds->data.F32[i+1]); +} + +graphdata.color = KapaColorByName ("black"); +graphdata.ptype = 0; +graphdata.size = 0.0; +graphdata.style = 0; +KapaPrepPlot (myKapa, dX->n, &graphdata); +KapaPlotVector (myKapa, dX->n, dX->data.F32, "x"); +KapaPlotVector (myKapa, vX->n, vX->data.F32, "y"); + +psFree (nX); +psFree (dX); +psFree (vX); + +psFree (nY); +psFree (dY); +psFree (vY); + +# endif + Index: /branches/ccl_branches/ipponly-20191108/psphot/test/notes.txt =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/notes.txt (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/notes.txt (revision 41083) @@ -0,0 +1,5 @@ + +psphot tests + +This is a long-overdue set of psphot unit tests (or are these system tests?) + Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap.pro (revision 41083) @@ -0,0 +1,52 @@ +# -*-sh-*- + +macro tapOK + if ($0 != 3) + echo "USAGE: tapOK (condition) (message)" + break + end + + if ($1) + echo "ok : $2" + $TAP_LAST = 1 + else + echo "not ok : $2" + $TAP_NFAIL ++ + $TAP_LAST = 0 + if ($TAP_BREAK) + break + end + end + $TAP_NDONE ++ +end + +macro tapPLAN + if ($0 != 2) + echo "USAGE: tapPLAN (Ntests)" + break + end + + $TAP_NTEST = $1 + $TAP_NFAIL = 0 + $TAP_NDONE = 0 + if (not($?TAP_BREAK)) set TAP_BREAK = 0 +end + +macro tapDONE + if ($0 != 1) + echo "USAGE: tapDONE" + break + end + + if ($TAP_NDONE != $TAP_NTEST) + echo "planned tests ($TAP_NTEST) not equal to done tests ($TAP_NDONE)" + end + + if ($TAP_NFAIL) + echo "failed $TAP_NFAIL of $TAP_NDONE" + end + + if ($TAP_NFAIL == 0) + echo "passed $TAP_NDONE tests" + end +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_dense.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_dense.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_dense.pro (revision 41083) @@ -0,0 +1,788 @@ +#!/usr/bin/env mana +# -*-sh-*- + +$KAPA = kapa -noX + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM F + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE T -Db STARS.REAL F -Db MATCH.DENSITY F -Db PSF.CONVOLVE F +$RealConfig = $RealConfig -Df STARS.DENSITY 100.0 +$RealConfig = $RealConfig -Df STARS.SIGMA.LIM 2.5 +$RealConfig = $RealConfig -Db GALAXY.FAKE F -Df GALAXY.MAG 17.0 +$RealConfig = $RealConfig -Db GALAXY.GRID F -D GALAXY.MODEL PS_MODEL_SERSIC +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 +$RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 +$RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 +$RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + mkref + + local i Npass imname stname + + $Npass = $1 + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf N "%02d" $i + onestack $N + end +end + +macro summary.stats + if ($0 != 2) + echo "USAGE: summary.stats (Npass)" + break + end + + local i Npass base stack imname stname + + $Npass = $1 + + exec /bin/rm -f psf.image.summary.stats + exec /bin/rm -f psf.stack.inputs.summary.stats + exec /bin/rm -f psf.stack.conv.summary.stats + exec /bin/rm -f psf.stack.unconv.summary.stats + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf base "$imname.%02d/$imname.stats" $i + sprintf stack "$stname.%02d/$stname" $i + echo $i $base $stack + exec grep PSF_INST_MAG $base >> psf.image.summary.stats + exec grep PSF_INST_MAG $stack.conv.stats >> psf.stack.inputs.summary.stats + exec grep PSF_INST_MAG $stack.pht.stats >> psf.stack.conv.summary.stats + exec grep PSF_INST_MAG $stack.unconv.pht.stats >> psf.stack.unconv.summary.stats + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref -Db PSF.CONVOLVE T + + exec ppSim $RefOptions $RefConfig refimage -nx 3000 -ny 3000 + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +macro onestack + if ($0 != 2) + echo "USAGE: onestack (N)" + break + end + + mkstack image.$1/image stack.$1/stack + ckstack image.$1/image stack.$1/stack + # rmstack image.$1/image stack.$1/stack +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i base stack + + $base = $1 + $stack = $2 + + dirname $base -var dir + mkdir $dir + + dirname $stack -var dir + mkdir $dir + + exec echo "INPUT MULTI" > $stack.mdc + exec echo "" >> $stack.mdc + + for i 0 $fwhm:n + mkexp $base.$i $fwhm:$i + + exec echo "INPUT METADATA" >> $stack.mdc + exec echo "IMAGE STR $base.$i.wrp.fits" >> $stack.mdc + exec echo "MASK STR $base.$i.wrp.mask.fits" >> $stack.mdc + exec echo "VARIANCE STR $base.$i.wrp.wt.fits" >> $stack.mdc + exec echo "PSF STR $base.$i.wrp.psf" >> $stack.mdc + exec echo "SOURCES STR $base.$i.wrp.cmf" >> $stack.mdc + exec echo "END" >> $stack.mdc + exec echo "" >> $stack.mdc + end + + exec ppStack -threads 4 -input $stack.mdc $stack -Db TEMP.DELETE F -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE NONE + + basename $stack -var stackbase + for i 0 $fwhm:n + exec /bin/cp -f /tmp/$stackbase.$i.conv.im.fits $stack.$i.conv.im.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.var.fits $stack.$i.conv.var.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.mk.fits $stack.$i.conv.mk.fits + end + + # XXX note that the output convolved variance is missing the covariance + for i 0 $fwhm:n + exec psphot -file $stack.$i.conv.im.fits -mask $stack.$i.conv.mk.fits $stack.$i.conv + end + + # basic photometry for the stack and unconvolved stack + exec psphot -file $stack.fits -mask $stack.mask.fits -variance $stack.weight.fits $stack.pht + exec psphot -file $stack.unconv.fits -mask $stack.unconv.mask.fits -variance $stack.unconv.wt.fits $stack.unconv.pht +end + +# run this on the files created with mkstack +macro ckstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i ZPT base stack + + $base = $1 + $stack = $2 + + # generate plots and stats for the per-image chip analysis + # and the psf-matched convolved images + $ZPT = -2.5*log($fwhm:n) + for i 0 $fwhm:n + ckchip $base.$i.in.cmf $base.$i.cmf $base.$i 0.0 + ckwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.$i.conv $ZPT + + stchip $base.$i.in.cmf $base.$i.cmf $base.stats 0.0 + stwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.conv.stats $ZPT + + completeness $base.$i.in.cmf $base.$i.cmf $base.$i.complete.png + end + + ckwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht $ZPT + ckwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht $ZPT + + stwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht.stats $ZPT + stwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht.stats $ZPT +end + +macro rmstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local base stack + $base = $1 + $stack = $2 + + for i 0 $fwhm:n + exec rm -f $base.$i.fits + exec rm -f $base.$i.ch.fits + exec rm -f $base.$i.ch.mk.fits + exec rm -f $base.$i.ch.wt.fits + + exec rm -f $base.$i.wrp.fits + exec rm -f $base.$i.wrp.mask.fits + exec rm -f $base.$i.wrp.wt.fits + + exec rm -f $stack.$i.conv.im.fits + exec rm -f $stack.$i.conv.mk.fits + exec rm -f $stack.$i.conv.wt.fits + end + + exec rm -f $stack.fits + exec rm -f $stack.mask.fits + exec rm -f $stack.weight.fits + + exec rm -f $stack.exp.fits + exec rm -f $stack.num.fits + exec rm -f $stack.expwt.fits + + exec rm -f $stack.unconv.fits + exec rm -f $stack.unconv.mask.fits + exec rm -f $stack.unconv.wt.fits + + exec rm -f $stack.unconv.exp.fits + exec rm -f $stack.unconv.num.fits + exec rm -f $stack.unconv.expwt.fits +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 3) + echo "USAGE: mkexp basename fwhm" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm -nx 3000 -ny 3000 + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm -nx 3000 -ny 3000 + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stchip + if ($0 != 5) + echo "USAGE: stchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +macro stwarp + if ($0 != 5) + echo "USAGE: stwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +macro completeness + if ($0 != 4) + echo "USAGE: completeness (raw) (out) (output)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out 1.5 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + png -name $3 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y '$word:0' -x '$word:3' +end + +# this list is used to compare a pair of vectors with an error it is +# assumed that the vector sets have been loaded and matched with +# match2d to generate index vectors 'index1' and index2'. the macro +# show.dpair generates a plot of the range vector vs (v1 - v2) / dv1. +# The indices for v1, dv1, and v2 are index1,1, and 2 respectively. The +# index for the range vector is defined by the integer following that +# vector. The y-limits of the plot are given by the last two numbers +# (use 'def') for the full default range of the delta vector +list dpairs + # v1 dv v2 range + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro ckradialflux + data $1 + read -fits Chip.xrad X_APER Y_APER PSF_FWHM APER_FLUX APER_FLUX_ERR APER_FLUX_STDEV APER_FILL + read -fits Chip.psf X_PSF Y_PSF PSF_INST_MAG + set mag = PSF_INST_MAG + # XXX include nradii in header + for i 0 11 + set mA$i = -2.5*log(APER_FLUX:$i) + set dM$i = mA$i - PSF_INST_MAG + vstat dM$i + $DM$i = $MEDIAN + end + for i 1 11 + $j = $i - 1 + echo {$DM$i - $DM$j} + end + + lim mag -2 2; clear; box + for i 0 11 + plot mag dM$i -c black + end +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_basic.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_basic.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_basic.pro (revision 41083) @@ -0,0 +1,211 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# macros to support tap tests in mana/dvo +input tap.pro + +# create a fake image (ppSim, all stars are fake) +macro make.fake + + local options + + # basic options for these image (r, 100sec) + $options = -type OBJECT -filter r -exptime 100.0 + $options = $options -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 -seeing 1.0 + $options = $options -D PSF.MODEL PS_MODEL_GAUSS + + $options = $options -Df STARS.DENSITY 20.0 + + $options = $options -camera SIMTEST -recipe PPSIM STACKTEST.MAKE + exec ppSim $options testimage +end + +macro run.psphot + exec psphot -file testimage.fits testimage.phot +end + +macro test.init + data testimage.phot.cmf + + # a bit hackish : cannot append to non-existent scalar + $fields = Chip.psf + for i 0 $pairs:n + list word -split $pairs:$i + $fields = $fields $word:0 + end + + read -fits $fields + + data testimage.dat + read Xraw 1 Yraw 2 Fraw 3 Galaxy 4 Mraw 5 dMraw 6 SXXraw 7 SYYraw 8 SXYraw 9 S7raw 10 + set dFraw = Fraw * dMraw + set dXraw = 4.0 * dMraw / 2.35 + set dYraw = 4.0 * dMraw / 2.35 + + match2d X_PSF Y_PSF Xraw Yraw 2.0 -index1 index1 -index2 index2 +end + +macro test.plots + if ($0 != 2) + echo "USAGE: test.plots (outroot)" + break + end + + clear -s + + resize 1200 1175 + + $nx = 0 + $ny = 0 + $np = 0 + for i 0 $pairs:n + section a$i {0.5*$nx} {0.8 - 0.2*$ny} 0.5 0.2 + show.pair $i + $ny ++ + if ($ny == 5) + $nx ++ + $ny = 0 + end + if ($nx == 2) + png -name $1.$np.png + clear -s + $np ++ + $nx = 0 + end + end + png -name $1.$np.png +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y $word:0 -x $word:3 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y $word:0 -x $word:3 +end + +macro show.pair.old + set delta = $1 - $2 + lim mag2 delta; clear; box; plot mag2 delta +end + +# list the entries that should be compared and the vector to use as a range +list dpairs + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# list the entries that should be compared and the vector to use as a range +list pairs + X_PSF Xraw Mraw 2 -1.01 1.01 V + Y_PSF Yraw Mraw 2 -1.01 1.01 V + X_PSF_SIG dXraw Mraw 2 -1.01 1.01 V + Y_PSF_SIG dYraw Mraw 2 -1.01 1.01 V + PSF_INST_MAG Mraw Mraw 2 -1.01 1.01 V + PSF_INST_MAG_SIG 0.0 Mraw 2 -0.01 1.01 S + PSF_INST_FLUX Fraw Mraw 2 def def V + PSF_INST_FLUX_SIG dFraw Mraw 2 def def V + AP_MAG Mraw Mraw 2 -1.01 1.01 V + AP_MAG_RAW Mraw Mraw 2 -1.01 1.01 V + AP_MAG_RADIUS 0.0 Mraw 2 -0.01 20.1 S + SKY 0.0 Mraw 2 def def S + SKY_SIGMA 0.0 Mraw 2 def def S + PSF_CHISQ 1.0 Mraw 2 def def S + CR_NSIGMA 0.0 Mraw 2 def def S + EXT_NSIGMA 0.0 Mraw 2 -5.01 5.01 S + PSF_MAJOR 0.0 Mraw 2 -0.01 5.01 S + PSF_MINOR 0.0 Mraw 2 -0.01 5.01 S + PSF_THETA 0.0 Mraw 2 -1.61 1.61 S + PSF_QF 0.0 Mraw 2 -0.10 1.10 S + PSF_QF_PERFECT 0.0 Mraw 2 -0.10 1.10 S + PSF_NDOF 0.0 Mraw 2 def def S + PSF_NPIX 0.0 Mraw 2 def def S + MOMENTS_XX 0.0 Mraw 2 -0.01 3.01 S + MOMENTS_XY 0.0 Mraw 2 -3.01 3.01 S + MOMENTS_YY 0.0 Mraw 2 -0.01 3.01 S + MOMENTS_M3C 0.0 Mraw 2 -3.01 3.01 S + MOMENTS_M3S 0.0 Mraw 2 -3.01 3.01 S + MOMENTS_M4C 0.0 Mraw 2 -2.01 2.01 S + MOMENTS_M4S 0.0 Mraw 2 -2.01 2.01 S + MOMENTS_R1 0.0 Mraw 2 -5.01 5.01 S + MOMENTS_RH 0.0 Mraw 2 -5.01 5.01 S + KRON_FLUX 0.0 Mraw 2 def def S + KRON_FLUX_ERR 0.0 Mraw 2 def def S + KRON_FLUX_INNER 0.0 Mraw 2 def def S + KRON_FLUX_OUTER 0.0 Mraw 2 def def S +# CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +if ($SCRIPT) + make.fake + exit 0 +end + + + # write tmp.dat X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG + # exec gcompare -m tmp.dat 1 2 testimage.dat 1 2 2.0 > tmp.match.dat + # + # data tmp.match.dat + # read x1 1 y1 2 dx1 3 dy1 4 psfmag1 5 dpsfmag1 6 x2 7 y2 8 flux 9 galaxy 10 mag2 11 dmag2 12 sxx 13 syy 14 sxy 15 s7 16 Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_deteff.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_deteff.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_deteff.pro (revision 41083) @@ -0,0 +1,396 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# detection efficiency: +# * does it depend on the stellar density? +# * does it scale correctly with the depth? + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 +$BaseOptions = $BaseOptions -nx 2500 -ny 2500 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 +$RefOptions = $RefOptions -Db PSF.CONVOLVE T +# add the density in mkref +# $RefOptions = $RefOptions -Df STARS.DENSITY 10.0 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE T +# we add the catdir in mkexp so we can change the refbase +# $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR catdir.$refbase + +# add galaxies for this test or not? +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + mkref + + local i Npass imname stname + + $Npass = $1 + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf N "%02d" $i + onestack $N + end +end + +macro summary.stats + if ($0 != 2) + echo "USAGE: summary.stats (Npass)" + break + end + + local i Npass base stack imname stname + + $Npass = $1 + + exec /bin/rm -f psf.image.summary.stats + exec /bin/rm -f psf.stack.inputs.summary.stats + exec /bin/rm -f psf.stack.conv.summary.stats + exec /bin/rm -f psf.stack.unconv.summary.stats + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf base "$imname.%02d/$imname.stats" $i + sprintf stack "$stname.%02d/$stname" $i + echo $i $base $stack + exec grep PSF_INST_MAG $base >> psf.image.summary.stats + exec grep PSF_INST_MAG $stack.conv.stats >> psf.stack.inputs.summary.stats + exec grep PSF_INST_MAG $stack.pht.stats >> psf.stack.conv.summary.stats + exec grep PSF_INST_MAG $stack.unconv.pht.stats >> psf.stack.unconv.summary.stats + end +end + +macro onestack + if ($0 != 2) + echo "USAGE: onestack (N)" + break + end + + mkstack image.$1/image stack.$1/stack + ckstack image.$1/image stack.$1/stack + rmstack image.$1/image stack.$1/stack +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + if ($0 != 3) + echo "mkref (refbase) (density)" + break + end + + local refbase + $refbase = $1 + + exec rm -rf $refbase.catdir + exec rm -f $refbase.fits + + $RefOptions = $RefOptions -Df STARS.DENSITY $2 + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + exec ppSim $RefOptions $RefConfig $refbase + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR $refbase.catdir -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes $refbase.cmf + exec relphot -averages -D CATDIR $refbase.catdir -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 4) + echo "USAGE: mkexp basename fwhm refbase" + break + end + + local fwhm basename refbase + $basename = $1 + $fwhm = $2 + $refbase = $3 + + $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename +end + +macro completeness + if ($0 != 5) + echo "USAGE: completeness (raw) (out) (output) (radius)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + line -c red -20 {log(1/25.0)} to 20 {log(1/25.0)} + line -c blue -20 {log(1/5.0)} to 20 {log(1/5.0)} + + png -name $3 +end + +macro completestack + if ($0 != 6) + echo "USAGE: completeness (raw) (out) (output) (radius) (zeropt)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set Mraw = PSF_INST_MAG_raw + $5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + histogram Mraw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (Mraw > dMag[$i]) && (Mraw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot Mraw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot Mraw ldmag + label +y log(S/N) + + line -c red -20 {log(1/25.0)} to 20 {log(1/25.0)} + line -c blue -20 {log(1/5.0)} to 20 {log(1/5.0)} + + png -name $3 +end + +macro falsepositive + if ($0 != 5) + echo "USAGE: falsepositive (raw) (out) (output) (radius)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_out nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + set fAll = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_out > dMag[$i]) && (PSF_INST_MAG_out <= dMag[$i+1]) + subset all = index2 if (inrange) + subset got = index2 if (inrange) && (index2 == -1) + if (all[] == 0) + fMag[$i] = 0 + else + # fraction of detections that are false + fMag[$i] = got[] / all[] + end + # fraction of detections from this bin that are fake + fAll[$i] = got[] / PSF_INST_MAG_out[] + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + # lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + lim dMag fAll; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fAll + label -x mag_inst -y false_frac + + section overlay1 0 0 1 1 + lim dMag fMag; plot -x 1 dMag fMag -c blue + + set found = (index2 >= 0) + plot PSF_INST_MAG_out found -c red + + set ldmag = log(PSF_INST_MAG_SIG_out) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_out ldmag + label +y log(S/N) + + line -c red -20 {log(1/25.0)} to 20 {log(1/25.0)} + line -c blue -20 {log(1/5.0)} to 20 {log(1/5.0)} + + png -name $3 +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +# for completeness, we only need a few of the fields +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG +# PSF_INST_FLUX +# PSF_INST_FLUX_SIG +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_diff.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_diff.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_diff.pro (revision 41083) @@ -0,0 +1,576 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE F + +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.00 + 1.25 +end + +list fwhmMax + 1.25 + 1.00 +end + +list aratio + 1.0 + 1.0 +end + +list thetas + 0.0 + 0.0 +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkdiff + if ($0 != 5) + echo "USAGE: mkdiff (rawbase) (diffbase) (init) (order)" + break + end + + # XXX check for local reference db and run mkref if not found + file catdir.ref found + if (not($found)) + echo "make reference catalog" + mkref + end + + local i base diff + + $base = $1 + $diff = $2 + + dirname $base -var dir + mkdir $dir + + dirname $diff -var dir + mkdir $dir + + $command = ppSub + + for i 0 $fwhm:n + if ($3 == 1) + mkexp $base.$i $fwhm:$i $aratio:$i $thetas:$i $fwhmMax:$i + end + + if ($i == 0) + $command = $command -inimage $base.$i.wrp.fits + $command = $command -inmask $base.$i.wrp.mask.fits + $command = $command -invariance $base.$i.wrp.wt.fits + $command = $command -insources $base.$i.wrp.cmf + else + $command = $command -refimage $base.$i.wrp.fits + $command = $command -refmask $base.$i.wrp.mask.fits + $command = $command -refvariance $base.$i.wrp.wt.fits + $command = $command -refsources $base.$i.wrp.cmf + end + end + +# $command = $command -recipe PSPHOT DIFF +# $command = $command -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE +# $command = $command -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF +# $command = $command -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF -D PSPHOT:OUTPUT.FORMAT PS1_DV2 + $command = $command -Db DUAL F -Db ADD.NOT.SUBTRACT T + $command = $command -Db INVERSE T -convolve 1 -Di SPATIAL.ORDER $4 + $command = $command -save-inconv -save-refconv + $command = $command -threads 4 + $command = $command -image_id 1 -source_id 1 + + echo $command $diff + # exec $command $diff +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 6) + echo "USAGE: mkexp basename fwhm aratio theta fwhmMax" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + $aratio = $3 + $theta = $4 + $fwhmMax = $5 + + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits -D PSPHOT:OUTPUT.FORMAT PS1_V2 +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (Next)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + + # M_raw is the predicted mag based on the raw input mags (need to add the zero point) + + set M_raw = PSF_INST_MAG_raw - 2.5*log($4) + set dM_raw = PSF_INST_MAG_SIG_raw / sqrt($4) + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out dM_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end + +macro ckvar + if ($0 != 2) + echo "ckvar (buffer)" + break + end + + cursor + stats $1 {$X1-10} {$Y1-10} 21 21 + $K2sum0 = $TOTAL + stats $1 {$X2-10} {$Y2-10} 21 21 + $K2sum1 = $TOTAL + echo {$K2sum0 * 193 + $K2sum1 * 193} +end + +macro ckresid + + rd res resid.fits + rd con1 conv1.fits + rd con2 conv2.fits + dev -n im; tvch 1; tv res -50 100; tvch 2; tv con1 -20 100; tvch 3; tv con2 -20 100 +end + +macro ckchisq + data chisq2.dat + read flux2 3 Mxx 9 Myy 11 chisq 13 npix 15 + set chisqv = chisq / npix + set flux = sqrt(flux2) + + set lflux = log(flux) + set lchi = log(chisqv) + set lMxx = log(Mxx) + + set dev = sqrt(chisq) + set ndev = sqrt(chisq)/npix + set lndev = log(ndev) + + lim -n 0 lchi lMxx; clear; box; plot lchi lMxx + lim -n 1 lflux lMxx; clear; box; plot lflux lMxx + lim -n 2 lflux lchi; clear; box; plot lflux lchi + lim -n 3 lflux lndev; clear; box; plot lflux lndev + + # set flux2 = flux + # set flux = sqrt(flux2) + # set dev = sqrt(chisq) + # set ndev = sqrt(chisq)/npix + # set lndev = log(ndev) + # set lflux = log(flux) + # lim -n 0 lflux lndev; clear; box; plot lflux lndev + # lim -n 1 lndev lMxx; clear; box; plot lndev lMxx + +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_diffsuite.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_diffsuite.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_diffsuite.pro (revision 41083) @@ -0,0 +1,629 @@ +#!/usr/bin/env mana +# -*-sh-*- + +if (not($?TESTING)) set TESTING = 0 + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE F + +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list params + 1.00 1.00 1.0 0.0 + 1.10 1.10 1.0 0.0 + 1.25 1.25 1.0 0.0 + 1.40 1.40 1.0 0.0 + 1.00 1.00 1.3 30.0 + 1.10 1.10 1.3 30.0 + 1.25 1.25 1.3 30.0 + 1.40 1.40 1.3 30.0 + 1.00 1.00 1.3 -30.0 + 1.10 1.10 1.3 -30.0 + 1.25 1.25 1.3 -30.0 + 1.40 1.40 1.3 -30.0 + 1.00 1.25 1.0 0.0 + 1.10 1.25 1.0 0.0 + 1.25 1.25 1.0 0.0 + 1.40 1.25 1.0 0.0 + 1.00 1.25 1.3 30.0 + 1.10 1.25 1.3 30.0 + 1.25 1.25 1.3 30.0 + 1.40 1.25 1.3 30.0 + 1.00 1.25 1.3 -30.0 + 1.10 1.25 1.3 -30.0 + 1.25 1.25 1.3 -30.0 + 1.40 1.25 1.3 -30.0 +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a set of input exposures using the range of parameters given in the lists +macro mkdiffsuite + if ($0 != 3) + echo "USAGE: mkdiffsuite (rawbase) (diffbase)" + break + end + + local i1 i2 rawbase diffbase + $rawbase = $1 + $diffbase = $2 + + for i1 0 $params:n + for i2 {$i1 + 1} $params:n + mkdiff $rawbase $diffbase.$i1.$i2 $i1 $i2 + end + end +end + +# create diff for a single pair of input +macro mkdiff + if ($0 != 5) + echo "USAGE: mkdiff (rawbase) (diffbase) (in1) (in2)" + break + end + + # XXX check for local reference db and run mkref if not found + + local in1 in2 diff + + $base = $1 + + $diff = $2 + dirname $diff -var dir + mkdir $dir + + $in1 = $3 + $in2 = $4 + + $command = ppSub + + $command = $command -inimage $base.$in1.wrp.fits + $command = $command -inmask $base.$in1.wrp.mask.fits + $command = $command -invariance $base.$in1.wrp.wt.fits + $command = $command -insources $base.$in1.wrp.cmf + + $command = $command -refimage $base.$in2.wrp.fits + $command = $command -refmask $base.$in2.wrp.mask.fits + $command = $command -refvariance $base.$in2.wrp.wt.fits + $command = $command -refsources $base.$in2.wrp.cmf + +# no photometry for now +# $command = $command -recipe PSPHOT DIFF +# $command = $command -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE +# $command = $command -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF +# $command = $command -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF -D PSPHOT:OUTPUT.FORMAT PS1_DV2 + + $command = $command -Db DUAL T -Db INVERSE T -convolve 1 -Df PENALTY 1000000 -Di SPATIAL.ORDER 1 + # $command = $command -Db DUAL T -Db INVERSE T -convolve 1 -Df PENALTY 3000000 -Di SPATIAL.ORDER 1 + # $command = $command -Db DUAL T -Db INVERSE T -convolve 1 -Df PENALTY 3000000000 -Di SPATIAL.ORDER 1 + $command = $command -Di KERNEL.SIZE 5 -Di STAMP.FOOTPRINT 5 + + $command = $command -save-inconv -save-refconv + $command = $command -threads 4 + $command = $command -image_id 1 -source_id 1 + + echo $command $diff + if ($TESTING) + exec $command $diff -visual ppsub 10 + else + exec $command $diff >& $diff.log + end +end + +# create a set of input exposures using the range of parameters given in the lists +macro mkexpsuite + if ($0 != 2) + echo "USAGE: mkexpsuite (rawbase)" + break + end + + local i base + + $base = $1 + + dirname $base -var dir + mkdir $dir + + for i 0 $params:n + list words -split $params:$i + mkexp $base.$i $words:0 $words:1 $words:2 $words:3 + end +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 6) + echo "USAGE: mkexp basename fwhmMin fwhmMax aratio theta" + break + end + + local fwhm basename + $basename = $1 + $fwhmMin = $2 + $fwhmMax = $3 + $aratio = $4 + $theta = $5 + + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhmMin -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhmMin -seeing-max $fwhmMax -seeing-ramp -Df PSF.ARATIO $aratio -Df PSF.THETA $theta + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits -D PSPHOT:OUTPUT.FORMAT PS1_V2 +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro ckvar + if ($0 != 2) + echo "ckvar (buffer)" + break + end + + cursor + stats $1 {$X1-10} {$Y1-10} 21 21 + $K2sum0 = $TOTAL + stats $1 {$X2-10} {$Y2-10} 21 21 + $K2sum1 = $TOTAL + echo {$K2sum0 * 193 + $K2sum1 * 193} +end + +macro ckresid + + rd res resid.fits + rd con1 conv1.fits + rd con2 conv2.fits + dev -n im; tvch 1; tv res -50 100; tvch 2; tv con1 -20 100; tvch 3; tv con2 -20 100 +end + +macro ckchisq + data chisq2.dat + read flux2 3 Mxx 9 Myy 11 chisq 13 npix 15 + set chisqv = chisq / npix + set flux = sqrt(flux2) + + set lflux = log(flux) + set lchi = log(chisqv) + set lMxx = log(Mxx) + + set dev = sqrt(chisq) + set ndev = sqrt(chisq)/npix + set lndev = log(ndev) + + lim -n 0 lchi lMxx; clear; box; plot lchi lMxx + lim -n 1 lflux lMxx; clear; box; plot lflux lMxx + lim -n 2 lflux lchi; clear; box; plot lflux lchi + lim -n 3 lflux lndev; clear; box; plot lflux lndev + + # set flux2 = flux + # set flux = sqrt(flux2) + # set dev = sqrt(chisq) + # set ndev = sqrt(chisq)/npix + # set lndev = log(ndev) + # set lflux = log(flux) + # lim -n 0 lflux lndev; clear; box; plot lflux lndev + # lim -n 1 lndev lMxx; clear; box; plot lndev lMxx + +end + +macro ckscore + if ($0 != 3) + echo "USAGE: ckscore (diffbase) (output)" + break + end + + local i j diffbase + $diffbase = $1 + + output $2 + for i 0 $params:n + for j {$i+1} $params:n + $line = `grep applying $diffbase.$i.$j.log` + list word -split $line + $order = $word:2 + $mode = $word:4 + $val = `grep chisq: $diffbase.$i.$j.log | head -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 2 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 3 | tail -n 1` + sprintf chisq3 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + echo "$order $mode $chisq1 $chisq2 $chisq3 $params:$i $params:$j" + end + end + output stdout +end + +macro ckslope + if ($0 != 3) + echo "USAGE: ckslope (diffbase) (output)" + break + end + + local i j diffbase + $diffbase = $1 + + output $2 + for i 0 $params:n + for j {$i+1} $params:n + $line = `grep applying $diffbase.$i.$j.log` + list word -split $line + $order = $word:2 + $mode = $word:4 + if ($mode == 1) + $val = `grep chisq: $diffbase.$i.$j.log | head -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 4 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + end + if ($mode == 2) + $val = `grep chisq: $diffbase.$i.$j.log | head -n 2 | tail -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 5 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + end + if ($mode == 4) + $val = `grep chisq: $diffbase.$i.$j.log | head -n 3 | tail -n 1` + sprintf chisq1 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + $val = `grep chisq: $diffbase.$i.$j.log | head -n 6 | tail -n 1` + sprintf chisq2 "%s %6.3f %s %6.3f %s %6.3f %s %6.3f %s %6.3f" $val + end + echo "$order $mode $chisq1 $chisq2 $params:$i $params:$j" + end + end + output stdout +end + +macro plotscore + if ($0 != 2) + echo "USAGE: plotscore (input)" + break + end + + data $1 + read x1 4 f1 10 x2 14 f2 20 xd 24 fd1 30 fd2 32 + set X1 = 2 * x1 / (1 + f1) + set X2 = 2 * x2 / (1 + f2) + set Xd = 2 * xd / (fd1 + fd2) + + subset X1_1 = X1 if (X1 < X2) && (X1 < Xd) + subset X2_1 = X2 if (X1 < X2) && (X1 < Xd) + subset Xd_1 = Xd if (X1 < X2) && (X1 < Xd) + + subset X1_2 = X1 if (X1 > X2) && (X2 < Xd) + subset X2_2 = X2 if (X1 > X2) && (X2 < Xd) + subset Xd_2 = Xd if (X1 > X2) && (X2 < Xd) + + subset X1_d = X1 if (Xd < X2) && (X1 > Xd) + subset X2_d = X2 if (Xd < X2) && (X1 > Xd) + subset Xd_d = Xd if (Xd < X2) && (X1 > Xd) + + lim -n 0 0.3 1.1 0.3 1.1; clear; box + line 0 0 to 2 2 + + plot X1_1 X2_1 -pt 0 -c blue -sz 0.5 + plot X1_2 X2_2 -pt 2 -c red + plot X1_d X2_d -pt 7 -c black + + lim -n 1 0.3 1.1 0.3 1.1; clear; box + line 0 0 to 2 2 + + plot X1_1 Xd_1 -pt 0 -c blue -sz 0.5 + plot X1_2 Xd_2 -pt 2 -c red + plot X1_d Xd_d -pt 7 -c black + + lim -n 2 0.3 1.1 0.3 1.1; clear; box + line 0 0 to 2 2 + + plot X2_1 Xd_1 -pt 0 -c blue -sz 0.5 + plot X2_2 Xd_2 -pt 2 -c red + plot X2_d Xd_d -pt 7 -c black + +end + +if ($SCRIPT) + # mkexpsuite image/image + mkdiffsuite image/image diff.wt4/diff + exit 0 +end + Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_forced.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_forced.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_forced.pro (revision 41083) @@ -0,0 +1,264 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# forced photometry + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 +$BaseOptions = $BaseOptions -nx 2500 -ny 2500 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 +$RefOptions = $RefOptions -Db PSF.CONVOLVE T +# add the density in mkref +# $RefOptions = $RefOptions -Df STARS.DENSITY 10.0 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE T +# we add the catdir in mkexp so we can change the refbase +# $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR catdir.$refbase + +# add galaxies for this test or not? +if (1) + $RealConfig = $RealConfig -Db GALAXY.FAKE F +else + $RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 + $RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 + $RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 + $RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 + $RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 + $RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 + $RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 +end + +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + local i Npass + $Npass = $1 + + # use a limiting stellar density of 1 / deg^2 + mkref ref 1.0 + + # use a FWHM of 1.0 arcsec + mkexp image/image 1.0 ref + + for i 0 $Npass + mkforce image/image v$i + forcedCompare image/image.cmf image/image.v$i.frc.cmf image/image.v$i.frc.png 1.0 + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + if ($0 != 3) + echo "mkref (refbase) (density)" + break + end + + local refbase + $refbase = $1 + + exec rm -rf $refbase.catdir + exec rm -f $refbase.fits + + $RefOptions = $RefOptions -Df STARS.DENSITY $2 + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + exec ppSim $RefOptions $RefConfig $refbase + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR $refbase.catdir -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes $refbase.cmf + exec relphot -averages -D CATDIR $refbase.catdir -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 4) + echo "USAGE: mkexp basename fwhm refbase" + break + end + + local fwhm basename refbase + $basename = $1 + $fwhm = $2 + $refbase = $3 + + $dirname = `dirname $basename` + exec mkdir -p $dirname + + $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR $refbase.catdir + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename -seed 12345 + exec ppImage $ppImageConfig -file $basename.fits $basename -seed 12345 +end + +macro mkforce + if ($0 != 3) + echo "USAGE: mkforce (basename) (version) " + break + end + + local basename + $basename = $1 + echo $basename + + data $basename.cmf + read -fits Chip.psf X_PSF Y_PSF + write $basename.$2.frc.dat X_PSF Y_PSF + echo "here" + + $forcedOpt = -file $basename.ch.fits + $forcedOpt = $forcedOpt -mask $basename.ch.mk.fits + $forcedOpt = $forcedOpt -variance $basename.ch.wt.fits + $forcedOpt = $forcedOpt -psf $basename.psf + $forcedOpt = $forcedOpt -srctext $basename.$2.frc.dat + exec psphotForced $forcedOpt $basename.$2.frc -seed 12345 +end + +macro forcedCompare + if ($0 != 5) + echo "USAGE: forcedCompare (raw) (out) (output) (radius)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # if we compare to the pre-convolution deltas, we need to do this: + # set X_raw = int(X_PSF_raw) + 0.5 + # set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out $4 -index1 index1 -index2 index2 -closest + + set m1 = PSF_INST_MAG_raw + set m1err = PSF_INST_MAG_SIG_raw + set m1err10 = PSF_INST_MAG_SIG_raw / 10.0 + reindex m2 = PSF_INST_MAG_out using index1 + set dm = m1 - m2 + + resize 1000 600 + label -fn courier 14 + lim m1 dm; clear; box; plot m1 dm -dy m1err10 + + set chi = dm^2 / m1err^2 + vstat -q chi + echo "median offset: $MEDIAN, chisq : $TOTAL, reduced chisq : {$TOTAL / $NPTS}" + + sprintf line "median offset: %6.4f, chisq : %6.4f, reduced chisq : %6.4f" $MEDIAN $TOTAL {$TOTAL / $NPTS} + label -x "standard inst mag" -y "standard mag - forced mag" -ur "$line" + label -ul "plotted errors are poisson / 10.0 for clarity" + png -name $3 +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +# for completeness, we only need a few of the fields +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG +# PSF_INST_FLUX +# PSF_INST_FLUX_SIG +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_galaxies.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_galaxies.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_galaxies.pro (revision 41083) @@ -0,0 +1,647 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT +$BaseOptions = $BaseOptions -filter r +$BaseOptions = $BaseOptions -skymags 20.86 +$BaseOptions = $BaseOptions -ra 270.70 +$BaseOptions = $BaseOptions -dec -23.70 +$BaseOptions = $BaseOptions -pa 0.0 +# $BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions +$RefOptions = $RefOptions -exptime 100.0 +$RefOptions = $RefOptions -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.DENSITY 10.0 +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 + +# options for the simulated images (using the refimage for the stars) +$FakeOptions = $BaseOptions +$FakeOptions = $FakeOptions -exptime 30.0 +# $FakeOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +$FakeOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +# sample alternate options: +# $FakeOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $FakeOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $FakeOptions = $FakeOptions -Df PSF.THETA +30.0 + +# create an image with fake sources (these are then inserted into the catdir) +$RefConfig = -camera SIMTEST +$RefConfig = $RefConfig -recipe PPSIM STACKTEST.MAKE + +# basic config for ppSim with randomly distributed stars and gridded galaxies +if (1) + $FakeConfig = -camera SIMTEST + $FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN + $FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref + $FakeConfig = $FakeConfig -Db MATCH.DENSITY F + $FakeConfig = $FakeConfig -Db PSF.CONVOLVE T + $FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies + $FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 + $FakeConfig = $FakeConfig -Db GALAXY.GRID F ; # generate a grid of galaxies (constant mag) + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_GAUSS + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_EXP + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_SERSIC + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_DEV + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN 10.0 + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX 1.0 + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN 0.25 + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX 1.00 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.66 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.66 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 120 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 120 +end + +# basic config for ppSim with randomly distributed stars and gridded galaxies +if (0) + $FakeConfig = -camera SIMTEST + $FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN + $FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref + $FakeConfig = $FakeConfig -Db MATCH.DENSITY F + $FakeConfig = $FakeConfig -Db PSF.CONVOLVE T + $FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies + $FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 + $FakeConfig = $FakeConfig -Db GALAXY.GRID T ; # generate a grid of galaxies (constant mag) + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_GAUSS + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_EXP + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_SERSIC + #$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_DEV + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN 10.0 + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX 10.0 + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN 0.25 + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX 0.25 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.66 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.66 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 120 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 120 +end + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro go + mkexp test.exp 1.0 EXP + fitexp test.exp test.exp.fit EXP_CONV + + mkexp test.ser 1.0 SERSIC + fitexp test.ser test.ser.fit SER_CONV + + mkexp test.dev 1.0 DEV + fitexp test.dev test.dev.fit DEV_CONV + + mkexp test.gau 1.0 GAUSS + fitexp test.gau test.gau.fit GAU_CONV + + mkexp test.pg 1.0 PGAUSS + fitexp test.pg test.pg.fit PGA_CONV + + mkexp test.qga 1.0 QGAUSS + fitexp test.qga test.qga.fit QGA_CONV + + mkexp test.p1 1.0 PS1_V1 + fitexp test.p1 test.p1.fit PS1_CONV +end + +macro go.ckgalaxy + foreach type exp ser dev gau pg qga p1 + ckgalaxy test.$type.dat test.$type.fit.cmf + wait $type + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + echo ppSim $RefOptions $RefConfig refimage + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf -quick-airmass + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 4) + echo "USAGE: mkexp basename (fwhm) (model)" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # create the raw image + $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_$3 + echo ppSim $FakeOptions $FakeConfig $basename -seeing $fwhm + exec ppSim $FakeOptions $FakeConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + #echo ppImage $ppImageConfig -file $basename.fits $basename + #exec ppImage $ppImageConfig -file $basename.fits $basename +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro fitexp + if ($0 != 4) + echo "USAGE: fitexp basename outname (fitModel)" + break + end + + local basename fitModel outname + $basename = $1 + $outname = $2 + $fitModel = $3 + + $psphotConfig = + $psphotConfig = $psphotConfig -recipe PSPHOT GALAXY_TEST + $psphotConfig = $psphotConfig -threads 4 + $psphotConfig = $psphotConfig -Db PSPHOT:LMM_FIT_CHISQ_CONVERGENCE F + $psphotConfig = $psphotConfig -Df PSPHOT:EXT_FIT_MIN_TOL 0.1 + $psphotConfig = $psphotConfig -Di PSPHOT:LMM_FIT_GAIN_FACTOR_MODE 2 + $psphotConfig = $psphotConfig -Db PSPHOT:SAVE.RESID T + $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_PS1_V1 + # $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_GAUSS + $psphotConfig = $psphotConfig -D PSPHOT:EXTENDED_SOURCE_MODELS_SELECTION $fitModel + # $psphotConfig = $psphotConfig -D PSPHOT:OUTPUT.FORMAT PS1_V5 + $psphotConfig = $psphotConfig -D PSPHOT:OUTPUT.FORMAT PS1_SV3 + $psphotConfig = $psphotConfig -Db PSPHOT:LENSING_PARAMETERS T + + # ppImage / psphot on the output + echo ppImage $ppImageConfig $psphotConfig -file $basename.fits $outname + exec ppImage $ppImageConfig $psphotConfig -file $basename.fits $outname +end + +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro ck.poserrors + reindex X_raw_m = X_raw using index2 + reindex X_out_m = X_PSF_out using index1 + reindex dX_out_m = X_PSF_SIG_out using index1 + set Xoff = X_raw_m - X_out_m + reindex M_raw_m = PSF_INST_MAG_raw using index2 + + # lim M_raw_m Xoff; clear; box; plot M_raw_m Xoff + set dC = Xoff^2 / dX_out_m^2 + lim M_raw_m dX_out_m; clear; box; plot M_raw_m dX_out_m +end + +macro ckgalaxy + if ($0 != 3) + echo "USAGE: ckgalaxy (dat) (cmf)" + break + end + + data $1 + read Xin_all 1 Yin_all 2 Type 4 Min_all 5 RmajIn_all 7 RminIn_all 8 ThetaIn_all 9 + + subset Xin = Xin_all if (Type == 1) + subset Yin = Yin_all if (Type == 1) + subset Min = Min_all if (Type == 1) + subset Tin = ThetaIn_all if (Type == 1) + + subset RmajIn = RmajIn_all if (Type == 1) + subset RminIn = RminIn_all if (Type == 1) + + data $2 + read -fits Chip.xfit X_EXT Y_EXT EXT_INST_MAG EXT_WIDTH_MAJ EXT_WIDTH_MIN EXT_THETA + set EXT_THETA_ALT = EXT_THETA * (EXT_THETA >= 0.0) + (EXT_THETA + 3.14159265) * (EXT_THETA < 0.0) + set EXT_THETA = EXT_THETA_ALT + + match2d X_EXT Y_EXT Xin Yin 1.0 -index1 index1 -index2 index2 + + reindex Xot_m = X_EXT using index1 + reindex Yot_m = Y_EXT using index1 + + reindex Xin_m = Xin using index2 + reindex Yin_m = Yin using index2 + + set dX = Xin_m - Xot_m + set dY = Yin_m - Yot_m + # lim -n 0 dX dY; clear; box; plot dX dY + + reindex Mot_m = EXT_INST_MAG using index1 + reindex Tot_m = EXT_THETA using index1 + + reindex Min_m = Min using index2 + reindex Tin_m = Tin using index2 + + reindex Rot_m = EXT_WIDTH_MAJ using index1 + reindex rot_m = EXT_WIDTH_MIN using index1 + + reindex Rin_m = RmajIn using index2 + reindex rin_m = RminIn using index2 + + set n = ramp(Xin_m) + + set dM = Min_m - Mot_m + set dT = (Tin_m - Tot_m) * 180 / 3.14159265 + set Tin_deg = Tin_m * 180 / 3.14159265 + + lim -n 0 Tin_deg -5 5; clear; box; plot Tin_deg dT; label -y "delta theta (deg)" + lim -n 1 n dM; clear; box; plot n dM; label -y "delta mag" + lim -n 2 n -5 5; clear; box; plot n dT; label -y "delta theta (deg)" + lim -n 3 -5 5 dM; clear; box; plot dT dM; label -x "delta theta (deg)" -y "delta mag" + + set dR = Rin_m - Rot_m + set dr = rin_m - rot_m + lim -n 4 n dR; clear; box; plot n dR; label -y "delta Rmaj" + lim -n 5 n dr; clear; box; plot n dr; label -y "delta Rmin" +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y '$word:0' -x '$word:3' +end + +# this list is used to compare a pair of vectors with an error it is +# assumed that the vector sets have been loaded and matched with +# match2d to generate index vectors 'index1' and index2'. the macro +# show.dpair generates a plot of the range vector vs (v1 - v2) / dv1. +# The indices for v1, dv1, and v2 are index1,1, and 2 respectively. The +# index for the range vector is defined by the integer following that +# vector. The y-limits of the plot are given by the last two numbers +# (use 'def') for the full default range of the delta vector +list dpairs + # v1 dv v2 range + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro plot.angles + if ($0 != 2) + echo "USAGE: plot.angles (file.dat)" + break + end + + data $1 + read x 1 y 2 type 4 trad 9 + set t = trad * 180 / 3.14159265 + subset xs = x if (type == 1) + subset ys = y if (type == 1) + subset ts = t if (type == 1) + + set cs = dcos(ts) + set sn = dsin(ts) + + delete xp yp + erase red + for i 0 xs[] + point red LINE xs[$i] ys[$i] {10*cs[$i]} {10*sn[$i]} + end + plot xp yp -pt 100 +end + +macro check.flux + if ($0 != 2) + echo "USAGE: plot.angles (file.dat)" + break + end + + data $1 + read f 3 type 4 + subset F = f if (type == 1) + set M = -2.5*log(F) + vstat M +end + +if ($SCRIPT) + echo "no default action defined" + exit 0 +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_galaxygrid.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_galaxygrid.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_galaxygrid.pro (revision 41083) @@ -0,0 +1,1445 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT +$BaseOptions = $BaseOptions -filter r +$BaseOptions = $BaseOptions -skymags 20.86 +$BaseOptions = $BaseOptions -ra 270.70 +$BaseOptions = $BaseOptions -dec -23.70 +$BaseOptions = $BaseOptions -pa 0.0 +# $BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions +$RefOptions = $RefOptions -exptime 100.0 +$RefOptions = $RefOptions -seeing 1.0 +$RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS +$RefOptions = $RefOptions -Df STARS.DENSITY 10.0 +$RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 +$RefOptions = $RefOptions -nx 3000 -ny 3000 + +if (not($?PSFMODEL)) + $PSFMODEL = PS1_V1 +end + +macro reset.options +# options for the simulated images (using the refimage for the stars) +$FakeOptions = $BaseOptions +$FakeOptions = $FakeOptions -exptime 30.0 +# $FakeOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS +$FakeOptions = $FakeOptions -D PSF.MODEL PS_MODEL_$PSFMODEL +$FakeOptions = $FakeOptions -nx 3000 -ny 3000 + +# sample alternate options: +# $FakeOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $FakeOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $FakeOptions = $FakeOptions -Df PSF.THETA +30.0 + +# create an image with fake sources (these are then inserted into the catdir) +$RefConfig = -camera SIMTEST +$RefConfig = $RefConfig -recipe PPSIM STACKTEST.MAKE + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$FakeConfig = -camera SIMTEST +$FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN +$FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref +$FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref +$FakeConfig = $FakeConfig -Db MATCH.DENSITY F +$FakeConfig = $FakeConfig -Db PSF.CONVOLVE T +$FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies +$FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 +$FakeConfig = $FakeConfig -Db GALAXY.GRID T ; # generate a grid of galaxies (constant mag) +#$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_GAUSS +#$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_EXP +#$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_SERSIC +#$FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_DEV +$FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN 10.0 +$FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX 10.0 +$FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN 0.25 +$FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX 0.25 +$FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 +$FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 +$FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.66 +$FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.66 +$FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 300 +$FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 300 +end + +if (not($?FakeConfig)) reset.options + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +if (not($?CONVOLVE_NSIGMA)) set CONVOLVE_NSIGMA = 5.0 + +# generate fake images and run psphot on them +macro mkexp.devexp.single + if ($0 != 5) + echo "USAGE: mkexp.devexp.single (basename) (type) (Rmajor) (fwhm)" + break + end + + $basename = $1 + $type = $2 + $Rmajor = $3 + $fwhm = $4 + + $Aratio = 1.0 + + $FakeConfig = -camera SIMTEST + $FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN + $FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref + $FakeConfig = $FakeConfig -Db MATCH.DENSITY F + $FakeConfig = $FakeConfig -Db PSF.CONVOLVE T + $FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies + $FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 + $FakeConfig = $FakeConfig -Df GALAXY.GRID.MAG 14.5 + $FakeConfig = $FakeConfig -Db GALAXY.GRID T ; # generate a grid of galaxies (constant mag) + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 300 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 300 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.0 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.0 + $FakeConfig = $FakeConfig -Df CONVOLVE.NSIGMA $CONVOLVE_NSIGMA + $BaseConfig = $FakeConfig + + $FakeConfig = $BaseConfig + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN $Aratio + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX $Aratio + + mkexp $basename $fwhm $type +end + +# generate fake images and run psphot on them +macro normtest.mkexp.devexp + $FakeConfig = -camera SIMTEST + $FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN + $FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref + $FakeConfig = $FakeConfig -Db MATCH.DENSITY F + $FakeConfig = $FakeConfig -Db PSF.CONVOLVE T + $FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies + $FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 + $FakeConfig = $FakeConfig -Df GALAXY.GRID.MAG 14.5 + $FakeConfig = $FakeConfig -Db GALAXY.GRID T ; # generate a grid of galaxies (constant mag) + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 300 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 300 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.0 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.0 + $BaseConfig = $FakeConfig + + # $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_GAUSS + # $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_EXP + # $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_SERSIC + # $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_DEV + # $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN 10.0 + # $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX 10.0 + # $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN 0.25 + # $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX 0.25 + # $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.66 + # $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.66 + + mkdir normtest + + $Nseq = 0 + foreach type EXP DEV + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 1.0 + $FakeConfig = $BaseConfig + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN $Aratio + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX $Aratio + + sprint name "normtest/test.%02d" $Nseq + mkexp $name $fwhm $type + $Nseq ++ + end + end + end + end +end + +# generate fake images and run psphot on them +macro grid.mkexp.devexp + if ($0 != 2) + echo "USAGE: grid.mkexp.devexp (dir)" + break + end + + mkdir $1 + + $FakeConfig = -camera SIMTEST + $FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN + $FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref + $FakeConfig = $FakeConfig -Db MATCH.DENSITY F + $FakeConfig = $FakeConfig -Db PSF.CONVOLVE T + $FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies + $FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 + $FakeConfig = $FakeConfig -Df GALAXY.GRID.MAG 14.5 + $FakeConfig = $FakeConfig -Db GALAXY.GRID T ; # generate a grid of galaxies (constant mag) + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 300 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 300 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN 1.0 + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX 1.0 + $BaseConfig = $FakeConfig + + $Nseq = 0 + foreach type EXP DEV + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + $FakeConfig = $BaseConfig + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN $Aratio + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX $Aratio + + sprint name "$1/sample.%02d" $Nseq + mkexp $name $fwhm $type + $Nseq ++ + end + end + end + end +end + +# generate fake images and run psphot on them +macro grid.fitexp.devexp + if ($0 != 3) + echo "USAGE: grid.fitexp.devexp (srcdir) (outdir)" + break + end + + #$Nseq = 0 + #foreach type EXP DEV + + mkdir $2 + + $Nseq = 0 + foreach type EXP DEV + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + sprint srcname "$1/sample.%02d" $Nseq + sprint outname "$2/sample.%02d.fit" $Nseq + fitexp $srcname $outname $type\_CONV + $Nseq ++ + end + end + end + end +end + +macro grid.load.devexp + if ($0 != 3) + echo "USAGE: grid.load.devexp (srcdir) (fitdir)" + break + end + + delete -q Xin_s Yin_s Min_s Tin_s Rin_s rin_s + delete -q Xot_s Yot_s Mot_s Tot_s Rot_s rot_s + delete -q min_S Min_S + + $Nseq = 0 + foreach type EXP DEV + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + sprint name "sample.%02d" $Nseq + cmf.load.concat $1/$name.dat $2/$name.fit.cmf $type + $Nseq ++ + end + end + end + end +end + +macro grid.plots.devexp + if ($0 != 2) + echo "USAGE: grid.plot.devexp (version)" + break + end + # things to examine: theta, Rmajor, AR + + # go.grid.check.devexp + + # check on theta + set dT = Tot_s - Tin_s + set ARin = rin_s / Rin_s + # lim ARin dT; clear; box; plot ARin dT + + subset dTx = dT if (ARin < 0.95) + histogram dTx NdT -8 8 0.1 -range dTi + lim -n 0$1 dTi NdT; clear; box; plot -x 1 dTi NdT + label -x "angle offset (degrees, only non-circular)" -y "number count" + resize 700 320 + + # check on Rmajor + set dR = Rin_s - Rot_s + # lim Rin_s dR; clear; box; plot Rin_s dR + # lim Rot_s dR; clear; box; plot Rot_s dR + # lim Rin_s dR; clear; box; plot Rin_s dR + create n 0 dR[] + set dRf = dR / Rin_s + lim -n 1$1 n -0.5 0.5; clear; box; plot n dRf + label -x sequence -y "1 - R_out| / R_in|" + resize 700 320 + + # check on A.Ratio + set ARot = rot_s / Rot_s + # lim ARin ARot; clear; box; plot ARin ARot + set fAR = ARot / ARin + lim -n 2$1 n 0.5 1.5; clear; box; plot n fAR + label -x sequence -y "AR_out| / AR_in|" + resize 700 320 + + # check on magnitude + set dM = Mot_s - Min_s + lim -n 3$1 n -0.5 0.5; clear; box; plot n dM + label -x sequence -y "M_out| - M_in|" + resize 700 320 + + # check on magnitude + set dI = Iot_s - Iin_s + lim -n 4$1 n -1.0 1.0; clear; box; plot n dI + label -x sequence -y "I_out| - I_in|" + resize 700 320 +end + +macro grid.plot.stars + if ($0 != 2) + echo "USAGE: grid.plot.stars (version)" + break + end + # things to examine: theta, Rmajor, AR + + # go.grid.check.devexp + + # check on position + set dX = Xot_s - int(Xin_s) - 0.5 + set dY = Yot_s - int(Yin_s) - 0.5 + set ARin = rin_s / Rin_s + set dR = Rin_s - Rot_s + + create n 0 dR[] + set dRf = dR / Rin_s + lim -n 1$1 n -0.5 0.5; clear; box; plot n dRf + label -x sequence -y "1 - R_out| / R_in|" + resize 700 320 + + # check on A.Ratio + set ARot = rot_s / Rot_s + # lim ARin ARot; clear; box; plot ARin ARot + set fAR = ARot / ARin + lim -n 2$1 n 0.5 1.5; clear; box; plot n fAR + label -x sequence -y "AR_out| / AR_in|" + resize 700 320 + + # check on magnitude + set dM = Mot_s - Min_s + lim -n 3$1 n -0.5 0.5; clear; box; plot n dM + label -x sequence -y "M_out| - M_in|" + resize 700 320 + + lim -n 4$1 n -1.5 1.5; clear; box; plot n dX + label -x sequence -y "X_out| - X_in|" + resize 700 320 + + lim -n 5$1 n -1.5 1.5; clear; box; plot n dY + label -x sequence -y "Y_out| - Y_in|" + resize 700 320 +end + +macro grid.mkexp.sersic + if ($0 != 2) + echo "USAGE: grid.mkexp.devexp (dir)" + break + end + + mkdir $1 + + $FakeConfig = -camera SIMTEST + $FakeConfig = $FakeConfig -recipe PPSIM STACKTEST.RUN + $FakeConfig = $FakeConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $FakeConfig = $FakeConfig -Db STARS.FAKE F ; # only use stars from catdir.ref + $FakeConfig = $FakeConfig -Db MATCH.DENSITY F + $FakeConfig = $FakeConfig -Db PSF.CONVOLVE T + $FakeConfig = $FakeConfig -Db GALAXY.FAKE T ; # generate a "realistic" distribution of galaxies + $FakeConfig = $FakeConfig -Df GALAXY.MAG 17.0 + $FakeConfig = $FakeConfig -Df GALAXY.GRID.MAG 14.5 + $FakeConfig = $FakeConfig -Db GALAXY.GRID T ; # generate a grid of galaxies (constant mag) + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MIN 0 + $FakeConfig = $FakeConfig -Df GALAXY.THETA.MAX 180 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DX 300 + $FakeConfig = $FakeConfig -Di GALAXY.GRID.DY 300 + $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_SERSIC + $BaseConfig = $FakeConfig + + $Nseq = 0 + foreach index 1 2 3 4 + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + $FakeConfig = $BaseConfig + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MIN $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.RMAJOR.MAX $Rmajor + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MIN $Aratio + $FakeConfig = $FakeConfig -Df GALAXY.ARATIO.MAX $Aratio + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MIN $index + $FakeConfig = $FakeConfig -Df GALAXY.INDEX.MAX $index + + sprint name "$1/sersic.%02d" $Nseq + mkexp $name $fwhm SERSIC + echo "$Nseq : $index $Rmajor $Aratio $fwhm" + $Nseq ++ + end + end + end + end +end + +macro grid.fitexp.sersic.devexp + if ($0 != 3) + echo "USAGE: grid.fitexp.sersic.devexp (srcdir) (outdir)" + break + end + + mkdir $2 + + $Nseq = 0 + foreach index 1 2 3 4 + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + sprint name "sersic.%02d" $Nseq + fitexp $1/$name $2/$name.fit EXP_CONV,DEV_CONV + $Nseq ++ + end + end + end + end +end + +macro grid.fitexp.sersic + if ($0 != 3) + echo "USAGE: grid.fitexp.sersic (srcdir) (outdir)" + break + end + + mkdir $2 + + $Nseq = 0 + foreach index 1 2 3 4 + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + sprint name "sersic.%02d" $Nseq + fitexp $1/$name $2/$name.fit SER\_CONV + $Nseq ++ + end + end + end + end +end + +macro grid.load.sersic + if ($0 != 3) + echo "USAGE: grid.load.devexp (srcdir) (fitdir)" + break + end + + delete -q Xin_s Yin_s Min_s Tin_s Rin_s rin_s MTin_s Iin_s + delete -q Xot_s Yot_s Mot_s Tot_s Rot_s rot_s MTot_s Iot_s + + $Nseq = 0 + foreach index 1 2 3 4 + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + sprint name "sersic.%02d" $Nseq + cmf.load.concat $1/$name.dat $2/$name.fit.cmf SERSIC + $Nseq ++ + end + end + end + end +end + +# I want to make plots of Iin_s vs Mkron, Mxx, and similar things +# this means I need to be able to join Chip.xfit things against Chip.psf +# and to join Chip.xfit(DEV) to Chip.xfit(EXP) + +# I think I need a generic 'JOIN' function + +macro grid.plots.sersic + if ($0 != 2) + echo "USAGE: grid.plot.devexp (version)" + break + end + # things to examine: theta, Rmajor, AR + + # go.grid.check.devexp + + # check on theta + set dT = Tot_s - Tin_s + set ARin = rin_s / Rin_s + # lim ARin dT; clear; box; plot ARin dT + + subset dTx = dT if (ARin < 0.95) + histogram dTx NdT -8 8 0.1 -range dTi + lim -n 0$1 dTi NdT; clear; box; plot -x 1 dTi NdT + label -x "angle offset (degrees, only non-circular)" -y "number count" + resize 700 320 + + # check on Rmajor + set dR = Rin_s - Rot_s + # lim Rin_s dR; clear; box; plot Rin_s dR + # lim Rot_s dR; clear; box; plot Rot_s dR + # lim Rin_s dR; clear; box; plot Rin_s dR + create n 0 dR[] + set dRf = dR / Rin_s + lim -n 1$1 n -0.5 0.5; clear; box; plot n dRf + label -x sequence -y "1 - R_out| / R_in|" + resize 700 320 + + # check on A.Ratio + set ARot = rot_s / Rot_s + # lim ARin ARot; clear; box; plot ARin ARot + set fAR = ARot / ARin + lim -n 2$1 n 0.5 1.5; clear; box; plot n fAR + label -x sequence -y "AR_out| / AR_in|" + resize 700 320 + + # check on magnitude + set dM = Mot_s - Min_s + lim -n 3$1 n -0.5 0.5; clear; box; plot n dM + label -x sequence -y "M_out| - M_in|" + resize 700 320 + + # check on index + set dI = Iot_s - Iin_s + lim -n 4$1 n -0.5 0.5; clear; box; plot n dI + label -x sequence -y "I_out| - I_in|" + resize 700 320 +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + echo ppSim $RefOptions $RefConfig refimage + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf -quick-airmass + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 4) + echo "USAGE: mkexp basename (fwhm) (model)" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # create the raw image + $FakeConfig = $FakeConfig -D GALAXY.MODEL PS_MODEL_$3 + echo ppSim $FakeOptions $FakeConfig $basename -seeing $fwhm + exec ppSim $FakeOptions $FakeConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + #echo ppImage $ppImageConfig -file $basename.fits $basename + #exec ppImage $ppImageConfig -file $basename.fits $basename +end + +if (not($?NSIGMA_CONV)) set NSIGMA_CONV = 5.0 + +# create a realistic distribution of fake stars, GAUSS PSF +macro fitexp + if ($0 != 4) + echo "USAGE: fitexp basename outname (fitModel)" + break + end + + local basename fitModel outname + $basename = $1 + $outname = $2 + $fitModel = $3 + + $psphotConfig = + $psphotConfig = $psphotConfig -recipe PSPHOT GALAXY_TEST + $psphotConfig = $psphotConfig -threads 4 + $psphotConfig = $psphotConfig -Db PSPHOT:LMM_FIT_CHISQ_CONVERGENCE F + $psphotConfig = $psphotConfig -Df PSPHOT:EXT_FIT_MIN_TOL 0.1 + $psphotConfig = $psphotConfig -Di PSPHOT:LMM_FIT_GAIN_FACTOR_MODE 2 + $psphotConfig = $psphotConfig -Db PSPHOT:SAVE.RESID T + $psphotConfig = $psphotConfig -D PSPHOT:EXTENDED_SOURCE_MODELS_SELECTION $fitModel + $psphotConfig = $psphotConfig -D PSPHOT:PSF_MODEL PS_MODEL_$PSFMODEL + + $psphotConfig = $psphotConfig -Db PSPHOT:PSF.RESIDUALS F + $psphotConfig = $psphotConfig -Db PSPHOT:POISSON.ERRORS.PHOT.LMM F + $psphotConfig = $psphotConfig -Db PSPHOT:EXTENDED_SOURCE_FITS_POISSON F + $psphotConfig = $psphotConfig -Di PSPHOT:EXT_FIT_ITER 15 + $psphotConfig = $psphotConfig -Df PSPHOT:PSF_FIT_RADIUS_SCALE 3.75 + $psphotConfig = $psphotConfig -Df PSPHOT:EXT_FIT_NSIGMA_CONV $NSIGMA_CONV + + # ppImage / psphot on the output + break -auto off + echo ppImage $ppImageConfig $psphotConfig -file $basename.fits $outname + exec echo ppImage $ppImageConfig $psphotConfig -file $basename.fits $outname >& $outname.log + exec ppImage $ppImageConfig $psphotConfig -file $basename.fits $outname >>& $outname.log + break -auto on +end + +macro cmf.load.reset + $fields = X Y M T R r MT I + foreach field $fields + foreach set in ot + delete -q $field\$set\_s + end + end + + delete -q min_S Min_S +end + +macro cmf.load.concat + if ($0 != 4) + echo "USAGE: cmf.load.concat (dat) (cmf) (inType)" + break + end + + data $1 + read Xin_all 1 Yin_all 2 Fin_all 3 Type 4 Min_all 5 RmajIn_all 7 RminIn_all 8 ThetaIn_all 9 IndexIn_all 10 + + $TYPE_S = 83 + $TYPE_D = 68 + $TYPE_E = 69 + if ("$3" == "SERSIC") + $InType = $TYPE_S + end + if ("$3" == "DEV") + $InType = $TYPE_D + set IndexIn_all = 0.125 + zero(Xin_all) + end + if ("$3" == "EXP") + $InType = $TYPE_E + set IndexIn_all = 0.5 + zero(Xin_all) + end + + # select only the galaxies + subset Xin = Xin_all if (Type == 1) + subset Yin = Yin_all if (Type == 1) + subset Min = Min_all if (Type == 1) + subset Fin = Fin_all if (Type == 1) + set min = -2.5*log(Fin) + + subset Tin_rad = ThetaIn_all if (Type == 1) + set Tin = Tin_rad * 180 / 3.14159265 + + subset RmajIn = RmajIn_all if (Type == 1) + subset RminIn = RminIn_all if (Type == 1) + + subset IndexIn = IndexIn_all if (Type == 1) + + data $2 + + break -auto off + output -err /dev/null + read -fits Chip.xfit X_EXT Y_EXT EXT_INST_MAG EXT_WIDTH_MAJ EXT_WIDTH_MIN EXT_THETA MODEL_TYPE EXT_PAR_07 + $reread = not($STATUS) + output -err stderr + break -auto on + if ($reread) + read -fits Chip.xfit X_EXT Y_EXT EXT_INST_MAG EXT_WIDTH_MAJ EXT_WIDTH_MIN EXT_THETA MODEL_TYPE + set EXT_PAR_07 = (MODEL_TYPE:9 == 68)*0.125 + (MODEL_TYPE:9 == 69)*0.5 + end + + set EXT_THETA_ALT = EXT_THETA * (EXT_THETA >= 0.0) + (EXT_THETA + 3.14159265) * (EXT_THETA < 0.0) + set EXT_THETA = EXT_THETA_ALT * 180 / 3.14159265 + + match2d X_EXT Y_EXT Xin Yin 1.0 -index1 index1 -index2 index2 + + reindex Xot_m = X_EXT using index1 + reindex Yot_m = Y_EXT using index1 + + reindex Xin_m = Xin using index2 + reindex Yin_m = Yin using index2 + + set MTin_m = $InType + zero(Xin_m) + reindex MTot_m = MODEL_TYPE:9 using index1 + + reindex Mot_m = EXT_INST_MAG using index1 + reindex Tot_m = EXT_THETA using index1 + + reindex Min_m = Min using index2 + reindex Tin_m = Tin using index2 + + reindex Rot_m = EXT_WIDTH_MAJ using index1 + reindex rot_m = EXT_WIDTH_MIN using index1 + + reindex Rin_m = RmajIn using index2 + reindex rin_m = RminIn using index2 + + reindex Iot_m = EXT_PAR_07 using index1 + reindex Iin_m = IndexIn using index2 + $fields = X Y M T R r MT I + + foreach field $fields + foreach set in ot + concat $field\$set\_m $field\$set\_s + end + end + + concat min min_S + concat Min Min_S +end + +macro cmf.load.stars.concat + if ($0 != 3) + echo "USAGE: cmf.load.concat (dat) (cmf)" + break + end + + data $1 + read Xin_all 1 Yin_all 2 Type 4 Min_all 5 RmajIn_all 7 RminIn_all 8 ThetaIn_all 9 IndexIn_all 10 + + subset Xin = Xin_all if (Type == 0) + subset Yin = Yin_all if (Type == 0) + subset Min = Min_all if (Type == 0) + subset RmajIn = RmajIn_all if (Type == 0) + subset RminIn = RminIn_all if (Type == 0) + subset IndexIn = IndexIn_all if (Type == 0) + subset Tin_rad = ThetaIn_all if (Type == 0) + set Tin = Tin_rad * 180 / 3.14159265 + + data $2 + + break -auto off + read -fits Chip.psf X_PSF Y_PSF PSF_INST_MAG PSF_MAJOR PSF_MINOR PSF_THETA + set PSF_THETA_ALT = PSF_THETA * (PSF_THETA >= 0.0) + (PSF_THETA + 3.14159265) * (PSF_THETA < 0.0) + set PSF_THETA = PSF_THETA_ALT * 180 / 3.14159265 + + match2d X_PSF Y_PSF Xin Yin 1.0 -index1 index1 -index2 index2 + + reindex Xot_m = X_PSF using index1 + reindex Yot_m = Y_PSF using index1 + + reindex Xin_m = Xin using index2 + reindex Yin_m = Yin using index2 + + reindex Mot_m = PSF_INST_MAG using index1 + reindex Tot_m = PSF_THETA using index1 + + reindex Min_m = Min using index2 + reindex Tin_m = Tin using index2 + + reindex Rot_m = PSF_MAJOR using index1 + reindex rot_m = PSF_MINOR using index1 + + reindex Rin_m = RmajIn using index2 + reindex rin_m = RminIn using index2 + + $fields = X Y M T R r + foreach field $fields + foreach set in ot + concat $field\$set\_m $field\$set\_s + end + end +end + +macro grid.load.sersic.test + if ($0 != 3) + echo "USAGE: grid.load.devexp.test (srcdir) (fitdir)" + break + end + + $fields_bt = X Y M T R r I + $fields_ot = Pmag Kmag Amag IDx MTot + + foreach field $fields_bt + foreach set in ot + delete -q $field\$set\_dev_s + delete -q $field\$set\_exp_s + end + end + foreach field $fields_ot + delete -q $field\_dev_s + delete -q $field\_exp_s + end + + $Nseq = 0 + foreach index 1 2 3 4 + foreach Rmajor 3 10 30 + foreach Aratio 0.25 0.5 1.0 + foreach fwhm 0.8 1.0 1.5 + sprint name "sersic.%02d" $Nseq + cmf.load.sersic.test $1/$name.dat $2/$name.fit.cmf + $Nseq ++ + end + end + end + end +end + +# I have run DEV and EXP against input models of type SERSIC +macro cmf.load.sersic.test + if ($0 != 3) + echo "USAGE: cmf.load.sersic.test (dat) (cmf)" + break + end + + # input parameters + data $1 + read Xin_all 1 Yin_all 2 Fin_all 3 Type 4 Min_all 5 RmajIn_all 7 RminIn_all 8 ThetaIn_all 9 IndexIn_all 10 + + # galaxies only + subset Xin = Xin_all if (Type == 1) + subset Yin = Yin_all if (Type == 1) + subset Min = Min_all if (Type == 1) + subset Fin = Fin_all if (Type == 1) + set min = -2.5*log(Fin) + + subset Tin_rad = ThetaIn_all if (Type == 1) + set Tin = Tin_rad * 180 / 3.14159265 + + subset RmajIn = RmajIn_all if (Type == 1) + subset RminIn = RminIn_all if (Type == 1) + + subset IndexIn = IndexIn_all if (Type == 1) + + $TYPE_S = 83 + $TYPE_D = 68 + $TYPE_E = 69 + + data $2 + + # load measured values from xfit + read -fits Chip.xfit IPP_IDET X_EXT Y_EXT EXT_INST_MAG EXT_WIDTH_MAJ EXT_WIDTH_MIN EXT_THETA MODEL_TYPE PSF_INST_MAG AP_MAG KRON_MAG + set EXT_PAR_07 = (MODEL_TYPE:9 == $TYPE_D)*4 + (MODEL_TYPE:9 == $TYPE_E) + + set EXT_THETA_ALT = EXT_THETA * (EXT_THETA >= 0.0) + (EXT_THETA + 3.14159265) * (EXT_THETA < 0.0) + set EXT_THETA = EXT_THETA_ALT * 180 / 3.14159265 + set IPP_IDET_EXT = IPP_IDET + + match2d X_EXT Y_EXT Xin Yin 1.0 -index1 index1 -index2 index2 + + reindex Xot_m = X_EXT using index1 + reindex Yot_m = Y_EXT using index1 + + reindex Xin_m = Xin using index2 + reindex Yin_m = Yin using index2 + + reindex Mot_m = EXT_INST_MAG using index1 + reindex Tot_m = EXT_THETA using index1 + + reindex Min_m = Min using index2 + reindex Tin_m = Tin using index2 + + reindex Rot_m = EXT_WIDTH_MAJ using index1 + reindex rot_m = EXT_WIDTH_MIN using index1 + + reindex Rin_m = RmajIn using index2 + reindex rin_m = RminIn using index2 + + reindex Pmag_m = PSF_INST_MAG using index1 + reindex Kmag_m = KRON_MAG using index1 + reindex Amag_m = AP_MAG using index1 + + reindex IDx_m = IPP_IDET_EXT using index1 + + reindex MTot_m = MODEL_TYPE:9 using index1 + + reindex Iot_m = EXT_PAR_07 using index1 + reindex Iin_m = IndexIn using index2 + + # load moments and other kron values from Chip.psf + read -fits Chip.psf IPP_IDET X_PSF Y_PSF MOMENTS_XX MOMENTS_XY MOMENTS_YY KRON_FLUX_INNER MOMENTS_R1 MOMENTS_RH + set IPP_IDET_PSF = IPP_IDET + + join -outer IPP_IDET_PSF IDx_m + reindex Xp = X_PSF using index2 + reindex Yp = Y_PSF using index2 + + reindex Mxx_m = MOMENTS_XX using index2 + reindex Mxy_m = MOMENTS_XY using index2 + reindex Myy_m = MOMENTS_YY using index2 + reindex Mr1_m = MOMENTS_R1 using index2 + reindex Mrh_m = MOMENTS_RH using index2 + reindex Kfi_m = KRON_FLUX_INNER using index2 + set Kmi_m = -2.5*log(Kfi_m) + + $fields_bt = X Y M T R r I + $fields_ot = Pmag Kmag Amag IDx MTot Mxx Mxy Myy Mr1 Mrh Kmi + + foreach field $fields_ot + subset $field\_exp_m = $field\_m where (MTot_m == $TYPE_E) + subset $field\_dev_m = $field\_m where (MTot_m == $TYPE_D) + end + foreach field $fields_bt + foreach set in ot + subset $field\$set\_exp_m = $field\$set\_m where (MTot_m == $TYPE_E) + subset $field\$set\_dev_m = $field\$set\_m where (MTot_m == $TYPE_D) + end + end + + join IDx_exp_m IDx_dev_m + foreach field $fields_ot + reindex $field\_exp_mr = $field\_exp_m using index1 + reindex $field\_dev_mr = $field\_dev_m using index2 + end + foreach field $fields_bt + foreach set in ot + reindex $field\$set\_exp_mr = $field\$set\_exp_m using index1 + reindex $field\$set\_dev_mr = $field\$set\_dev_m using index2 + end + end + + # concat + foreach field $fields_bt + foreach set in ot + concat $field\$set\_dev_mr $field\$set\_dev_s + concat $field\$set\_exp_mr $field\$set\_exp_s + end + end + foreach field $fields_ot + concat $field\_dev_mr $field\_dev_s + concat $field\_exp_mr $field\_exp_s + end + + concat min min_S + concat Min Min_S +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro sersic.integral + if ($0 != 2) + echo "sersic.integral (index)" + break + end + + $index = $1 + create r 0.0 200.0 0.001 + set q = r^(1/$index) + set f = exp(-q) + set rf = f*r + integrate r rf r[0] r[-1] +end + +macro sersic.integral.rmax + if ($0 != 4) + echo "sersic.integral.rmax (index) (kappa) (rmax)" + echo "kappa is a guess for kappa" + break + end + + local index kappa + + $index = $1 + $kappa = $2 + + create r 0.0 $3 0.003 + set q = r^(1/$index) + set f = exp(-$kappa*q) + set rf = f*r + integrate r rf r[0] r[-1] +end + +# S(r) = exp(-kappa*(r/Reff)^(1/index)) +# integrate S(r) r dr [ignores 2pi and change-of-variable factors (Rmaj*Rmin)] +macro sersic.integral.reff.rmax + if ($0 != 5) + echo "sersic.integral.rmax (index) (kappa) (reff) (rmax)" + echo "kappa is a guess for kappa" + break + end + + local index kappa Reff + + $index = $1 + $kappa = $2 + $Reff = $3 + + create r 0.0 $4 0.01 + set q = (r/$Reff)^(1/$index) + set f = exp(-$kappa*q) + set rf = f*r + integrate r rf r[0] r[-1] +end + +# integrate to r = reff (rho = 1), applying kappa +macro sersic.integral.reff + if ($0 != 3) + echo "sersic.integral.reff (index) (kappa)" + break + end + + local index kappa + + $index = $1 + $kappa = $2 + create r 0.0 100.0 0.001 + set q = r^(1/$index) + set f = exp(-$kappa*q) + set rf = f*r + integrate r rf r[0] r[-1] + $S0 = $sum + integrate r rf r[0] 1.0 + $S1 = $sum + echo $S0 $S1 {$S1 / $S0} +end + +macro sersic.integral.find.reff + if ($0 != 3) + echo "sersic.integral.find.reff (index) (kappa)" + echo "kappa is a guess for kappa" + break + end + + local kappa index Rmax i + $index = $1 + $kappa = $1 + + $Rmin = 0; $Vmin = 0 + $Rmax = 4000 + sersic.integral.rmax $index $kappa $Rmax + $Vmax = $sum + + $Rtry = 0.5*($Rmin + $Rmax) + + while (abs($Rmax - $Rmin) > 0.001) + sersic.integral.rmax $index $kappa $Rtry + $Vtry = $sum + + $Rold = $Rtry + if ($Vtry > 0.5*$Vmax) + $Rtry = 0.5*($Rmin + $Rtry) + $Rmax = $Rold + else + $Rtry = 0.5*($Rmax + $Rtry) + $Rmin = $Rold + end + # fprintf "%5.2f %5.2f %5.2f | %5.2f | %6.3f %6.3f | %6.3f" $Rmin $Rold $Rmax $Rtry {$Vtry / $Vmax} $Vmax $Vtry + end + # echo $Rtry + # echo {$kappa * $Rtry ^ (1.0/$index)} + $KappaReal = $kappa * $Rtry ^ (1.0/$index) +end + +macro sersic.integral.index + + local i kappa + + delete -q sersic_sum sersic_norm + # vlist idx 0.5 0.75 1.0 1.5 2.0 2.5 3.0 3.5 4.0 4.5 5.0 + # create idx 0.5 5.1 0.1 + vlist idx 1 2 4 + for i 0 idx[] + $kappa = -0.275552 + 1.972625*idx[$i] + 0.003487*idx[$i]*idx[$i] + sersic.integral.rmax idx[$i] $kappa 300 + concat $sum sersic_sum + + $bn = 1.9992*idx[$i] - 0.3271; + $Io = exp($bn); + + # the integral of a Sersic (supposedly) has an analytical form as follows: + $logGamma = lgamma(2.0*idx[$i]); + $bnFactor = $bn^(2.0*idx[$i]); + $norm = idx[$i] * $Io * exp($logGamma) / $bnFactor; + concat $norm sersic_norm + + echo idx[$i] $kappa $sum $logGamma $bn $bnFactor $norm + end + set lsersic_sum = ln(sersic_sum) + break + + $order = 2 + delete -q sersic_fit fit_idx + + # fit to ranges: + delete lsersic_fit fit_idx + vlist bound 0.0 1.0 2.0 3.0 4.0 5.5 + for i 0 {bound[] - 1} + subset idxs = idx if (idx >= bound[$i]) && (idx < bound[$i+1]) + subset sums = lsersic_sum if (idx >= bound[$i]) && (idx < bound[$i+1]) + fit idxs sums $order + applyfit idxs fits + concat fits lsersic_fit + concat idxs fit_idx + end + set sersic_fit = exp(lsersic_fit) + + lim -n 1 idx sersic_sum; clear; box; plot idx sersic_sum; plot -c blue -pt 7 idx sersic_norm; plot -x 0 -c red fit_idx sersic_fit + lim -n 2 idx lsersic_sum; clear; box; plot idx lsersic_sum; plot -x 0 -c red fit_idx lsersic_fit +end + +macro sersic.kappa.index + + local i + + delete -q sersic_kappa + # vlist idx 0.5 0.75 1.0 1.5 2.0 2.5 3.0 3.5 4.0 4.5 5.0 + create idx 0.5 5.1 0.1 + for i 0 idx[] + sersic.integral.find.reff idx[$i] 10 + concat $KappaReal sersic_kappa + end + + # $order = 2 + # delete -q sersic_kfit kfit_idx + # + # # fit to ranges: + # vlist bound 0.0 1.0 2.0 3.0 4.0 5.5 + # for i 0 {bound[] - 1} + # subset idxs = idx if (idx >= bound[$i]) && (idx < bound[$i+1]) + # subset sums = lsersic_sum if (idx >= bound[$i]) && (idx < bound[$i+1]) + # fit idxs sums $order + # applyfit idxs fits + # concat fits lsersic_fit + # concat idxs fit_idx + # end + # set sersic_fit = exp(lsersic_fit) + + lim -n 1 idx sersic_kappa; clear; box; plot idx sersic_kappa; # plot -c blue -pt 7 idx sersic_norm; plot -x 0 -c red fit_idx sersic_fit + # lim -n 2 idx lsersic_sum; clear; box; plot idx lsersic_sum; plot -x 0 -c red fit_idx lsersic_fit + + # kappa(index) : + # y = -0.275552 x^0 1.972625 x^1 0.003487 x^2 +end + +macro sersic.central.pixel + if ($0 != 3) + echo "USAGE: sersic.central.pixel (index) (Reff)" + break + end + + local index Reff + + $index = $1 + $Reff = $2 + + $kappa = -0.275552 + 1.972625*$index + 0.003487 * $index^2 + # echo "kappa: $kappa" + + sersic.norm $index + sersic.integral.reff.rmax $index $kappa $Reff {10*$Reff}; set sumFull = $sum; echo $sum {$Reff^2*$myNorm} {$sum / ($Reff^2*$myNorm)} + # sersic.integral.reff.rmax $index $kappa $Reff $Reff ; # echo $sum {$sum / $sumFull} + # sersic.integral.reff.rmax $index $kappa $Reff 0.1 ; # echo $sum {$sum / $sumFull} + # sersic.integral.reff.rmax $index $kappa $Reff 0.2 ; # echo $sum {$sum / $sumFull} + # sersic.integral.reff.rmax $index $kappa $Reff 0.5 ; # echo $sum {$sum / $sumFull} + # sersic.integral.reff.rmax $index $kappa $Reff 1.0 ; # echo $sum {$sum / $sumFull} + # sersic.integral.reff.rmax $index $kappa $Reff 2.0 ; # echo $sum {$sum / $sumFull} + sersic.integral.reff.rmax $index $kappa $Reff 0.564 ; echo $index $Reff {$sum / $sumFull} + +end + +macro sersic.norm + if ($0 != 2) + echo "USAGE: sersic.norm (index)" + break + end + + local index + $index = $1 + + if (($index >= 0.0) && ($index < 1.0)) + $norm = 0.201545 - 0.950965 * $index - 0.315248 * $index^2 + echo $norm {exp($norm)} + $myNorm = exp($norm) + return + end + + if (($index >= 1.0) && ($index < 2.0)) + $norm = 0.402084 - 1.357775 * $index - 0.105102 * $index^2 + echo $norm {exp($norm)} + $myNorm = exp($norm) + return + end + + if (($index >= 2.0) && ($index < 3.0)) + $norm = 0.619093 - 1.591674 * $index - 0.041576 * $index^2 + echo $norm {exp($norm)} + $myNorm = exp($norm) + return + end + + if (($index >= 3.0) && ($index < 4.0)) + $norm = 0.770263 - 1.696421 * $index - 0.023363 * $index^2 + echo $norm {exp($norm)} + $myNorm = exp($norm) + return + end + + if (($index >= 4.0) && ($index < 5.5)) + $norm = 0.885891 - 1.755684 * $index - 0.015753 * $index^2 + echo $norm {exp($norm)} + $myNorm = exp($norm) + return + end +end + +macro load.model.im + if ($0 != 2) + echo "USAGE: load.model.im (N)" + break + end + + rd obj$1 obj.$1.fits + rd cnv$1 cnv.$1.fits + rd var$1 var.$1.fits + rd msk$1 msk.$1.fits + for i 1 7 + rd dpar$i.$1 dpar.$i.$1.fits + end + set dC$1 = (msk$1 == 0) * (obj$1 - cnv$1)^2 / var$1 + tv dC$1 -0.01 3.0 +end + +macro load.normdata + if ($0 != 3) + echo "USAGE: load.normdata (file) [clear/noclear]" + break + end + + data $1 + read f 3 t 4 m 5 + set M = -2.5*log(f) + subset Mg = M if (t == 1) + subset mg = m if (t == 1) + set dm = mg - Mg + set n = ramp(dm) + if ("$2" == "clear") + lim n dm; clear; box; + end + plot n dm +end + +if ($SCRIPT) + echo "no default action defined" + exit 0 +end + +# note that t1 = original confi +# t2 = no residuals +# t3 = const weight PSF fit +# t4 = const weight galaxy fits + +macro run.radius.loop + local radius Nrun + + if (1) + $Nrun = 0 + foreach radius 1.0 1.5 2.0 2.5 3.0 4.0 6.0 8.0 + mkexp.devexp.single tests.20131120/testrad.ps1v1.exp.$Nrun EXP $radius 1.0 + fitexp tests.20131120/testrad.ps1v1.exp.$Nrun tests.20131120/testrad.ps1v1.exp.$Nrun.t4b EXP_CONV + $Nrun ++ + end + end + + $Nrun = 0 + foreach radius 1.0 1.5 2.0 2.5 3.0 4.0 6.0 8.0 + cmf.load.concat tests.20131120/testrad.ps1v1.exp.$Nrun.dat tests.20131120/testrad.ps1v1.exp.$Nrun.t4b.cmf EXP + echo -no-return $radius "" + check.fit tests.20131120/testrad.ps1v1.exp.$Nrun.dat tests.20131120/testrad.ps1v1.exp.$Nrun.t4b.cmf EXP + $Nrun ++ + end + + grid.plots.devexp a +end + +macro check.radius.loop + if ($0 != 2) + echo "USAGE: check.radius.loop (ext)" + break + end + + # cmf.load.reset + + local radius Nrun + + $Nrun = 0 + foreach radius 1.0 1.5 2.0 2.5 3.0 4.0 6.0 8.0 + echo -no-return $radius "" + check.fit tests.20131120/testrad.ps1v1.exp.$Nrun.dat tests.20131120/testrad.ps1v1.exp.$Nrun.$1.cmf EXP + $Nrun ++ + end + + grid.plots.devexp a +end + +macro check.fit + if ($0 != 4) + echo "USAGE: check.fit (dat) (cmf) (type)" + break + end + + cmf.load.concat $1 $2 $3 + set dM_m = Mot_m - Min_m + vstat -q dM_m + $mag_off = $MEDIAN + set dR_m = Rot_m - Rin_m + vstat -q dR_m + $rad_off = $MEDIAN + echo $mag_off $rad_off +end + +macro make.circles + + delete Xo Yo Ro + + for ix 0 16 + for iy $ix 16 + $r2 = $ix^2 + $iy^2 + if ($r2 > 15^2) continue + concat $ix Xo + concat $iy Yo + concat $r2 Ro + end + end + + sort Ro Xo Yo + $Rold = -1 + $N = -1 + for i 0 Ro[] + if (Ro[$i] != $Rold) + $N ++ + $Rold = Ro[$i] + end + if ((Xo[$i] == 0) && (Yo[$i] == 0)) + fprintf "// center is 0,0" + continue + end + if (Xo[$i] == 0) + fprintf "ADD_AXIS (%3d, %2d) // r^2 = %3d" $N Yo[$i] Ro[$i] + continue + end + if (Xo[$i] == Yo[$i]) + fprintf "ADD_DIAG (%3d, %2d) // r^2 = %3d" $N Xo[$i] Ro[$i] + continue + end + + fprintf "ADD_RAND (%3d, %2d, %2d) // r^2 = %3d" $N Xo[$i] Yo[$i] Ro[$i] + end +end + +# EXP : tests.20131120/test.nsig +# DEV : tests.20131120/test.dev + +# run.convolve.loop tests.20131120/test.nsig tests.20131120/test.nsig EXP EXP_CONV +# run.convolve.loop tests.20131120/test.dev tests.20131120/test.dev DEV DEV_CONV +# run.convolve.loop tests.20131120/test.nsig tests.20131120/test.serexp EXP SER_CONV +# run.convolve.loop tests.20131120/test.dev tests.20131120/test.serdev DEV SER_CONV + +macro run.convolve.loop + if ($0 != 5) + echo "run.convolve.loop (inName) (fitName) (inType) (fitType)" + break + end + + local radius + $radius = 3.0 + cmf.load.reset + + # foreach Cin 3 5 7 9 11 + foreach Cin 11 + $CONVOLVE_NSIGMA = $Cin + sprintf nameIn %s.%02d $1 $Cin + mkexp.devexp.single $nameIn $3 $radius 1.0 + foreach Cot 3 5 7 9 11 + $NSIGMA_CONV = $Cot + sprintf nameOt %s.%02d.%02d $2 $Cin $Cot + fitexp $nameIn $nameOt $4 + end + end + + # foreach Cin 3 5 7 9 11 + foreach Cin 11 + sprintf nameIn %s.%02d $1 $Cin + foreach Cot 3 5 7 9 11 + sprintf nameOt %s.%02d.%02d $2 $Cin $Cot + check.fit $nameIn.dat $nameOt.cmf + end + end + grid.plots.devexp a +end + +macro check.convolve.loop + if ($0 != 4) + echo "run.convolve.loop (inName) (fitName) (inType)" + break + end + + local radius + $radius = 3.0 + cmf.load.reset + + # foreach Cin 3 5 7 9 11 + foreach Cin 11 + sprintf nameIn %s.%02d $1 $Cin + foreach Cot 3 5 7 9 11 + sprintf nameOt %s.%02d.%02d $2 $Cin $Cot + echo -no-return $Cin $Cot " " + check.fit $nameIn.dat $nameOt.cmf $3 + end + end + grid.plots.devexp a +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_stack.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_stack.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_stack.pro (revision 41083) @@ -0,0 +1,768 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE T +$RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 +$RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 +$RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 +$RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 +$RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + mkref + + local i Npass imname stname + + $Npass = $1 + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf N "%02d" $i + onestack $N + end +end + +macro summary.stats + if ($0 != 2) + echo "USAGE: summary.stats (Npass)" + break + end + + local i Npass base stack imname stname + + $Npass = $1 + + exec /bin/rm -f psf.image.summary.stats + exec /bin/rm -f psf.stack.inputs.summary.stats + exec /bin/rm -f psf.stack.conv.summary.stats + exec /bin/rm -f psf.stack.unconv.summary.stats + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf base "$imname.%02d/$imname.stats" $i + sprintf stack "$stname.%02d/$stname" $i + echo $i $base $stack + exec grep PSF_INST_MAG $base >> psf.image.summary.stats + exec grep PSF_INST_MAG $stack.conv.stats >> psf.stack.inputs.summary.stats + exec grep PSF_INST_MAG $stack.pht.stats >> psf.stack.conv.summary.stats + exec grep PSF_INST_MAG $stack.unconv.pht.stats >> psf.stack.unconv.summary.stats + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +macro onestack + if ($0 != 2) + echo "USAGE: onestack (N)" + break + end + + mkstack image.$1/image stack.$1/stack + ckstack image.$1/image stack.$1/stack + rmstack image.$1/image stack.$1/stack +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i base stack + + $base = $1 + $stack = $2 + + dirname $base -var dir + mkdir $dir + + dirname $stack -var dir + mkdir $dir + + exec echo "INPUT MULTI" > $stack.mdc + exec echo "" >> $stack.mdc + + for i 0 $fwhm:n + mkexp $base.$i $fwhm:$i + + exec echo "INPUT METADATA" >> $stack.mdc + exec echo "IMAGE STR $base.$i.wrp.fits" >> $stack.mdc + exec echo "MASK STR $base.$i.wrp.mask.fits" >> $stack.mdc + exec echo "VARIANCE STR $base.$i.wrp.wt.fits" >> $stack.mdc + exec echo "PSF STR $base.$i.wrp.psf" >> $stack.mdc + exec echo "SOURCES STR $base.$i.wrp.cmf" >> $stack.mdc + exec echo "END" >> $stack.mdc + exec echo "" >> $stack.mdc + end + + exec ppStack -threads 4 -input $stack.mdc $stack -Db TEMP.DELETE F + + basename $stack -var stackbase + for i 0 $fwhm:n + exec /bin/cp -f /tmp/$stackbase.$i.conv.im.fits $stack.$i.conv.im.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.var.fits $stack.$i.conv.var.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.mk.fits $stack.$i.conv.mk.fits + end + + # XXX note that the output convolved variance is missing the covariance + for i 0 $fwhm:n + exec psphot -file $stack.$i.conv.im.fits -mask $stack.$i.conv.mk.fits $stack.$i.conv + end + + # basic photometry for the stack and unconvolved stack + exec psphot -file $stack.fits -mask $stack.mask.fits -variance $stack.weight.fits $stack.pht + exec psphot -file $stack.unconv.fits -mask $stack.unconv.mask.fits -variance $stack.unconv.wt.fits $stack.unconv.pht +end + +# run this on the files created with mkstack +macro ckstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i ZPT base stack + + $base = $1 + $stack = $2 + + # generate plots and stats for the per-image chip analysis + # and the psf-matched convolved images + $ZPT = -2.5*log($fwhm:n) + for i 0 $fwhm:n + ckchip $base.$i.in.cmf $base.$i.cmf $base.$i 0.0 + ckwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.$i.conv $ZPT + + stchip $base.$i.in.cmf $base.$i.cmf $base.stats 0.0 + stwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.conv.stats $ZPT + + completeness $base.$i.in.cmf $base.$i.cmf $base.$i.complete.png + end + + ckwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht $ZPT + ckwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht $ZPT + + stwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht.stats $ZPT + stwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht.stats $ZPT +end + +macro rmstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local base stack + $base = $1 + $stack = $2 + + for i 0 $fwhm:n + exec rm -f $base.$i.fits + exec rm -f $base.$i.ch.fits + exec rm -f $base.$i.ch.mk.fits + exec rm -f $base.$i.ch.wt.fits + + exec rm -f $base.$i.wrp.fits + exec rm -f $base.$i.wrp.mask.fits + exec rm -f $base.$i.wrp.wt.fits + + exec rm -f $stack.$i.conv.im.fits + exec rm -f $stack.$i.conv.mk.fits + exec rm -f $stack.$i.conv.wt.fits + end + + exec rm -f $stack.fits + exec rm -f $stack.mask.fits + exec rm -f $stack.weight.fits + + exec rm -f $stack.exp.fits + exec rm -f $stack.num.fits + exec rm -f $stack.expwt.fits + + exec rm -f $stack.unconv.fits + exec rm -f $stack.unconv.mask.fits + exec rm -f $stack.unconv.wt.fits + + exec rm -f $stack.unconv.exp.fits + exec rm -f $stack.unconv.num.fits + exec rm -f $stack.unconv.expwt.fits +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 3) + echo "USAGE: mkexp basename fwhm" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + + echo psphot -file $basename.wrp.fits -mask $basename.wrp.mask.fits -variance $basename.wrp.wt.fits $basename.wrp.pht + exec psphot -file $basename.wrp.fits -mask $basename.wrp.mask.fits -variance $basename.wrp.wt.fits $basename.wrp.pht +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stchip + if ($0 != 5) + echo "USAGE: stchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +macro stwarp + if ($0 != 5) + echo "USAGE: stwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +macro completeness + if ($0 != 4) + echo "USAGE: completeness (raw) (out) (output)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out 1.5 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + png -name $3 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y '$word:0' -x '$word:3' +end + +# this list is used to compare a pair of vectors with an error it is +# assumed that the vector sets have been loaded and matched with +# match2d to generate index vectors 'index1' and index2'. the macro +# show.dpair generates a plot of the range vector vs (v1 - v2) / dv1. +# The indices for v1, dv1, and v2 are index1,1, and 2 respectively. The +# index for the range vector is defined by the integer following that +# vector. The y-limits of the plot are given by the last two numbers +# (use 'def') for the full default range of the delta vector +list dpairs + # v1 dv v2 range + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_stackphot.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_stackphot.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_stackphot.pro (revision 41083) @@ -0,0 +1,793 @@ +#!/usr/bin/env mana +# -*-sh-*- + +$KAPA = kapa -noX + +# config for ppImage to generate chip, mask, weight +$ppImageConfig = -recipe PPIMAGE PPIMAGE_N +$ppImageConfig = $ppImageConfig -Db BACKGROUND T +$ppImageConfig = $ppImageConfig -Db CHIP.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T +$ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T +$ppImageConfig = $ppImageConfig -Db BASE.FITS F +$ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T +$ppImageConfig = $ppImageConfig -Db PHOTOM T + +# basic options for the these images (filter, location, obstype) +$BaseOptions = -type OBJECT -filter r -skymags 20.86 -ra 270.70 -dec -23.70 -pa 0.0 +$BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + +# options for the reference image +$RefOptions = $BaseOptions -exptime 100.0 -seeing 1.0 -D PSF.MODEL PS_MODEL_GAUSS -Df STARS.DENSITY 10.0 -Df STARS.SIGMA.LIM 0.5 + +# options for the repeated images +$FakeOptions = $BaseOptions -exptime 30.0 + +# basic config for ppSim with randomly distributed stars and gridded galaxies +$RealConfig = -camera SIMTEST -recipe PPSIM STACKTEST.RUN -D PSASTRO:PSASTRO.CATDIR catdir.ref +$RealConfig = $RealConfig -Db STARS.FAKE F -Db MATCH.DENSITY F -Db PSF.CONVOLVE T +$RealConfig = $RealConfig -Db GALAXY.FAKE T -Df GALAXY.MAG 17.0 +$RealConfig = $RealConfig -Db GALAXY.GRID T -D GALAXY.MODEL PS_MODEL_SERSIC +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MIN 0.5 +$RealConfig = $RealConfig -Df GALAXY.ARATIO.MAX 0.5 +$RealConfig = $RealConfig -Df GALAXY.THETA.MAX 180 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MIN 1 +$RealConfig = $RealConfig -Df GALAXY.INDEX.MAX 1 +$RealConfig = $RealConfig -Di GALAXY.GRID.DX 120 +$RealConfig = $RealConfig -Di GALAXY.GRID.DY 120 + +# sample alternate options: +# $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 +# $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 +# $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 +$ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + +list fwhm + 1.0 + 1.1 + 1.2 + 1.5 +end + +macro fulltest + if ($0 != 2) + echo "USAGE: fulltest (Npass)" + break + end + + mkref + + local i Npass imname stname + + $Npass = $1 + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf N "%02d" $i + onestack $N + end +end + +macro summary.stats + if ($0 != 2) + echo "USAGE: summary.stats (Npass)" + break + end + + local i Npass base stack imname stname + + $Npass = $1 + + exec /bin/rm -f psf.image.summary.stats + exec /bin/rm -f psf.stack.inputs.summary.stats + exec /bin/rm -f psf.stack.conv.summary.stats + exec /bin/rm -f psf.stack.unconv.summary.stats + + $imname = image.fg + $stname = stack.fg + + for i 0 $Npass + sprintf base "$imname.%02d/$imname.stats" $i + sprintf stack "$stname.%02d/$stname" $i + echo $i $base $stack + exec grep PSF_INST_MAG $base >> psf.image.summary.stats + exec grep PSF_INST_MAG $stack.conv.stats >> psf.stack.inputs.summary.stats + exec grep PSF_INST_MAG $stack.pht.stats >> psf.stack.conv.summary.stats + exec grep PSF_INST_MAG $stack.unconv.pht.stats >> psf.stack.unconv.summary.stats + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + exec rm -rf catdir.ref + exec rm -f refimage.fits + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST -recipe PPSIM STACKTEST.MAKE -D PSASTRO:PSASTRO.CATDIR catdir.ref -Db PSF.CONVOLVE T + + exec ppSim $RefOptions $RefConfig refimage + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +macro onestack + if ($0 != 2) + echo "USAGE: onestack (N)" + break + end + + mkstack image.$1/image stack.$1/stack + ckstack image.$1/image stack.$1/stack + # rmstack image.$1/image stack.$1/stack +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i base stack + + $base = $1 + $stack = $2 + + dirname $base -var dir + mkdir $dir + + dirname $stack -var dir + mkdir $dir + + exec echo "INPUT MULTI" > $stack.mdc + exec echo "" >> $stack.mdc + + for i 0 $fwhm:n + mkexp $base.$i $fwhm:$i + + exec echo "INPUT METADATA" >> $stack.mdc + exec echo "IMAGE STR $base.$i.wrp.fits" >> $stack.mdc + exec echo "MASK STR $base.$i.wrp.mask.fits" >> $stack.mdc + exec echo "VARIANCE STR $base.$i.wrp.wt.fits" >> $stack.mdc + exec echo "PSF STR $base.$i.wrp.psf" >> $stack.mdc + exec echo "SOURCES STR $base.$i.wrp.cmf" >> $stack.mdc + exec echo "END" >> $stack.mdc + exec echo "" >> $stack.mdc + end + + exec ppStack -threads 4 -input $stack.mdc $stack -Db TEMP.DELETE F -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE NONE + + basename $stack -var stackbase + for i 0 $fwhm:n + exec /bin/cp -f /tmp/$stackbase.$i.conv.im.fits $stack.$i.conv.im.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.var.fits $stack.$i.conv.var.fits + exec /bin/cp -f /tmp/$stackbase.$i.conv.mk.fits $stack.$i.conv.mk.fits + end + + # XXX note that the output convolved variance is missing the covariance + for i 0 $fwhm:n + exec psphot -file $stack.$i.conv.im.fits -mask $stack.$i.conv.mk.fits $stack.$i.conv + end + + # basic photometry for the stack and unconvolved stack + exec psphot -file $stack.fits -mask $stack.mask.fits -variance $stack.weight.fits $stack.pht + exec psphot -file $stack.unconv.fits -mask $stack.unconv.mask.fits -variance $stack.unconv.wt.fits $stack.unconv.pht +end + +# run this on the files created with mkstack +macro ckstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local i ZPT base stack + + $base = $1 + $stack = $2 + + # generate plots and stats for the per-image chip analysis + # and the psf-matched convolved images + $ZPT = -2.5*log($fwhm:n) + for i 0 $fwhm:n + ckchip $base.$i.in.cmf $base.$i.cmf $base.$i 0.0 + ckwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.$i.conv $ZPT + + stchip $base.$i.in.cmf $base.$i.cmf $base.stats 0.0 + stwarp $base.$i.in.cmf $stack.$i.conv.cmf $stack.conv.stats $ZPT + + completeness $base.$i.in.cmf $base.$i.cmf $base.$i.complete.png + end + + ckwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht $ZPT + ckwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht $ZPT + + stwarp $base.$i.in.cmf $stack.pht.cmf $stack.pht.stats $ZPT + stwarp $base.$i.in.cmf $stack.unconv.pht.cmf $stack.unconv.pht.stats $ZPT +end + +macro rmstack + if ($0 != 3) + echo "USAGE: mkstack (rawbase) (stackbase)" + break + end + + local base stack + $base = $1 + $stack = $2 + + for i 0 $fwhm:n + exec rm -f $base.$i.fits + exec rm -f $base.$i.ch.fits + exec rm -f $base.$i.ch.mk.fits + exec rm -f $base.$i.ch.wt.fits + + exec rm -f $base.$i.wrp.fits + exec rm -f $base.$i.wrp.mask.fits + exec rm -f $base.$i.wrp.wt.fits + + exec rm -f $stack.$i.conv.im.fits + exec rm -f $stack.$i.conv.mk.fits + exec rm -f $stack.$i.conv.wt.fits + end + + exec rm -f $stack.fits + exec rm -f $stack.mask.fits + exec rm -f $stack.weight.fits + + exec rm -f $stack.exp.fits + exec rm -f $stack.num.fits + exec rm -f $stack.expwt.fits + + exec rm -f $stack.unconv.fits + exec rm -f $stack.unconv.mask.fits + exec rm -f $stack.unconv.wt.fits + + exec rm -f $stack.unconv.exp.fits + exec rm -f $stack.unconv.num.fits + exec rm -f $stack.unconv.expwt.fits +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 3) + echo "USAGE: mkexp basename fwhm" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # create the raw image + echo ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec ppSim $ppSimOptions $RealConfig $basename -seeing $fwhm + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename + + # XXX pswarp is using all 7k+ sources to measure the PSF : can we reduce this? + echo pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + exec pswarp -threads 4 -astrom $basename.cmf -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $basename.wrp refimage.fits + + echo psphot -file $basename.wrp.fits -mask $basename.wrp.mask.fits -variance $basename.wrp.wt.fits $basename.wrp.pht + exec psphot -file $basename.wrp.fits -mask $basename.wrp.mask.fits -variance $basename.wrp.wt.fits $basename.wrp.pht +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stchip + if ($0 != 5) + echo "USAGE: stchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +macro stwarp + if ($0 != 5) + echo "USAGE: stwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + label -y '$word:0' -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +macro completeness + if ($0 != 4) + echo "USAGE: completeness (raw) (out) (output)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out 1.5 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + png -name $3 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y '$word:0' -x '$word:3' +end + +# this list is used to compare a pair of vectors with an error it is +# assumed that the vector sets have been loaded and matched with +# match2d to generate index vectors 'index1' and index2'. the macro +# show.dpair generates a plot of the range vector vs (v1 - v2) / dv1. +# The indices for v1, dv1, and v2 are index1,1, and 2 respectively. The +# index for the range vector is defined by the integer following that +# vector. The y-limits of the plot are given by the last two numbers +# (use 'def') for the full default range of the delta vector +list dpairs + # v1 dv v2 range + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro ckradialflux + data $1 + read -fits Chip.xrad X_APER Y_APER PSF_FWHM APER_FLUX APER_FLUX_ERR APER_FLUX_STDEV APER_FILL + read -fits Chip.psf X_PSF Y_PSF PSF_INST_MAG + set mag = PSF_INST_MAG + # XXX include nradii in header + for i 0 11 + set mA$i = -2.5*log(APER_FLUX:$i) + set dM$i = mA$i - PSF_INST_MAG + vstat dM$i + $DM$i = $MEDIAN + end + for i 1 11 + $j = $i - 1 + echo {$DM$i - $DM$j} + end + + lim mag -2 2; clear; box + for i 0 11 + plot mag dM$i -c black + end +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end Index: /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_varmodel.pro =================================================================== --- /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_varmodel.pro (revision 41083) +++ /branches/ccl_branches/ipponly-20191108/psphot/test/tap_psphot_varmodel.pro (revision 41083) @@ -0,0 +1,894 @@ +#!/usr/bin/env mana +# -*-sh-*- + +# This script includes a set of tests to demonstrate the dependence of the faint-end bias on the weighting scheme +# We have 3 weighting schemes : +# CONSTANT -- per-pixel weight is fixed (disadvantage: lower S/N, especially for higher sky?) +# IMAGE_VAR - per-pixel weight is Poisson from image (disadvantage: faint-end bias) +# MODEL_VAR - per-pixel weight is Poisson from model (disadvantage: 2 linear-fit passes) + +# Functions: +# +# init : initialize variables +# mkref : generate a fake reference catalog in a DVO database +# mkexp : generate a fake exposure from fake catalog and detrend it (saves basename.in.cmf & basename.fits) +# runphot : run psphot on an exposure (saves basename.cmf) +# ckchip.mags : generate a set of summary plots for the psphot analysis + +# I would like to produce a grid of tests: +# sky (bright, middle, dark) +# fwhm (1.0, 1.3, 1.6) +# PSF_MODEL input (GAUSS PS1_V1) +# PSF_MODEL apply (GAUSS PS1_V1) +# variance mode: CONSTANT, IMAGE_VAR, MODEL_VAR + +macro ppsimtest + data $1 + # sf1 : measured flux inserted in image + # sm2 : theoretical magnitude for star + read x 1 y 2 sf1 3 sm2 5 Io 12 + set sm1 = -2.5*log(sf1) + set dsm = sm1 - sm2 + lim -n 2 sm1 -0.2 0.2; clear; box; plot sm1 dsm +end + +macro go.one + mkdir test + + local sky fwhm psf_in psf_out + + # mkref creates refimage.* and catdir.ref + file catdir.ref found + if (not($found)) + mkref + end + + $KAPA = kapa -noX + if (not($?RefConfig)) init + + foreach sky 20.0 + foreach fwhm 1.0 + foreach psf_in PS1_V1 + + sprintf name "test/test.%02d.%02d.%s" $sky {10*$fwhm} $psf_in + mkexp $name $sky $fwhm $psf_in + data $name.dat + read fluxIn 3 MagPredIn 5 + set MagRealIn = -2.5*log(fluxIn) + set dMagIn = MagRealIn - MagPredIn + + foreach psf_out GAUSS PS1_V1 + foreach mode CONSTANT IMAGE_VAR MODEL_VAR + runphot $name $name.$psf_out.$mode "-D LINEAR_FIT_VARIANCE_MODE $mode -D PSF_MODEL PS_MODEL_$psf_out" + ckchip.mags $name.in.cmf $name.$psf_out.$mode.cmf $name.$psf_out.$mode 0.0 + end + end + end + end + end +end + +macro go + mkdir test + + local sky fwhm psf_in psf_out + + # mkref creates refimage.* and catdir.ref + file catdir.ref found + if (not($found)) + mkref + end + + $KAPA = kapa -noX + if (not($?RefConfig)) init + + foreach sky 19.0 20.0 21.0 + foreach fwhm 1.0 1.3 1.6 + foreach psf_in GAUSS PS1_V1 + + sprintf name "test/test.%02d.%02d.%s" $sky {10*$fwhm} $psf_in + # mkexp $name $sky $fwhm $psf_in + data $name.dat + read fluxIn 3 MagPredIn 5 + set MagRealIn = -2.5*log(fluxIn) + set dMagIn = MagRealIn - MagPredIn + + foreach psf_out GAUSS PS1_V1 + foreach mode CONSTANT IMAGE_VAR MODEL_VAR + # runphot $name $name.$psf_out.$mode "-D LINEAR_FIT_VARIANCE_MODE $mode -D PSF_MODEL PS_MODEL_$psf_out" + ckchip.mags $name.in.cmf $name.$psf_out.$mode.cmf $name.$psf_out.$mode 0.0 + end + end + end + end + end +end + +# create a reference database of fake stars to be used by ppSim below +macro mkref + if (not($?RefConfig)) init + + exec rm -rf catdir.ref + exec rm -f refimage.fits + + exec time ppSim $RefConfig $RefOptions refimage -nx 3000 -ny 3000 + + file synth.photcodes found + if (not($found)) + echo "making photcodes file" + mkphotcodes synth.photcodes + end + + exec addstar -D CAMERA simtest -D CATDIR catdir.ref -accept-astrom -photcode SYNTH.r -D PHOTCODE_FILE synth.photcodes refimage.cmf + exec relphot -averages -D CATDIR catdir.ref -update -region 260 280 -33 -13 +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp + if ($0 != 5) + echo "USAGE: mkexp basename sky fwhm psf_model" + break + end + + local fwhm basename psf_model + $basename = $1 + $sky = $2 + $fwhm = $3 + $psf_model = $4 + + $ExtraOptions = -D PSF.MODEL PS_MODEL_$psf_model + + # create the raw image + echo ppSim -seeing $fwhm -skymags $sky -nx 3000 -ny 3000 $RealConfig $RealOptions $ExtraOptions $basename + exec ppSim -seeing $fwhm -skymags $sky -nx 3000 -ny 3000 $RealConfig $RealOptions $ExtraOptions $basename + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp.deep + if ($0 != 3) + echo "USAGE: mkexp basename fwhm" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + $RealOptionsDeep = $BaseOptions -exptime 100 + + # create the raw image + echo ppSim -seeing $fwhm -nx 3000 -ny 3000 $RealConfig $RealOptionsDeep $ExtraOptions $basename + exec ppSim -seeing $fwhm -nx 3000 -ny 3000 $RealConfig $RealOptionsDeep $ExtraOptions $basename + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename +end + +# create a realistic distribution of fake stars, GAUSS PSF +macro mkexp.bright + if ($0 != 3) + echo "USAGE: mkexp basename fwhm" + break + end + + local fwhm basename + $basename = $1 + $fwhm = $2 + + # basic options for the these images (filter, location, obstype) + $BaseOptions = -type OBJECT -filter r -skymags 20.0 -ra 270.70 -dec -23.70 -pa 0.0 + $BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + $RealOptionsDeep = $BaseOptions -exptime 100 + + # create the raw image + echo ppSim -seeing $fwhm -nx 3000 -ny 3000 $RealConfig $RealOptionsDeep $ExtraOptions $basename + exec ppSim -seeing $fwhm -nx 3000 -ny 3000 $RealConfig $RealOptionsDeep $ExtraOptions $basename + exec /bin/mv -f $basename.cmf $basename.in.cmf + + # create the chip output + echo ppImage $ppImageConfig -file $basename.fits $basename + exec ppImage $ppImageConfig -file $basename.fits $basename +end + +macro runphot + if ($0 != 4) + echo "USAGE: runphot basename outname options" + break + end + + local basename + $basename = $1 + $outname = $2 + $options = $3 + + # create the chip output + echo psphot -threads 4 -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $outname $options + exec psphot -threads 4 -file $basename.ch.fits -mask $basename.ch.mk.fits -variance $basename.ch.wt.fits $outname $options +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip.mags + if ($0 != 5) + echo "USAGE: ckchip.mags (raw) (out) (output) (zpt_off)" + break + end + +list pairs + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -0.21 0.21 V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -0.21 0.21 V +end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + resize 1000 1000 + + clear + section a0 0.0 0.0 1.0 0.5 + label -fn courier 14; + style -pt 0 -sz 0.4 + show.pair 0 + reindex ap = AP_MAG_out using index1 + set dap = ap - v2 + plot -c red -pt 2 -sz 0.5 rv dap + + delete -q imag_V dmag_V smag_V Dmag_V dmag_Vraw + for imag -11 -6 0.2 + subset dmsub = delta if (rv > $imag) && (rv < $imag + 0.2) + vstat -q dmsub + concat $imag imag_V + concat $MEAN dmag_V + concat $MEDIAN Dmag_V + concat $SIGMA smag_V + end + + vstat -q dmag_V + $offset = $MEDIAN + set dmag_V = dmag_V - $offset + set Dmag_V = Dmag_V - $offset + + section a1 0.0 0.50 1.0 0.25 + lim rv -0.025 0.075; label -fn courier 14; box; + # plot -c black -pt 0 -sz 0.3 MagPredIn dMagIn + plot -c red -pt 7 -sz 2.0 imag_V dmag_V + plot -c darkgreen -pt 3 -sz 2.0 imag_V Dmag_V + # plot -c gold -pt 4 -sz 2.0 imag_V dmag_Vraw + plot -c blue -pt 2 -sz 2.0 imag_V smag_V + # label -y "mean (red), median (green), unclipped mean (gold), sigma (blue) of delta" + label -y "mean (red), median (green), sigma (blue) of delta" + sprintf line "OFFSET: %6.3f" $offset + textline -frac 0.1 0.8 -fn courier 24 "$line" + + subset dmsub = delta if (rv < -13) && (rv > -16) + vstat -q dmsub + sprintf line "STDEV: %6.3f" $SIGMA + textline -frac 0.1 0.7 -fn courier 24 "$line" + + + set lChiNorm = log(PSF_CHISQ_out / PSF_NDOF_out) + reindex chi = lChiNorm using index1 + section a2 0.0 0.75 1.0 0.25 + label -fn courier 14; + lim rv chi; box; + plot -c red -pt 0 -sz 0.5 rv chi + + label -y "chisq" + png -name $3.png + + # section a1 0.0 0.5 1.0 0.5 + # style -pt 0 -sz 0.4 + # show.pair 1 +end + +macro go.phot + runphot test.00 test.00.varmode.C "-Db SAVE.RESID T -D LINEAR_FIT_VARIANCE_MODE CONSTANT" + runphot test.00 test.00.varmode.I "-Db SAVE.RESID T -D LINEAR_FIT_VARIANCE_MODE IMAGE_VAR" + runphot test.00 test.00.varmode.M "-Db SAVE.RESID T -D LINEAR_FIT_VARIANCE_MODE MODEL_VAR" + runphot test.00 test.00.varmode.S "-Db SAVE.RESID T -D LINEAR_FIT_VARIANCE_MODE MODEL_SKY" +end + +macro go.vars + dev -n varC; ckchip.mags test.00.in.cmf test.00.varmode.C.cmf test.00.varmode.C 0.0 + dev -n varI; ckchip.mags test.00.in.cmf test.00.varmode.I.cmf test.00.varmode.I 0.0 + dev -n varM; ckchip.mags test.00.in.cmf test.00.varmode.M.cmf test.00.varmode.M 0.0 + dev -n varS; ckchip.mags test.00.in.cmf test.00.varmode.S.cmf test.00.varmode.S 0.0 +end + +# compare two cmf files with extname Chip.psf +# things to compare: +# * completeness (which sources in (1) are not detected in (2) +# * positions (X_PSF, Y_PSF) +# * instrumental psf mags +# * position errors (no input errors; use a model?) +# * measured FWHM? +# * kron mags (fluxes) +# * etc, etc +macro ckchip + if ($0 != 5) + echo "USAGE: ckchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i NX NY nx ny N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end + + # plot (input - output) vs mag +end + +macro stchip + if ($0 != 5) + echo "USAGE: stchip (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +# compare chip to warp +macro ckwarp + if ($0 != 5) + echo "USAGE: ckwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.0 -index1 index1 -index2 index2 + + local i nx ny NX NY N + + device -n compare + resize 1000 1000 + + # plot trends as a function of mag + $NX = 2 + $NY = 5 + $nx = 0 + $ny = 0 + $N = 0 + clear -s + + for i 0 $pairs:n + section a$nx\$ny {$nx/$NX} {$ny/$NY} {1/$NX} {1/$NY} + show.pair $i + $ny ++ + if ($ny == $NY) + $ny = 0 + $nx ++ + end + if ($nx == $NX) + png -name $3.$N.png + clear -s + $nx = 0 + $ny = 0 + $N ++ + end + end +end + +macro stwarp + if ($0 != 5) + echo "USAGE: stwarp (raw) (out) (output) (zpt_off)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 SkyChip.psf out + + # images generated with convolution will not have the right output positions + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + set M_raw = PSF_INST_MAG_raw + $4 + set K_out = -2.5*log(KRON_FLUX_out) + match2d X_PSF_out Y_PSF_out X_PSF_raw Y_PSF_raw 1.5 -index1 index1 -index2 index2 + + local i + + for i 0 $spairs:n + stats.pair $i $3 + end +end + +macro stats.pair + if ($0 != 3) + echo "USAGE: stats.pair (N) (output)" + break + end + + list word -split $spairs:$1 + if ($word:n != 8) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + reindex v2 = $word:1 using index2 + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + subset d1 = delta if ($word:4 < rv) && (rv < $word:5) && (abs(delta) < $word:7) + subset d2 = delta if ($word:5 < rv) && (rv < $word:6) && (abs(delta) < $word:7) + + vstats -q d1 -sigma-clip 3.0 + $M1 = $MEAN + $S1 = $SIGMA + vstats -q d2 -sigma-clip 3.0 + $M2 = $MEAN + $S2 = $SIGMA + + output $2 + fprintf "%-18s %7.4f %7.4f %7.4f %7.4f" $word:0 $M1 $S1 $M2 $S2 + output stdout +end + +macro show.pair + if ($0 != 2) + echo "USAGE: show.pair (N)" + break + end + + list word -split $pairs:$1 + if ($word:n != 7) + echo "invalid pair $1" + break + end + + $Nr = $word:3 + + reindex v1 = $word:0 using index1 + if ("$word:6" == "V") + reindex v2 = $word:1 using index2 + end + if ("$word:6" == "S") + set v2 = $word:1 + zero(index1) + end + reindex rv = $word:2 using index$Nr + + set delta = v1 - v2 + if (("$word:4" == "def") || ("$word:5" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:4 $word:5; box; plot rv delta + end + $line = '$word:0' - '$word:1' + label -y "$line" -x '$word:2' +end + +# This list is used to compare a pair of vectors (sans error) or a +# vector and an expected (constant) value. The last field defines a +# vector or constant for the comparison. It is assumed that the +# vector sets have been loaded and matched with match2d to generate +# index vectors 'index1' and index2'. The macro 'show.pair' generates +# a plot of the range vector vs (v1 - v2). The indices for v1, v2 are +# index1 and 2 respectively. The index for the range vector is defined +# by the integer following that vector. the y-limits of the plot are +# given by the last two numbers +list pairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_out X_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_out Y_PSF_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + X_PSF_SIG_out X_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + Y_PSF_SIG_out Y_PSF_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_MAG_out PSF_INST_MAG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + PSF_INST_MAG_SIG_out PSF_INST_MAG_SIG_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + #PSF_INST_FLUX_out PSF_INST_FLUX_raw PSF_INST_MAG_raw 2 def def V + #PSF_INST_FLUX_SIG_out PSF_INST_FLUX_SIG_raw PSF_INST_MAG_raw 2 def def V + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RAW_out M_raw PSF_INST_MAG_raw 2 -1.01 1.01 V + AP_MAG_RADIUS_out 0.0 PSF_INST_MAG_raw 2 -0.01 20.1 S + SKY_out 0.0 PSF_INST_MAG_raw 2 def def S + SKY_SIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_CHISQ_out 1.0 PSF_INST_MAG_raw 2 def def S + CR_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 def def S + EXT_NSIGMA_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + PSF_MAJOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_MINOR_out 0.0 PSF_INST_MAG_raw 2 -0.01 5.01 S + PSF_THETA_out 0.0 PSF_INST_MAG_raw 2 -1.61 1.61 S + PSF_QF_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_QF_PERFECT_out 0.0 PSF_INST_MAG_raw 2 -0.10 1.10 S + PSF_NDOF_out 0.0 PSF_INST_MAG_raw 2 def def S + PSF_NPIX_out 0.0 PSF_INST_MAG_raw 2 def def S + MOMENTS_XX_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_XY_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_YY_out 0.0 PSF_INST_MAG_raw 2 -0.01 3.01 S + MOMENTS_M3C_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M3S_out 0.0 PSF_INST_MAG_raw 2 -3.01 3.01 S + MOMENTS_M4C_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_M4S_out 0.0 PSF_INST_MAG_raw 2 -2.01 2.01 S + MOMENTS_R1_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + MOMENTS_RH_out 0.0 PSF_INST_MAG_raw 2 -5.01 5.01 S + K_out M_raw PSF_INST_MAG_raw 2 def def V + KRON_FLUX_ERR_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_INNER_out 0.0 PSF_INST_MAG_raw 2 def def S + KRON_FLUX_OUTER_out 0.0 PSF_INST_MAG_raw 2 def def S +# CAL_PSF_MAG CAL_PSF_MAG none Mraw 2 -1.01 1.01 V +# CAL_PSF_MAG_SIG CAL_PSF_MAG_SIG none Mraw 2 -1.01 1.01 V +# RA_PSF RA_PSF none Mraw 2 -1.01 1.01 V +# DEC_PSF DEC_PSF none Mraw 2 -1.01 1.01 V +# PEAK_FLUX_AS_MAG PEAK_FLUX_AS_MAG none Mraw 2 -1.01 1.01 V +# FLAGS FLAGS 0.0 Mraw 2 -1.01 1.01 S +# FLAGS2 FLAGS2 0.0 Mraw 2 -1.01 1.01 S +end + +macro load.cmf + if ($0 != 4) + echo "load.cmf (filename) (ext) (label)" + break + end + + data $1 + + # create the list of fields to load + delete -q myFields + for i 0 $fields:n + if ($?myFields) + $myFields = $myFields $fields:$i + else + $myFields = $fields:$i + end + end + + read -fits $2 $myFields + + # rename the loaded vectors appending the supplied lable + for i 0 $fields:n + set $fields:$i\_$3 = $fields:$i + delete $fields:$i + end +end + +# this list defines the fields to be loaded from file +list fields + X_PSF + Y_PSF + X_PSF_SIG + Y_PSF_SIG + PSF_INST_MAG + PSF_INST_MAG_SIG + PSF_INST_FLUX + PSF_INST_FLUX_SIG + AP_MAG + AP_MAG_RAW + AP_MAG_RADIUS + SKY + SKY_SIGMA + PSF_CHISQ + CR_NSIGMA + EXT_NSIGMA + PSF_MAJOR + PSF_MINOR + PSF_THETA + PSF_QF + PSF_QF_PERFECT + PSF_NDOF + PSF_NPIX + MOMENTS_XX + MOMENTS_XY + MOMENTS_YY + MOMENTS_M3C + MOMENTS_M3S + MOMENTS_M4C + MOMENTS_M4S + MOMENTS_R1 + MOMENTS_RH + KRON_FLUX + KRON_FLUX_ERR + KRON_FLUX_INNER + KRON_FLUX_OUTER +# CAL_PSF_MAG +# CAL_PSF_MAG_SIG +# RA_PSF +# DEC_PSF +# PEAK_FLUX_AS_MAG +# FLAGS +# FLAGS2 +end + +# use these cmf entries to measure average stats of the given pairs +list spairs + X_PSF_out X_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + Y_PSF_out Y_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + PSF_INST_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 + AP_MAG_out M_raw PSF_INST_MAG_raw 2 -15 -10 -8.5 0.1 +end + +# # XXX this is a hack: the cmf file created by ppSim is not +# # compatible with the image for warp (because one is a chip-mosaic +# # and the other is not) +# exec rm -f fix.hdr +# output fix.hdr +# echo "PSMOSAIC= 'CHIP ' / Mosaicked level" +# output stdout +# fix the header to be compatible with the chip file (to avoid running psphot) +# exec fits_insert $basename.cmf fix.hdr + +macro completeness + if ($0 != 4) + echo "USAGE: completeness (raw) (out) (output)" + break + end + + load.cmf $1 Chip.psf raw + load.cmf $2 Chip.psf out + set X_raw = int(X_PSF_raw) + 0.5 + set Y_raw = int(Y_PSF_raw) + 0.5 + match2d X_PSF_raw Y_PSF_raw X_PSF_out Y_PSF_out 1.5 -index1 index1 -index2 index2 -closest + + histogram PSF_INST_MAG_raw nMag -16.0 -3.0 0.25 -range dMag + set fMag = zero(dMag) + for i 0 {dMag[]-1} + set inrange = (PSF_INST_MAG_raw > dMag[$i]) && (PSF_INST_MAG_raw <= dMag[$i+1]) + subset all = index1 if (inrange) + subset got = index1 if (inrange) && (index1 >= 0) + if (all[] == 0) + fMag[$i] = 0 + else + fMag[$i] = got[] / all[] + end + end + + device -n complete + resize 1000 600 + + clear -s + + section default 0 0 1 1 + lim dMag fMag; clear; box -ypad 5 +ypad 5 -ticks 1110; plot -x 1 dMag fMag + label -x mag_inst -y det_frac + + set found = (index1 >= 0) + plot PSF_INST_MAG_raw found -c red + + set ldmag = log(PSF_INST_MAG_SIG_raw) + section overlay 0 0 1 1; lim dMag -5 1.2; box -ypad 5 +ypad 5 -ticks 1011 -labels 1001; plot PSF_INST_MAG_raw ldmag + label +y log(S/N) + + png -name $3 +end + +macro show.dpair + if ($0 != 2) + echo "USAGE: show.dpair (N)" + break + end + + list word -split $dpairs:$1 + if ($word:n != 7) + echo "invalid dpair $1" + break + end + + $Nr = $word:4 + + reindex v1 = $word:0 using index1 + reindex dv = $word:1 using index1 + reindex v2 = $word:2 using index2 + reindex rv = $word:3 using index$Nr + + set delta = (v1 - v2) / dv + if (("$word:5" == "def") || ("$word:6" == "def")) + lim rv delta; box; plot rv delta + else + lim rv $word:5 $word:6; box; plot rv delta + end + label -y '$word:0' -x '$word:3' +end + +# this list is used to compare a pair of vectors with an error it is +# assumed that the vector sets have been loaded and matched with +# match2d to generate index vectors 'index1' and index2'. the macro +# show.dpair generates a plot of the range vector vs (v1 - v2) / dv1. +# The indices for v1, dv1, and v2 are index1,1, and 2 respectively. The +# index for the range vector is defined by the integer following that +# vector. The y-limits of the plot are given by the last two numbers +# (use 'def') for the full default range of the delta vector +list dpairs + # v1 dv v2 range + X_PSF X_PSF_SIG Xraw Mraw 2 -10.0 10.0 + Y_PSF Y_PSF_SIG Yraw Mraw 2 -10.0 10.0 + PSF_INST_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + PSF_INST_FLUX PSF_INST_FLUX_SIG Fraw Mraw 2 -10.0 10.0 + AP_MAG PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 + AP_MAG_RAW PSF_INST_MAG_SIG Mraw Mraw 2 -10.0 10.0 +end + +# if we run this test as a stand-alone program somewhere, we may need to create a local copy of the photcode file: +macro mkphotcodes + if ($0 != 2) + echo "USAGE: mkphotcodes (filename)" + break + end + + exec /bin/rm -f $1 + output $1 + echo "# airmass color astrometry mag photom astrom mask photom mask" + echo "# code name type zero slope offset c1 c2 slope zero equiv sys scale scale sys poor bad poor bad" + echo " 1 g_SYNTH sec 0.000 0.000 0.000 1 3 0.0000 0 21 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 2 r_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 22 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3 i_SYNTH sec 0.000 0.000 0.000 2 3 0.0000 0 23 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 4 z_SYNTH sec 0.000 0.000 0.000 3 4 0.0000 0 24 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 5 y_SYNTH sec 0.000 0.000 0.000 4 5 0.0000 0 25 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3001 SYNTH.g ref 0.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3002 SYNTH.r ref 0.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3003 SYNTH.i ref 0.000 0.000 0.000 - - 0.0000 0 3 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3004 SYNTH.z ref 0.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + echo " 3005 SYNTH.y ref 0.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000" + output stdout +end + +macro ckradialflux + data $1 + read -fits Chip.xrad X_APER Y_APER PSF_FWHM APER_FLUX APER_FLUX_ERR APER_FLUX_STDEV APER_FILL + read -fits Chip.psf X_PSF Y_PSF PSF_INST_MAG + set mag = PSF_INST_MAG + # XXX include nradii in header + for i 0 11 + set mA$i = -2.5*log(APER_FLUX:$i) + set dM$i = mA$i - PSF_INST_MAG + vstat dM$i + $DM$i = $MEDIAN + end + for i 1 11 + $j = $i - 1 + echo {$DM$i - $DM$j} + end + + lim mag -2 2; clear; box + for i 0 11 + plot mag dM$i -c black + end +end + +macro init + # config for ppImage to generate chip, mask, weight + $ppImageConfig = -recipe PPIMAGE PPIMAGE_N + $ppImageConfig = $ppImageConfig -Db BACKGROUND T + $ppImageConfig = $ppImageConfig -Db CHIP.FITS T + $ppImageConfig = $ppImageConfig -Db CHIP.MASK.FITS T + $ppImageConfig = $ppImageConfig -Db CHIP.VARIANCE.FITS T + $ppImageConfig = $ppImageConfig -Db BASE.FITS F + $ppImageConfig = $ppImageConfig -Db VARIANCE.BUILD T + $ppImageConfig = $ppImageConfig -Db PHOTOM F + + # basic options for the these images (filter, location, obstype) + $BaseOptions = -type OBJECT -filter r -ra 270.70 -dec -23.70 -pa 0.0 + $BaseOptions = $BaseOptions -Df PSASTRO:DVO.GETSTAR.MAX.RHO 50000.0 + + # PSF.CONVOLVE : if true, we insert delta functions (and optionally + # galaxies) and smooth the image with the psf model + # (uses a GAUSS regardless of the model). Note that + # PSF.CONVOLVE = T is faster than F, but (a) only + # allows Gauss models and (b) only yields quantized + # locations + + # create an image with fake sources and insert the resulting cmf file into a dvodb + $RefConfig = -camera SIMTEST + $RefConfig = $RefConfig -recipe PPSIM STACKTEST.MAKE + $RefConfig = $RefConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $RefConfig = $RefConfig -Db PSF.CONVOLVE F + + # options for the reference image + $RefOptions = $BaseOptions + $RefOptions = $RefOptions -exptime 100.0 + $RefOptions = $RefOptions -seeing 1.0 + $RefOptions = $RefOptions -skymags 21.0 + $RefOptions = $RefOptions -D PSF.MODEL PS_MODEL_GAUSS + $RefOptions = $RefOptions -Df STARS.DENSITY 10.0 + $RefOptions = $RefOptions -Df STARS.SIGMA.LIM 0.5 + + # basic config for ppSim with randomly distributed stars and NO galaxies + $RealConfig = -camera SIMTEST + $RealConfig = $RealConfig -recipe PPSIM STACKTEST.RUN + $RealConfig = $RealConfig -D PSASTRO:PSASTRO.CATDIR catdir.ref + $RealConfig = $RealConfig -Db STARS.FAKE F + $RealConfig = $RealConfig -Db STARS.REAL T + $RealConfig = $RealConfig -Db MATCH.DENSITY F + $RealConfig = $RealConfig -Db PSF.CONVOLVE F + $RealConfig = $RealConfig -Df STARS.DENSITY 10.0 + $RealConfig = $RealConfig -Df STARS.SIGMA.LIM 1.0 + $RealConfig = $RealConfig -Db GALAXY.FAKE F + $RealConfig = $RealConfig -Db GALAXY.GRID F + + # options for the repeated images + $RealOptions = $BaseOptions -exptime 30.0 + + # sample alternate options: + # $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_PS1_V1 + # $ppSimOptions = $FakeOptions -Df PSF.ARATIO 1.2 + # $ppSimOptions = $FakeOptions -Df PSF.THETA +30.0 + # $ppSimOptions = $FakeOptions -D PSF.MODEL PS_MODEL_GAUSS + + list fwhm + 1.0 + 1.1 + 1.2 + 1.5 + end +end + +if ($SCRIPT) + fulltest 4 + exit 0 +end