Index: branches/eam_branches/20090715/DataStoreServer/scripts/Makefile.am =================================================================== --- branches/eam_branches/20090715/DataStoreServer/scripts/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/DataStoreServer/scripts/Makefile.am (revision 25022) @@ -4,8 +4,5 @@ install_files = \ dsreg \ - dsprodtool \ - dsrootindex \ - dsprodindex \ - dsfsindex + dsprodtool install_SCRIPTS = $(install_files) Index: branches/eam_branches/20090715/Doxyfile =================================================================== --- branches/eam_branches/20090715/Doxyfile (revision 25020) +++ branches/eam_branches/20090715/Doxyfile (revision 25022) @@ -1,3 +1,3 @@ -# Doxyfile 1.5.4 +# Doxyfile 1.5.8 # This file describes the settings to be used by the documentation system @@ -15,9 +15,9 @@ #--------------------------------------------------------------------------- -# This tag specifies the encoding used for all characters in the config file that -# follow. The default is UTF-8 which is also the encoding used for all text before -# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into -# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of -# possible encodings. +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. DOXYFILE_ENCODING = UTF-8 @@ -55,8 +55,9 @@ # The default language is English, other supported languages are: # Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, -# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, -# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, -# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, -# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. +# Croatian, Czech, Danish, Dutch, Farsi, Finnish, French, German, Greek, +# Hungarian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, Polish, +# Portuguese, Romanian, Russian, Serbian, Serbian-Cyrilic, Slovak, Slovene, +# Spanish, Swedish, and Ukrainian. OUTPUT_LANGUAGE = English @@ -155,11 +156,4 @@ MULTILINE_CPP_IS_BRIEF = NO -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = YES - # If the INHERIT_DOCS tag is set to YES (the default) then an undocumented # member inherits the documentation from any documented member that it @@ -196,12 +190,35 @@ # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. OPTIMIZE_OUTPUT_JAVA = NO -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to -# include (a tag file for) the STL sources as input, then you should +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it parses. +# With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this tag. +# The format is ext=language, where ext is a file extension, and language is one of +# the parsers supported by doxygen: IDL, Java, Javascript, C#, C, C++, D, PHP, +# Objective-C, Python, Fortran, VHDL, C, C++. For instance to make doxygen treat +# .inc files as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should # set this tag to YES in order to let doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); v.s. @@ -211,5 +228,5 @@ BUILTIN_STL_SUPPORT = NO -# If you use Microsoft's C++/CLI language, you should set this option to YES to +# If you use Microsoft's C++/CLI language, you should set this option to YES to # enable parsing support. @@ -221,4 +238,13 @@ SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen to replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC @@ -237,13 +263,29 @@ SUBGROUPING = NO -# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is -# documented as struct with the name of the typedef. So +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So # typedef struct TypeS {} TypeT, will appear in the documentation as a struct # with name TypeT. When disabled the typedef will appear as a member of a file, # namespace, or class. And the struct will be named TypeS. This can typically -# be useful for C code where the coding convention is that all structs are -# typedef'ed and only the typedef is referenced never the struct's name. +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. TYPEDEF_HIDES_STRUCT = NO + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penality. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will rougly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 #--------------------------------------------------------------------------- @@ -281,8 +323,9 @@ EXTRACT_LOCAL_METHODS = NO -# If this flag is set to YES, the members of anonymous namespaces will be extracted -# and appear in the documentation as a namespace called 'anonymous_namespace{file}', -# where file will be replaced with the base name of the file that contains the anonymous -# namespace. By default anonymous namespace are hidden. +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespace are hidden. EXTRACT_ANON_NSPACES = NO @@ -363,9 +406,15 @@ SORT_BRIEF_DOCS = NO +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be # sorted by fully-qualified names, including namespaces. If set to # NO (the default), the class list will be sorted only by class name, # not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. # Note: This option applies only to the class list, not to the # alphabetical list. @@ -424,7 +473,20 @@ SHOW_DIRECTORIES = YES +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + # The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from the -# version control system). Doxygen will invoke the program by executing (via +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via # popen()) the command , where is the value of # the FILE_VERSION_FILTER tag, and is the name of an input file @@ -433,4 +495,13 @@ FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed by +# doxygen. The layout file controls the global structure of the generated output files +# in an output format independent way. The create the layout file that represents +# doxygen's defaults, run doxygen with the -l option. You can optionally specify a +# file name after the option, if omitted DoxygenLayout.xml will be used as the name +# of the layout file. + +LAYOUT_FILE = #--------------------------------------------------------------------------- @@ -494,23 +565,24 @@ # with spaces. -INPUT = mainpage.dox \ - ippdb/src \ - Nebulous/nebclient/src \ - ppArith/src \ - ppConfigDump/src \ - ppImage/src \ - ppMerge/src \ - ppStack/src \ - ppSub/src \ - psLib/src \ - psastro/src \ - psphot/src \ - pswarp/src \ - psModules/src - -# This tag can be used to specify the character encoding of the source files that -# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default -# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. -# See http://www.gnu.org/software/libiconv for the list of possible encodings. +INPUT = mainpage.dox \ + ippdb/src \ + Nebulous/nebclient/src \ + ppArith/src \ + ppConfigDump/src \ + ppImage/src \ + ppMerge/src \ + ppStack/src \ + ppSub/src \ + psLib/src \ + psastro/src \ + psphot/src \ + pswarp/src \ + psModules/src + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. INPUT_ENCODING = UTF-8 @@ -523,7 +595,7 @@ # *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 -FILE_PATTERNS = *.c \ - *.h \ - *.in +FILE_PATTERNS = *.c \ + *.h \ + *.in # The RECURSIVE tag can be used to turn specify whether or not subdirectories @@ -554,7 +626,8 @@ # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the output. -# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, -# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test EXCLUDE_SYMBOLS = @@ -591,5 +664,6 @@ # is the value of the INPUT_FILTER tag, and is the name of an # input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be # ignored. @@ -597,6 +671,8 @@ # The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: # pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further # info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER @@ -618,7 +694,5 @@ # be generated. Documented entities will be cross-referenced with these sources. # Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH -# then you must also enable this option. If you don't then doxygen will produce -# a warning and turn it on anyway +# VERBATIM_HEADERS is set to NO. SOURCE_BROWSER = YES @@ -635,5 +709,5 @@ STRIP_CODE_COMMENTS = NO -# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# If the REFERENCED_BY_RELATION tag is set to YES # then for each documented function all documented # functions referencing it will be listed. @@ -641,5 +715,5 @@ REFERENCED_BY_RELATION = YES -# If the REFERENCES_RELATION tag is set to YES (the default) +# If the REFERENCES_RELATION tag is set to YES # then for each documented function all documented entities # called/used by that function will be listed. @@ -647,8 +721,9 @@ REFERENCES_RELATION = YES -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. Otherwise they will link to the documentstion. +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. REFERENCES_LINK_SOURCE = YES @@ -739,11 +814,4 @@ HTML_ALIGN_MEMBERS = YES -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML # documentation will contain sections that can be hidden and shown after the @@ -754,4 +822,37 @@ HTML_DYNAMIC_SECTIONS = NO +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + # If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can # be used to specify the file name of the resulting .chm file. You @@ -774,4 +875,10 @@ GENERATE_CHI = NO +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + # If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag # controls whether a binary table of contents is generated (YES) or a @@ -785,4 +892,53 @@ TOC_EXPAND = YES +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and QHP_VIRTUAL_FOLDER +# are set, an additional index file will be generated that can be used as input for +# Qt's qhelpgenerator to generate a Qt Compressed Help (.qch) of the generated +# HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to add. +# For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the custom filter to add.For more information please see +# Qt Help Project / Custom Filters. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this project's +# filter section matches. +# Qt Help Project / Filter Attributes. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + # The DISABLE_INDEX tag can be used to turn on/off the condensed index at # top of each HTML page. The value NO (the default) enables the index and @@ -796,10 +952,18 @@ ENUM_VALUES_PER_LINE = 10 -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to FRAME, a side panel will be generated +# containing a tree-like index structure (just like the one that # is generated for HTML Help). For this to work a browser that supports # JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, # Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. +# probably better off using the HTML help feature. Other possible values +# for this tag are: HIERARCHIES, which will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list; +# ALL, which combines the behavior of FRAME and HIERARCHIES; and NONE, which +# disables this behavior completely. For backwards compatibility with previous +# releases of Doxygen, the values YES and NO are equivalent to FRAME and NONE +# respectively. GENERATE_TREEVIEW = NO @@ -810,4 +974,12 @@ TREEVIEW_WIDTH = 250 + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 #--------------------------------------------------------------------------- @@ -1023,6 +1195,8 @@ # If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this # tag is set to NO the size of the Perl module output will be much smaller # and Perl will parse it just the same. @@ -1111,12 +1285,14 @@ # can be added for each tagfile. The format of a tag file without # this location is as follows: -# TAGFILES = file1 file2 ... +# +# TAGFILES = file1 file2 ... # Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... +# +# TAGFILES = file1=loc1 "file2 = loc2" ... # where "loc1" and "loc2" can be relative or absolute paths or # URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) # If a tag file is not located in the directory in which doxygen # is run, you must also specify the path to the tagfile here. @@ -1160,8 +1336,9 @@ # You can define message sequence charts within doxygen comments using the \msc -# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to -# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to -# specify the directory where the mscgen tool resides. If left empty the tool is assumed to -# be found in the default search path. +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. MSCGEN_PATH = @@ -1180,4 +1357,27 @@ HAVE_DOT = YES +# By default doxygen will write a font called FreeSans.ttf to the output +# directory and reference it in all dot files that doxygen generates. This +# font does not include all possible unicode characters however, so when you need +# these (or just want a differently looking font) you can specify the font name +# using DOT_FONTNAME. You need need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = FreeSans + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + # If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen # will generate a graph for each documented class showing the direct and @@ -1224,17 +1424,17 @@ INCLUDED_BY_GRAPH = YES -# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. CALL_GRAPH = NO -# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will -# generate a caller dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable caller graphs for selected -# functions only using the \callergraph command. +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. CALLER_GRAPH = NO @@ -1247,5 +1447,5 @@ # If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES # then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include +# in a graphical way. The dependency relations are determined by the #include # relations between the files in the directories. @@ -1253,5 +1453,5 @@ # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif +# generated by dot. Possible values are png, jpg, or gif # If left blank png will be used. @@ -1269,10 +1469,10 @@ DOTFILE_DIRS = -# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of # nodes that will be shown in the graph. If the number of nodes in a graph # becomes larger than this value, doxygen will truncate the graph, which is -# visualized by representing a node as a red box. Note that doxygen if the number -# of direct children of the root node in a graph is already larger than -# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note # that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. @@ -1290,8 +1490,8 @@ # Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, which results in a white background. -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). DOT_TRANSPARENT = YES @@ -1317,5 +1517,5 @@ #--------------------------------------------------------------------------- -# Configuration::additions related to the search engine +# Options related to the search engine #--------------------------------------------------------------------------- Index: branches/eam_branches/20090715/Nebulous-Server/Changes =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/Changes (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/Changes (revision 25022) @@ -35,4 +35,6 @@ - change Nebulous::Server->stat_object() to return both the total number of instance and just those that are available + - add Nebulous::Server->prune_object() API + - don't replicate to the volume being implied by the key being replicated! 0.16 Index: branches/eam_branches/20090715/Nebulous-Server/MANIFEST =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/MANIFEST (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/MANIFEST (revision 25022) @@ -49,3 +49,4 @@ t/17_server_swap_objects.t t/18_server_chmod_object.t +t/19_server_prune_object.t t/75_parse_neb_key.t Index: branches/eam_branches/20090715/Nebulous-Server/bin/neb-voladm =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/bin/neb-voladm (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/bin/neb-voladm (revision 25022) @@ -91,5 +91,4 @@ my $query = $dbh->prepare($q); $query->execute(@bind); - $dbh->do("call getmountedvol"); $dbh->commit; }; Index: branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server.pm =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server.pm (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server.pm (revision 25022) @@ -696,5 +696,5 @@ $log->debug("entered - @_"); - my ($key, $vol_name) = validate_pos(@_, + my ($key, $dest_vol_name) = validate_pos(@_, { type => SCALAR, @@ -726,28 +726,35 @@ # instance on it. If all avilable volume already have an instance on them # then we should throw an error - - # vol_name overrides the key implied volume - eval { - $key = parse_neb_key($key, $vol_name); + # volume names implied as part of the key are *IGNORED* as the source and + # *SHOULD NOT* be used as the destination either + + eval { + $key = parse_neb_key($key); }; $log->logdie("$@") if $@; - $vol_name = $key->volume; my $db = $self->db($key); - if (defined $vol_name + # puke if the source volume is bogus, we may want to actually use this as + # the instance to be copied later + if (defined $key->volume and not $self->_is_valid_volume_name($key, $key->volume)) { unless ($key->hard_volume) { - $log->warn( "$vol_name is not a known volume name" ); - $vol_name = undef; + $log->warn($key->volume . " not a known volume name"); } else { - $log->logdie("$vol_name is not a valid volume name"); - } + $log->logdie("$key is not a valid volume name"); + } + } + # puke if the source volume is bogus, we may want to actually use this as + # the instance to be copied later + if (defined $dest_vol_name + and not $self->_is_valid_volume_name($key, $dest_vol_name)) { + $log->logdie($key->volume . " is not a valid volume name"); } my ($vol_id, $vol_host, $vol_path, $vol_xattr); - if (defined $vol_name) { + if (defined $dest_vol_name) { ($vol_id, $vol_host, $vol_path, $vol_xattr) - = $self->_get_storage_volume($key, $vol_name); + = $self->_get_storage_volume($key, $dest_vol_name); } else { ($vol_id, $vol_host, $vol_path, $vol_xattr) @@ -810,5 +817,5 @@ $db->rollback; # handle soft volumes - if (defined $vol_name and not defined $key->hard_volume) { + if (defined $dest_vol_name and not defined $key->hard_volume) { $log->debug("retrying with 'any' volume"); return $self->replicate_object($key->path, 'any'); @@ -841,4 +848,99 @@ return "$uri"; +} + + +sub prune_object +{ + my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); + + my ($key) = validate_pos(@_, + { + type => SCALAR, + callbacks => { + 'is valid object key' + => sub { $self->_is_valid_object_key($_[0]) }, + }, + }, + ); + + my $sql = $self->sql; + + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + + my $db = $self->db($key); + + my $rows_removed = 0; +TRANS: while (1) { + eval { + # remove key from cache + $self->cache->delete($key->path) if defined $self->cache; + + my $so_id; + { + my $query = $db->prepare_cached( $sql->find_object_by_ext_id ); + $query->execute( $key->path ); + $so_id = $query->fetchrow_hashref->{'so_id'}; + $query->finish; + } + + # record the path of the innaccesible files for deferred + # deletion + my $rows_copied; + { + my $query = $db->prepare_cached( $sql->copy_dead_instances_to_deleted ); + $rows_copied = $query->execute( $so_id ); + } + + # check to see if there is anything to be done + unless ($rows_copied > 0) { + $db->rollback; + return; + } + + # In MySQL you can't select from a table your deleting rows from so + # we first have to get a list of instances to be removed, and then + # removed them. + my $rows_found; + { + my $query = $db->prepare_cached( $sql->find_dead_instances_by_so_id ); + $rows_found = $query->execute( $so_id ); + foreach my $row ($query->fetchrow_hashref) { + # remove dead instances + my $query = $db->prepare_cached( $sql->delete_instance_by_ins_id); + $rows_removed += $query->execute( $row->{ins_id} ); + } + $query->finish; + } + + # sanity check + die("instances inaccessible ($rows_copied) != instances removed ($rows_removed)") + unless $rows_copied == $rows_removed; + + $db->commit; + $log->debug("commit"); + }; + if ($@) { + $db->rollback; + $log->debug("rollback"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; + redo TRANS; + } + $log->logdie("error: $@"); + } + last; + } + + $log->debug("leaving"); + + return $rows_removed; } @@ -1674,17 +1776,7 @@ # remove key from cache $self->cache->delete($key->path) if defined $self->cache; - - # record the path of the innaccesible files for deferred - # deletion - { - my $query = $db->prepare_cached( $sql->copy_instances_to_deleted ); - $query->execute( $so_id ); - } - # remove all instances... not strictly nessicary as the delete - # from storage_object should cascade but the fkey was specified - # without the cascade - { - my $query = $db->prepare_cached( $sql->delete_instance_by_so_id ); - $query->execute( $so_id ); + + if ($total > $available) { + $self->prune_object("$key"); } Index: branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server.pod =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server.pod (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server.pod (revision 25022) @@ -29,5 +29,6 @@ Nebulous::Server->stat_object( $key ); Nebulous::Server->mounts(); - Nebulous::Server->chmod_object( $key, $mode) ; + Nebulous::Server->chmod_object( $key, $mode); + Nebulous::Server->prune_object( $key ); =head1 DESCRIPTION @@ -212,4 +213,10 @@ exception on failure. +=item * prune_object( $key ); + +Removes all of the inaccessible instances from an object. + +Accepts 1 mandatory parameter. Returns the number of inaccessible instances actually pruned from the object. + =back Index: branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server/Config.pm =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server/Config.pm (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server/Config.pm (revision 25022) @@ -13,5 +13,5 @@ use Log::Log4perl qw( :levels ); -use Params::Validate qw( validate validate_pos SCALAR ARRAYREF ); +use Params::Validate qw( validate validate_pos SCALAR ARRAYREF UNDEF ); our %LEVELS = ( @@ -32,5 +32,5 @@ dsn => { type => SCALAR }, dbuser => { type => SCALAR }, - dbpasswd => { type => SCALAR }, + dbpasswd => { type => SCALAR | UNDEF }, }; @@ -48,5 +48,5 @@ dsn => { type => SCALAR, optional => 1 }, dbuser => { type => SCALAR, optional => 1 }, - dbpasswd => { type => SCALAR, optional => 1 }, + dbpasswd => { type => SCALAR | UNDEF, optional => 1 }, memcached_servers => { type => ARRAYREF, optional => 1 }, }; Index: branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 25022) @@ -183,8 +183,4 @@ WHERE ins_id = ? }, - delete_instance_by_so_id => qq{ - DELETE FROM instance - WHERE so_id = ? - }, get_object_from_uri => qq{ SELECT so_id @@ -218,10 +214,22 @@ GROUP BY so_id }, - copy_instances_to_deleted => qq{ + copy_dead_instances_to_deleted => qq{ INSERT INTO deleted SELECT vol_id, uri, NULL FROM instance + LEFT JOIN mountedvol + USING(vol_id) WHERE so_id = ? + AND (mountedvol.available IS NULL || mountedvol.available = 0) + }, + find_dead_instances_by_so_id => qq{ + SELECT so_id, ins_id, vol_id, uri + FROM instance + LEFT JOIN mountedvol + USING(vol_id) + WHERE + so_id = ? + AND (mountedvol.available IS NULL || mountedvol.available = 0) }, get_object_instances => qq{ @@ -329,5 +337,5 @@ }, find_object_by_ext_id => qq{ - SELECT ext_id, ext_id_basename + SELECT so_id, ext_id, ext_id_basename FROM storage_object WHERE ext_id = ? Index: branches/eam_branches/20090715/Nebulous-Server/t/05_server_replicate_object.t =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/t/05_server_replicate_object.t (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/t/05_server_replicate_object.t (revision 25022) @@ -8,5 +8,5 @@ use warnings FATAL => qw( all ); -use Test::More tests => 26; +use Test::More tests => 25; use lib qw( ./t ./lib ); Index: branches/eam_branches/20090715/Nebulous-Server/t/09_server_delete_instance.t =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/t/09_server_delete_instance.t (revision 25020) +++ branches/eam_branches/20090715/Nebulous-Server/t/09_server_delete_instance.t (revision 25022) @@ -58,5 +58,5 @@ my $key = "foo"; my $uri1 = $neb->create_object($key, 'node01'); - my $uri2 = $neb->replicate_object($key, 'node2'); + my $uri2 = $neb->replicate_object($key, 'node02'); # make one of the instances unavailable Index: branches/eam_branches/20090715/Nebulous-Server/t/19_server_prune_object.t =================================================================== --- branches/eam_branches/20090715/Nebulous-Server/t/19_server_prune_object.t (revision 25022) +++ branches/eam_branches/20090715/Nebulous-Server/t/19_server_prune_object.t (revision 25022) @@ -0,0 +1,73 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 08_server_delete_instance.t,v 1.10.22.1 2008-12-14 22:52:37 eugene Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 6; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $uri1 = $neb->create_object($key, 'node01'); + my $uri2 = $neb->replicate_object($key, 'node02'); + + # make one of the instances unavailable + my $dbh = test_dbh(); + $dbh->do("UPDATE mountedvol SET available = 0 WHERE name = 'node01'"); + + ok($neb->prune_object($key), "prune object"); + + expected_dataset_ok( + deleted => [vol_id => 1, uri => $uri1], + ); + + expected_dataset_ok( + instance => [ins_id => 2, so_id => 1, vol_id => 2, uri => $uri2], + ); +} + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $uri1 = $neb->create_object($key); + + is($neb->prune_object($key), 0, "no dead instances to remove"); +} + +Test::Nebulous->setup; + +eval { + $neb->prune_object(); +}; +like($@, qr/1 was expected/, "no params"); + +Test::Nebulous->setup; + +eval { + my $key = "foo"; + my $uri1 = $neb->create_object($key); + + $neb->prune_object("foo", 2); +}; +like($@, qr/1 was expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090715/Nebulous/Changes =================================================================== --- branches/eam_branches/20090715/Nebulous/Changes (revision 25020) +++ branches/eam_branches/20090715/Nebulous/Changes (revision 25022) @@ -6,4 +6,7 @@ - add neb-rm --force flag - change neb-locate's output seperator to \n + - add Nebulous::Client->prune() method + - add Nebulous::Client->there_can_be_only_one() method + - add neb-cull --one_only flag 0.10 Index: branches/eam_branches/20090715/Nebulous/MANIFEST =================================================================== --- branches/eam_branches/20090715/Nebulous/MANIFEST (revision 25020) +++ branches/eam_branches/20090715/Nebulous/MANIFEST (revision 25022) @@ -99,5 +99,8 @@ t/67_client_swap.t t/68_client_chmod.t +t/69_client_prune.t t/70_neb-ls.t +t/71_client_there_can_be_only_one.t +t/72_neb-cull.t t/90_nebclient.t t/TEST.PL Index: branches/eam_branches/20090715/Nebulous/bin/neb-cull =================================================================== --- branches/eam_branches/20090715/Nebulous/bin/neb-cull (revision 25020) +++ branches/eam_branches/20090715/Nebulous/bin/neb-cull (revision 25022) @@ -16,5 +16,5 @@ use Pod::Usage qw( pod2usage ); -my ($server, $volume); +my ($server, $volume, $one_only); $server = $ENV{'NEB_SERVER'} unless $server; @@ -23,4 +23,5 @@ 'server|s=s' => \$server, 'volume|v=s' => \$volume, + 'one_only|o' => \$one_only, ) || pod2usage( 2 ); @@ -40,10 +41,24 @@ unless defined $neb; +my @cull_args; +push @cull_args, $key; if (defined $volume) { - $neb->cull($key, $volume) - or die "failed to replicate Nebulous key: $key"; + push @cull_args, $volume; +} + +if ($one_only) { + my $status = $neb->there_can_be_only_one(@cull_args); + if ($status == 0) { + die "not enough instances of Nebulous key: $key"; + } + if (not defined $status or $status < 1) { + die "failed to cull Nebulous key: $key - " . $neb->err + if defined $neb->err; + die "failed to cull Nebulous key: $key"; + } } else { - $neb->cull($key) - or die "failed to replicate Nebulous key: $key"; + unless (defined $neb->cull(@cull_args)) { + die "failed to cull Nebulous key: $key - " . $neb->err; + } } @@ -58,5 +73,5 @@ =head1 SYNOPSIS - neb-cull [--server ] [--volume ] + neb-cull [--server ] [--volume ] [--one_only] =head1 DESCRIPTION @@ -77,4 +92,12 @@ Symbolic name of the volume to create the new instance on. + +Optional. + +=item * --one_only|-o + +Removes all but one instances of an object. Any inaccessible instances are +removed. If C<--volume> is specified, and an instance exists on that volume, +all instances but the one on that volume are removed. Optional. @@ -110,5 +133,5 @@ =head1 COPYRIGHT -Copyright (C) 2007-2008 Joshua Hoblitt. All rights reserved. +Copyright (C) 2007-2009 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/eam_branches/20090715/Nebulous/bin/neb-rm =================================================================== --- branches/eam_branches/20090715/Nebulous/bin/neb-rm (revision 25020) +++ branches/eam_branches/20090715/Nebulous/bin/neb-rm (revision 25022) @@ -42,5 +42,5 @@ foreach my $key (@ARGV) { unless (defined $move) { - $neb->delete($key) or warn "failed to delete key: $key"; + $neb->delete($key, $force) or warn "failed to delete key: $key"; } else { $neb->move($key, "${key}.rm") or warn "failed to move key: $key"; Index: branches/eam_branches/20090715/Nebulous/lib/Nebulous/Client.pm =================================================================== --- branches/eam_branches/20090715/Nebulous/lib/Nebulous/Client.pm (revision 25020) +++ branches/eam_branches/20090715/Nebulous/lib/Nebulous/Client.pm (revision 25022) @@ -297,5 +297,5 @@ my $instances = $stats[6]; if (defined $instances and $instances < 2) { - $log->debug("can not cull - not enough instances"); + $self->set_err("can not cull - not enough instances"); $log->debug("leaving"); @@ -305,5 +305,5 @@ $locations = $self->find_instances($key, $vol_name); unless (defined $locations) { - $log->debug( "no instances" ); + $self->set_err( "no instances" ); $log->debug( "leaving" ); @@ -313,5 +313,5 @@ $locations = $self->find_instances($key); unless ($locations) { - $log->debug("no instances"); + $self->set_err( "no instances" ); $log->debug("leaving"); @@ -320,5 +320,5 @@ if (scalar @{ $locations } < 2) { - $log->debug("can not cull - not enough instances"); + $self->set_err("can not cull - not enough instances"); $log->debug("leaving"); @@ -339,4 +339,75 @@ return $uri; +} + + +sub there_can_be_only_one +{ + my $self = shift; + + my ($key, $vol_name) = validate_pos(@_, + { + type => SCALAR, + }, + { + # volume + type => SCALAR, + optional => 1, + }, + ); + + $log->debug( "entered - @_" ); + + my $locations; + my $removed = 0; + eval { + # first - strip off any inaccesible instances + if (not defined $self->prune($key)) { + # use the prune() method to also determine if $key is valid + die "invalid key: $key"; + } + + # check to see if there is an instance on $vol_name that should be the + # sole survivor + if (defined $vol_name) { + $locations = $self->find_instances($key, $vol_name); + } + + if (defined $locations) { + my $instances = $self->find_instances($key); + if (scalar @$instances < 2) { + die "not enough instances"; + } + foreach my $victim (@$instances) { + next if $victim eq $locations->[0]; + $self->delete_instance($key, $victim); + $removed++; + } + } else { + # nuke whatever + my $stats = $self->stat($key); + # start at one so cull() is called one less time then the # of + # instances + if ($stats->[6] < 2) { + die "not enough instances"; + } + for (my $i = 1; $i < $stats->[6]; $i++) { + $self->cull($key); + $removed++; + } + } + + }; + if ($@) { + if ($@ =~ qr/invalid key/) { + $self->set_err($@); + return; + } + $log->logdie($@); + } + + $log->debug("leaving"); + + return $removed; } @@ -1117,4 +1188,37 @@ +sub prune +{ + my $self = shift; + + my ( $key ) = validate_pos( @_, + { + type => SCALAR, + }, + ); + + $log->debug( "entered - @_" ); + + my $response = $self->{ 'server' }->prune_object( $key ); + if ( $response->fault ) { + $self->set_err($response->faultstring); + if ($response->faultstring =~ /is valid object key/) { + $log->debug( "leaving" ); + return; + } + + $log->logdie("unhandled fault - ", $self->err); + } + + $log->debug( "server returned a stat" ); + + my $n_removed = $response->result; + + $log->debug( "leaving" ); + + return $n_removed; +} + + sub err { Index: branches/eam_branches/20090715/Nebulous/lib/Nebulous/Client.pod =================================================================== --- branches/eam_branches/20090715/Nebulous/lib/Nebulous/Client.pod (revision 25020) +++ branches/eam_branches/20090715/Nebulous/lib/Nebulous/Client.pod (revision 25022) @@ -35,4 +35,6 @@ my $mounts = $neb->mounts(); my $mode = $neb->chmod( $key, $mode ); + my $n_dead = $neb->prune( $key ); + my $n_dead = $neb->there_can_be_only_one( $key, $vol ); =head1 DESCRIPTION @@ -405,4 +407,38 @@ Returns an arrayref of scalars. + +=item * prune($key) + +Removes all of the inaccessible instances from an object. + +=over 4 + +=item * key + +The storage object key (name). + +=back + +The number of instances removed. + +=item * there_can_be_only_one($key, $vol) + +Removes all but one instance of an object and automatically removes any +inaccessible instances. + +=over 4 + +=item * key + +The storage object key (name). + +=item * vol + +If specified, the only remaining instance will be left on this volume (if an +instance already exists there). + +=back + +The number of avaiable instances removed. =item * mounts() @@ -464,5 +500,5 @@ =head1 COPYRIGHT -Copyright (C) 2004-2008 Joshua Hoblitt. All rights reserved. +Copyright (C) 2004-2009 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/eam_branches/20090715/Nebulous/nebclient/configure.ac =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/configure.ac (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/configure.ac (revision 25022) @@ -5,5 +5,5 @@ AC_PREREQ(2.61) -AC_INIT([nebclient], [0.08], [jhoblitt@cpan.org]) +AC_INIT([nebclient], [0.17], [jhoblitt@cpan.org]) AC_CONFIG_SRCDIR([nebulous.wsdl]) Index: branches/eam_branches/20090715/Nebulous/nebclient/nebulous.wsdl =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/nebulous.wsdl (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/nebulous.wsdl (revision 25022) @@ -149,4 +149,12 @@ + + + + + + + + @@ -256,4 +264,11 @@ --> + + + + + @@ -417,4 +432,15 @@ + + + + + + + + + soap, server->endpoint, NULL, (char *)key, (char *)volume, (char **)&response) != SOAP_OK) { @@ -117,6 +116,9 @@ } + char *filename; if (!nebParseURI(server, (char *)response.result, &filename)) { nebSetErr(server, "can not parse URI"); + nebFree(filename); + return NULL; } @@ -131,20 +133,21 @@ } + int nebOpenCreate(nebServer *server, const char *key, const char *volume, char **URI) { - char *filename; - int file; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return -1; - } - - filename = nebCreate(server, key, volume, URI); - - file = open(filename, O_RDWR|O_TRUNC, 0660); + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return -1; + } + + char *filename = nebCreate(server, key, volume, URI); + if (!filename) { + return -1; + } + + int file = open(filename, O_RDWR|O_TRUNC, 0660); if (file == -1) { nebSetErr(server, "can not open %s: %s", filename, strerror(errno)); @@ -159,10 +162,7 @@ } + bool nebReplicate(nebServer *server, const char *key, const char *volume, char **URI) { - struct ns1__replicate_USCOREobjectResponse response; - char *filename; - int sourceFH, destFH; - REQUIRE_SERVER; @@ -177,8 +177,7 @@ // we don't do this, it's possible that open will find & open the new // instance that we're in the process of creating - sourceFH = nebOpen(server, (char *)key, NEB_READ); + int sourceFH = nebOpen(server, (char *)key, NEB_READ); if (sourceFH == -1) { nebSetErr(server, "can not open key %s", key); - nebFree(filename); return false; @@ -186,4 +185,5 @@ // ask the server for a new instance attached to our key + struct ns1__replicate_USCOREobjectResponse response; if (soap_call_ns1__replicate_USCOREobject(server->soap, server->endpoint, NULL, (char *)key, (char *)volume, (char **)&response) != SOAP_OK) { @@ -193,4 +193,5 @@ } + char *filename = NULL; if (!nebParseURI(server, (char *)response.result, &filename)) { nebSetErr(server, "can not parse URI"); @@ -199,5 +200,5 @@ } - destFH = open(filename, O_RDWR|O_TRUNC, 0660); + int destFH = open(filename, O_RDWR|O_TRUNC, 0660); if (destFH == -1) { nebSetErr(server, "can not open %s: %s\n", filename, strerror(errno)); @@ -234,19 +235,21 @@ } + bool nebCull(nebServer *server, const char *key) { - nebObjectInstances *locations; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return false; - } - - locations = nebFindInstances(server, key, NULL); + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return false; + } + + nebObjectInstances *locations = nebFindInstances(server, key, NULL); if (!locations || locations->n <= 1) { nebSetErr(server, "can not cull - not enough instances"); + if (locations) { + nebObjectInstancesFree(locations); + } return false; @@ -264,23 +267,21 @@ } + bool nebLock(nebServer *server, const char *key, nebRW flag) { - time_t endtime; - char *type; - int response; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return false; - } - - type = flag == NEB_READ ? "read" : "write"; - - endtime = time(NULL) + LOCK_TIMEOUT; + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return false; + } + + char *type = flag == NEB_READ ? "read" : "write"; + + time_t endtime = time(NULL) + LOCK_TIMEOUT; while (time(NULL) < endtime) { + int response; if (soap_call_ns1__lock_USCOREobject(server->soap, server->endpoint, NULL, (char *)key, type, &response) != SOAP_OK) { @@ -302,19 +303,18 @@ } + bool nebUnlock(nebServer *server, const char *key, nebRW flag) { - char *type; - int response; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return false; - } - - type = flag == NEB_READ ? "read" : "write"; - + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return false; + } + + char *type = flag == NEB_READ ? "read" : "write"; + + int response; if (soap_call_ns1__unlock_USCOREobject(server->soap, server->endpoint, NULL, (char *)key, type, &response) != SOAP_OK) { @@ -466,17 +466,13 @@ nebObjectInstances *nebFindInstances(nebServer *server, const char *key, const char *volume) { + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return NULL; + } + struct ns1__find_USCOREinstancesResponse response; - int resultElements = 0; - char **resultArray; - nebObjectInstances *locations = NULL; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return NULL; - } - // FIXME is this leaking memory when response goes out of scope? the gsoap // manual seems to 'suggest' that this is temporary data that gets cleaed @@ -488,11 +484,12 @@ } - resultElements = response.result->__size; - + int resultElements = response.result->__size; + + nebObjectInstances *locations = NULL; if (resultElements) { locations = xmalloc(sizeof(nebObjectInstances)); locations->n = resultElements; - resultArray = response.result->__ptr; + char **resultArray = response.result->__ptr; locations->URI = xmalloc(sizeof(char*) * resultElements); @@ -508,4 +505,5 @@ } + void nebObjectInstancesFree(nebObjectInstances *locations) { @@ -521,9 +519,7 @@ } + char *nebFind(nebServer *server, const char *key) { - nebObjectInstances *locations; - char *filename; - REQUIRE_SERVER; @@ -534,5 +530,5 @@ } - locations = nebFindInstances(server, key, NULL); + nebObjectInstances *locations = nebFindInstances(server, key, NULL); if (!locations) { if (!strstr(nebErr(server), "no instances on storage volume")) { @@ -548,4 +544,5 @@ } + char *filename; if (!nebParseURI(server, locations->URI[0], &filename)) { nebSetErr(server, "can not parse URI"); @@ -560,19 +557,16 @@ } + int nebOpen(nebServer *server, const char *key, nebRW flag) { - nebObjectInstances *locations; - char *filename; - int fh; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return -1; - } - - locations = nebFindInstances(server, key, NULL); + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return -1; + } + + nebObjectInstances *locations = nebFindInstances(server, key, NULL); if (!locations) { nebSetErr(server, "no instances found"); @@ -581,4 +575,5 @@ } + char *filename; if (!nebParseURI(server, locations->URI[0], &filename)) { nebSetErr(server, "can not parse URI"); @@ -588,4 +583,5 @@ } + int fh = 0; if (flag == NEB_WRITE) { if (locations->n > 1) { @@ -613,17 +609,16 @@ } + bool nebDelete(nebServer *server, const char *key) { - nebObjectInstances *locations; - - REQUIRE_SERVER; - - if (!key) { - nebSetErr(server, "parameter 'key' may not be NULL"); - - return false; - } - - locations = nebFindInstances(server, key, "any"); + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return false; + } + + nebObjectInstances *locations = nebFindInstances(server, key, "any"); if (!locations) { nebSetErr(server, "no instances found"); @@ -646,9 +641,7 @@ } + bool nebCopy(nebServer *server, const char *key, const char *newKey) { - int sourceFH; - int destFH; - REQUIRE_SERVER; @@ -665,5 +658,5 @@ } - sourceFH = nebOpen(server, (char *)key, NEB_READ); + int sourceFH = nebOpen(server, (char *)key, NEB_READ); if (sourceFH < 0) { // propigate nebOpen() error @@ -672,5 +665,5 @@ } - destFH = nebOpenCreate(server, newKey, NULL, NULL); + int destFH = nebOpenCreate(server, newKey, NULL, NULL); if (destFH < 0) { // propigate nebCreate() error @@ -696,4 +689,5 @@ } + bool nebMove(nebServer *server, const char *key, const char *newKey) { @@ -723,8 +717,7 @@ } + bool nebSwap(nebServer *server, const char *key1, const char *key2) { - struct ns1__create_USCOREobjectResponse response; - REQUIRE_SERVER; @@ -741,4 +734,5 @@ } + struct ns1__create_USCOREobjectResponse response; if (soap_call_ns1__swap_USCOREobjects(server->soap, server->endpoint, NULL, (char *)key1, (char *)key2, (char **)&response) != SOAP_OK) { @@ -749,10 +743,7 @@ } + bool nebDeleteInstance(nebServer *server, const char *key, const char *URI) { - int response; - char *filename; - bool status; - REQUIRE_SERVER; @@ -763,4 +754,5 @@ } + char *filename; if (!nebParseURI(server, URI, &filename)) { nebSetErr(server, "can not parse URI"); @@ -769,4 +761,5 @@ } + int response; if (soap_call_ns1__delete_USCOREinstance(server->soap, server->endpoint, NULL, (char *)key, (char *)URI, &response) != SOAP_OK) { @@ -776,4 +769,5 @@ } + bool status; if (response > 0) { status = nebNukeFile(server, filename); @@ -787,11 +781,7 @@ } + nebObjectStat *nebStat(nebServer *server, const char *key) { - struct ns1__stat_USCOREobjectResponse response; - nebObjectStat *stat; - int resultElements = 0; - char **resultArray = NULL; - REQUIRE_SERVER; @@ -802,9 +792,10 @@ } - stat = xmalloc(sizeof(nebObjectStat)); + nebObjectStat *stat = xmalloc(sizeof(nebObjectStat)); // FIXME is this leaking memory when response goes out of scope? the gsoap // manual seems to 'suggest' that this is temporary data that gets cleaed // up on the next soap function call + struct ns1__stat_USCOREobjectResponse response; if (soap_call_ns1__stat_USCOREobject(server->soap, server->endpoint, NULL, (char *)key, &response) != SOAP_OK) { @@ -813,19 +804,20 @@ } - resultElements = response.result->__size; - resultArray = response.result->__ptr; - - if (resultElements != 7) { + int resultElements = response.result->__size; + char **resultArray = response.result->__ptr; + + if (resultElements != 8) { nebSetErr(server, "server didn't return the proper number of stat elements"); return NULL; } - nullstrncpy(stat->so_id, resultArray[0], 256); - nullstrncpy(stat->ext_id, resultArray[1], 256); - nullstrncpy(stat->read_lock, resultArray[2], 256); + stat->so_id = atol(resultArray[0]); + stat->ext_id = atol(resultArray[1]); + stat->read_lock = atoi(resultArray[2]); nullstrncpy(stat->write_lock, resultArray[3], 256); nullstrncpy(stat->epoch, resultArray[4], 256); nullstrncpy(stat->mtime, resultArray[5], 256); - stat->instances = atoi(resultArray[6]); + stat->available = atoi(resultArray[6]); + stat->instances = atoi(resultArray[7]); return stat; @@ -833,8 +825,16 @@ +void nebObjectStatFree(nebObjectStat *stat) +{ + if (!stat) { + return; + } + + nebFree(stat); +} + + int nebChmod(nebServer *server, const char *key, mode_t mode) { - int response; - REQUIRE_SERVER; @@ -848,4 +848,5 @@ // manual seems to 'suggest' that this is temporary data that gets cleaed // up on the next soap function call + int response; if (soap_call_ns1__chmod_USCOREobject(server->soap, server->endpoint, NULL, (char *)key, mode, &response) != SOAP_OK) { @@ -858,4 +859,92 @@ +int nebPrune(nebServer *server, const char *key) +{ + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return -1; + } + + // FIXME is this leaking memory when response goes out of scope? the gsoap + // manual seems to 'suggest' that this is temporary data that gets cleaed + // up on the next soap function call + int response; + if (soap_call_ns1__prune_USCOREobject(server->soap, server->endpoint, + NULL, (char *)key, &response) != SOAP_OK) { + nebSetServerErr(server); + return -1; + } + + return response; +} + + +int nebThereCanBeOnlyOne(nebServer *server, const char *key, const char *volume) +{ + REQUIRE_SERVER; + + if (!key) { + nebSetErr(server, "parameter 'key' may not be NULL"); + + return -1; + } + + // first - strip off any inaccesible instances + if (nebPrune(server, key) < 0 ) { + // use the prune() method to also determine if $key is valid + return -1; + } + + // check to see if there is an instance on $vol_name that should be the + // sole survivor + nebObjectInstances *locations = NULL; + if (volume) { + locations = nebFindInstances(server, key, NULL); + } + + long removed = 0; + if (locations && locations->n) { + nebObjectInstances *instances = nebFindInstances(server, key, NULL); + for (long i = 0; i < instances->n; i++) { + if (strcmp(locations->URI[0], instances->URI[i]) == 0) { + continue; + } + if (!nebDeleteInstance(server, key, instances->URI[i])) { + nebObjectInstancesFree(instances); + nebObjectInstancesFree(locations); + return -1; + } + removed++; + } + nebObjectInstancesFree(instances); + nebObjectInstancesFree(locations); + } else { + nebObjectInstancesFree(locations); + // nuke whatever + nebObjectStat *stat = nebStat(server, key); + if (!stat) { + return -1; + } + + int available = stat->available; + nebObjectStatFree(stat); + + // start at one so cull() is called one less time then the # of + // instances + for (long i = 1; i < available; i++) { + if (!nebCull(server, key)) { + return -1; + } + removed++; + } + } + + return removed; +} + + char *nebErr(nebServer *server) { @@ -863,5 +952,7 @@ } + void nebFree(void *ptr) { + if (!ptr) { return; } xfree(ptr); } @@ -870,10 +961,6 @@ static void p_nebSetErr(const char * filename, unsigned long lineno, const char *function, nebServer *server, const char *format, ...) { - va_list args; - size_t err_size; - char err_location[ERRBUF_SIZE]; - size_t err_location_size; - - err_location_size = snprintf(err_location, ERRBUF_SIZE, "%s:%lu %s() - ", + char err_location[ERRBUF_SIZE]; + size_t err_location_size = snprintf(err_location, ERRBUF_SIZE, "%s:%lu %s() - ", filename, lineno, function); @@ -888,7 +975,8 @@ while (1) { + va_list args; va_start(args, format); // append error string after error location info - err_size = vsnprintf( + size_t err_size = vsnprintf( server->err_buf + err_location_size, server->err_buf_size - err_location_size, @@ -914,27 +1002,22 @@ } + static void nebSetServerErr(nebServer *server) { - const char **code; - const char **string; - - REQUIRE_SERVER; - - code = soap_faultcode(server->soap); - string = soap_faultstring(server->soap); + REQUIRE_SERVER; + + const char **code = soap_faultcode(server->soap); + const char **string = soap_faultstring(server->soap); nebSetErr(server, "%s - %s", *code, *string); } + #if NEBCOPYFILE static off_t nebCopyFile(nebServer *server, const char *source, const char *dest) { - int sourceFH; - int destFH; - off_t bytesCopied; - - REQUIRE_SERVER; - - sourceFH = open(source, O_RDONLY); + REQUIRE_SERVER; + + int sourceFH = open(source, O_RDONLY); if (sourceFH == -1) { nebSetErr(server, "can not open %s: %s", source, strerror(errno)); @@ -943,5 +1026,5 @@ } - destFH = open(dest, O_WRONLY|O_CREAT|O_CREAT, 0600); + int destFH = open(dest, O_WRONLY|O_CREAT|O_CREAT, 0600); if (destFH == -1) { nebSetErr(server, "can not open %s: %s", dest, strerror(errno)); @@ -950,5 +1033,5 @@ } - bytesCopied = nebCopyFilehandle(server, sourceFH, destFH); + off_t bytesCopied = nebCopyFilehandle(server, sourceFH, destFH); if (bytesCopied < 0) { return -1; @@ -971,14 +1054,10 @@ #endif + static off_t nebCopyFilehandle(nebServer *server, int sourceFH, int destFH) { - off_t bytesTotal; - off_t bytesRemaining; - off_t writeSize; - char writeBuf[64 * 1024]; + REQUIRE_SERVER; + struct stat sourceStat; - - REQUIRE_SERVER; - if(fstat(sourceFH, &sourceStat)) { nebSetErr(server, "can not stat filehandles: %s", strerror(errno)); @@ -988,5 +1067,5 @@ // the size of the file to copied - bytesTotal = sourceStat.st_size; + off_t bytesTotal = sourceStat.st_size; /* @@ -996,7 +1075,9 @@ */ - bytesRemaining = bytesTotal; + off_t bytesRemaining = bytesTotal; while (bytesRemaining) { + off_t writeSize; + char writeBuf[64 * 1024]; if (bytesRemaining <= sizeof(writeBuf)) { writeSize = bytesRemaining; @@ -1023,15 +1104,7 @@ } + static size_t nebParseURI(nebServer *server, const char *URI, char **filename) { - regex_t myregex; - regmatch_t mymatch[2]; - char errbuf[ERRBUF_SIZE]; - char *pattern = "^file://(.*)$"; - int status; - size_t matchStart, matchEnd; - size_t matchLength; - size_t filename_size; - REQUIRE_SERVER; @@ -1048,5 +1121,10 @@ } - status = regcomp(&myregex, pattern, REG_EXTENDED); + regex_t myregex; + regmatch_t mymatch[2]; + char *pattern = "^file://(.*)$"; + char errbuf[ERRBUF_SIZE]; + + int status = regcomp(&myregex, pattern, REG_EXTENDED); if (status != 0) { regerror(status, &myregex, errbuf, ERRBUF_SIZE); @@ -1066,7 +1144,7 @@ regfree(&myregex); - matchStart = (size_t)mymatch[1].rm_so; - matchEnd = (size_t)mymatch[1].rm_eo; - matchLength = matchEnd - matchStart; + size_t matchStart = (size_t)mymatch[1].rm_so; + size_t matchEnd = (size_t)mymatch[1].rm_eo; + size_t matchLength = matchEnd - matchStart; if (matchStart == -1) { @@ -1074,5 +1152,5 @@ } - filename_size = matchLength + 1; + size_t filename_size = matchLength + 1; *filename = xmalloc(filename_size); @@ -1081,4 +1159,5 @@ return strlen(*filename); } + static bool nebNukeFile(nebServer *server, const char *filename) @@ -1100,4 +1179,5 @@ return true; } + static char *nullstrncpy(char *dest, const char *src, size_t n) { @@ -1107,4 +1187,8 @@ } + if (!dest) { + dest = xmalloc(n + 1); + } + return strncpy(dest, src, n); } Index: branches/eam_branches/20090715/Nebulous/nebclient/src/nebclient.h =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/nebclient.h (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/nebclient.h (revision 25022) @@ -38,11 +38,12 @@ typedef struct { - char so_id[256]; ///< storage object ID - char ext_id[256]; ///< storage object key (name) - char read_lock[256]; ///< read lock value + long so_id; ///< storage object ID + long ext_id; ///< storage object key (name) + int read_lock; ///< read lock value char write_lock[256]; ///< write lock value char epoch[256]; ///< creation time stamp char mtime[256]; ///< modification time stamp - int instances; ///< number of instances + int available; ///< number of available instances + int instances; ///< total number of instances } nebObjectStat; @@ -295,5 +296,12 @@ nebObjectStat *nebStat( nebServer *server, ///< nebServer object - const char *key /// storage object key (name) + const char *key ///< storage object key (name) +); + + +/** Deallocates a stat object + */ +void nebObjectStatFree( + nebObjectStat *stat ///< nebObjectStat object ); @@ -308,6 +316,27 @@ int nebChmod( nebServer *server, ///< nebServer object - const char *key, /// storage object key (name) - mode_t mode /// chmod(2) compatible mode (mode_t) + const char *key, ///< storage object key (name) + mode_t mode ///< chmod(2) compatible mode (mode_t) +); + +/** Removes all of the inaccessible instances from an object + * + * @return the number of inaccessible instances removed + */ + +int nebPrune( + nebServer *server, ///< nebServer object + const char *key ///< storage object key (name) +); + +/** Removes all but one instances of an object + * + * @return the number of available instances removed + */ + +int nebThereCanBeOnlyOne( + nebServer *server, ///< nebServer object + const char *key, ///< storage object key (name) + const char *volume ///< leave remain instance on this volume ); Index: branches/eam_branches/20090715/Nebulous/nebclient/src/nebulous.h =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/nebulous.h (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/nebulous.h (revision 25022) @@ -1,5 +1,5 @@ /* src/nebulous.h Generated by wsdl2h 1.2.12 from nebulous.wsdl and typemap.dat - 2009-06-01 23:55:50 GMT + 2009-07-25 01:30:29 GMT Copyright (C) 2001-2008 Robert van Engelen, Genivia Inc. All Rights Reserved. This part of the software is released under one of the following licenses: @@ -120,4 +120,5 @@ - @ref ns1__stat_USCOREobject - @ref ns1__chmod_USCOREobject + - @ref ns1__prune_USCOREobject @section SOAP_ports Endpoints of Binding "SOAP" @@ -947,3 +948,53 @@ ); +/******************************************************************************\ + * * + * ns1__prune_USCOREobject * + * * +\******************************************************************************/ + + +/// Operation "ns1__prune_USCOREobject" of service binding "SOAP" + +/** + +Operation details: + + - SOAP RPC encodingStyle="http://schemas.xmlsoap.org/soap/encoding/" + - SOAP action="urn:Nebulous/Server/SOAP#prune_object" + +C stub function (defined in soapClient.c[pp] generated by soapcpp2): +@code + int soap_call_ns1__prune_USCOREobject( + struct soap *soap, + NULL, // char *endpoint = NULL selects default endpoint for this operation + NULL, // char *action = NULL selects default action for this operation + // request parameters: + char* key, + // response parameters: + int *result + ); +@endcode + +C server function (called from the service dispatcher defined in soapServer.c[pp]): +@code + int ns1__prune_USCOREobject( + struct soap *soap, + // request parameters: + char* key, + // response parameters: + int *result + ); +@endcode + +*/ + +//gsoap ns1 service method-style: prune_USCOREobject rpc +//gsoap ns1 service method-encoding: prune_USCOREobject http://schemas.xmlsoap.org/soap/encoding/ +//gsoap ns1 service method-action: prune_USCOREobject urn:Nebulous/Server/SOAP#prune_object +int ns1__prune_USCOREobject( + char* key, ///< Request parameter + int *result ///< Response parameter +); + /* End of src/nebulous.h */ Index: branches/eam_branches/20090715/Nebulous/nebclient/src/soapC.c =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/soapC.c (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/soapC.c (revision 25022) @@ -12,5 +12,5 @@ #endif -SOAP_SOURCE_STAMP("@(#) soapC.c ver 2.7.12 2009-06-01 23:55:50 GMT") +SOAP_SOURCE_STAMP("@(#) soapC.c ver 2.7.12 2009-07-25 01:30:29 GMT") @@ -161,4 +161,8 @@ case SOAP_TYPE_int: return soap_in_int(soap, NULL, NULL, "xsd:int"); + case SOAP_TYPE_ns1__prune_USCOREobject: + return soap_in_ns1__prune_USCOREobject(soap, NULL, NULL, "ns1:prune_object"); + case SOAP_TYPE_ns1__prune_USCOREobjectResponse: + return soap_in_ns1__prune_USCOREobjectResponse(soap, NULL, NULL, "ns1:prune_objectResponse"); case SOAP_TYPE_ns1__chmod_USCOREobject: return soap_in_ns1__chmod_USCOREobject(soap, NULL, NULL, "ns1:chmod_object"); @@ -254,4 +258,12 @@ return soap_in_int(soap, NULL, NULL, NULL); } + if (!soap_match_tag(soap, t, "ns1:prune_object")) + { *type = SOAP_TYPE_ns1__prune_USCOREobject; + return soap_in_ns1__prune_USCOREobject(soap, NULL, NULL, NULL); + } + if (!soap_match_tag(soap, t, "ns1:prune_objectResponse")) + { *type = SOAP_TYPE_ns1__prune_USCOREobjectResponse; + return soap_in_ns1__prune_USCOREobjectResponse(soap, NULL, NULL, NULL); + } if (!soap_match_tag(soap, t, "ns1:chmod_object")) { *type = SOAP_TYPE_ns1__chmod_USCOREobject; @@ -452,4 +464,8 @@ case SOAP_TYPE_int: return soap_out_int(soap, tag, id, (const int *)ptr, "xsd:int"); + case SOAP_TYPE_ns1__prune_USCOREobject: + return soap_out_ns1__prune_USCOREobject(soap, tag, id, (const struct ns1__prune_USCOREobject *)ptr, "ns1:prune_object"); + case SOAP_TYPE_ns1__prune_USCOREobjectResponse: + return soap_out_ns1__prune_USCOREobjectResponse(soap, tag, id, (const struct ns1__prune_USCOREobjectResponse *)ptr, "ns1:prune_objectResponse"); case SOAP_TYPE_ns1__chmod_USCOREobject: return soap_out_ns1__chmod_USCOREobject(soap, tag, id, (const struct ns1__chmod_USCOREobject *)ptr, "ns1:chmod_object"); @@ -543,4 +559,10 @@ switch (type) { + case SOAP_TYPE_ns1__prune_USCOREobject: + soap_serialize_ns1__prune_USCOREobject(soap, (const struct ns1__prune_USCOREobject *)ptr); + break; + case SOAP_TYPE_ns1__prune_USCOREobjectResponse: + soap_serialize_ns1__prune_USCOREobjectResponse(soap, (const struct ns1__prune_USCOREobjectResponse *)ptr); + break; case SOAP_TYPE_ns1__chmod_USCOREobject: soap_serialize_ns1__chmod_USCOREobject(soap, (const struct ns1__chmod_USCOREobject *)ptr); @@ -1217,4 +1239,150 @@ #endif +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__prune_USCOREobject(struct soap *soap, struct ns1__prune_USCOREobject *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + soap_default_string(soap, &a->key); +} + +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__prune_USCOREobject(struct soap *soap, const struct ns1__prune_USCOREobject *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + soap_serialize_string(soap, &a->key); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__prune_USCOREobject(struct soap *soap, const struct ns1__prune_USCOREobject *a, const char *tag, const char *type) +{ + register int id = soap_embed(soap, (void*)a, NULL, 0, tag, SOAP_TYPE_ns1__prune_USCOREobject); + if (soap_out_ns1__prune_USCOREobject(soap, tag, id, a, type)) + return soap->error; + return soap_putindependent(soap); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__prune_USCOREobject(struct soap *soap, const char *tag, int id, const struct ns1__prune_USCOREobject *a, const char *type) +{ + if (soap_element_begin_out(soap, tag, soap_embedded_id(soap, id, a, SOAP_TYPE_ns1__prune_USCOREobject), type)) + return soap->error; + if (soap_out_string(soap, "key", -1, &a->key, "")) + return soap->error; + return soap_element_end_out(soap, tag); +} + +SOAP_FMAC3 struct ns1__prune_USCOREobject * SOAP_FMAC4 soap_get_ns1__prune_USCOREobject(struct soap *soap, struct ns1__prune_USCOREobject *p, const char *tag, const char *type) +{ + if ((p = soap_in_ns1__prune_USCOREobject(soap, tag, p, type))) + if (soap_getindependent(soap)) + return NULL; + return p; +} + +SOAP_FMAC3 struct ns1__prune_USCOREobject * SOAP_FMAC4 soap_in_ns1__prune_USCOREobject(struct soap *soap, const char *tag, struct ns1__prune_USCOREobject *a, const char *type) +{ + size_t soap_flag_key = 1; + if (soap_element_begin_in(soap, tag, 0, type)) + return NULL; + a = (struct ns1__prune_USCOREobject *)soap_id_enter(soap, soap->id, a, SOAP_TYPE_ns1__prune_USCOREobject, sizeof(struct ns1__prune_USCOREobject), 0, NULL, NULL, NULL); + if (!a) + return NULL; + soap_default_ns1__prune_USCOREobject(soap, a); + if (soap->body && !*soap->href) + { + for (;;) + { soap->error = SOAP_TAG_MISMATCH; + if (soap_flag_key && (soap->error == SOAP_TAG_MISMATCH || soap->error == SOAP_NO_TAG)) + if (soap_in_string(soap, "key", &a->key, "xsd:string")) + { soap_flag_key--; + continue; + } + if (soap->error == SOAP_TAG_MISMATCH) + soap->error = soap_ignore_element(soap); + if (soap->error == SOAP_NO_TAG) + break; + if (soap->error) + return NULL; + } + if (soap_element_end_in(soap, tag)) + return NULL; + } + else + { a = (struct ns1__prune_USCOREobject *)soap_id_forward(soap, soap->href, (void*)a, 0, SOAP_TYPE_ns1__prune_USCOREobject, 0, sizeof(struct ns1__prune_USCOREobject), 0, NULL); + if (soap->body && soap_element_end_in(soap, tag)) + return NULL; + } + return a; +} + +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__prune_USCOREobjectResponse(struct soap *soap, struct ns1__prune_USCOREobjectResponse *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + a->result = NULL; +} + +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__prune_USCOREobjectResponse(struct soap *soap, const struct ns1__prune_USCOREobjectResponse *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + soap_serialize_PointerToint(soap, &a->result); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__prune_USCOREobjectResponse(struct soap *soap, const struct ns1__prune_USCOREobjectResponse *a, const char *tag, const char *type) +{ + register int id = soap_embed(soap, (void*)a, NULL, 0, tag, SOAP_TYPE_ns1__prune_USCOREobjectResponse); + if (soap_out_ns1__prune_USCOREobjectResponse(soap, tag, id, a, type)) + return soap->error; + return soap_putindependent(soap); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__prune_USCOREobjectResponse(struct soap *soap, const char *tag, int id, const struct ns1__prune_USCOREobjectResponse *a, const char *type) +{ + if (soap_element_begin_out(soap, tag, soap_embedded_id(soap, id, a, SOAP_TYPE_ns1__prune_USCOREobjectResponse), type)) + return soap->error; + if (soap_out_PointerToint(soap, "result", -1, &a->result, "")) + return soap->error; + return soap_element_end_out(soap, tag); +} + +SOAP_FMAC3 struct ns1__prune_USCOREobjectResponse * SOAP_FMAC4 soap_get_ns1__prune_USCOREobjectResponse(struct soap *soap, struct ns1__prune_USCOREobjectResponse *p, const char *tag, const char *type) +{ + if ((p = soap_in_ns1__prune_USCOREobjectResponse(soap, tag, p, type))) + if (soap_getindependent(soap)) + return NULL; + return p; +} + +SOAP_FMAC3 struct ns1__prune_USCOREobjectResponse * SOAP_FMAC4 soap_in_ns1__prune_USCOREobjectResponse(struct soap *soap, const char *tag, struct ns1__prune_USCOREobjectResponse *a, const char *type) +{ + size_t soap_flag_result = 1; + if (soap_element_begin_in(soap, tag, 0, type)) + return NULL; + a = (struct ns1__prune_USCOREobjectResponse *)soap_id_enter(soap, soap->id, a, SOAP_TYPE_ns1__prune_USCOREobjectResponse, sizeof(struct ns1__prune_USCOREobjectResponse), 0, NULL, NULL, NULL); + if (!a) + return NULL; + soap_default_ns1__prune_USCOREobjectResponse(soap, a); + if (soap->body && !*soap->href) + { + for (;;) + { soap->error = SOAP_TAG_MISMATCH; + if (soap_flag_result && soap->error == SOAP_TAG_MISMATCH) + if (soap_in_PointerToint(soap, "result", &a->result, "xsd:int")) + { soap_flag_result--; + continue; + } + if (soap->error == SOAP_TAG_MISMATCH) + soap->error = soap_ignore_element(soap); + if (soap->error == SOAP_NO_TAG) + break; + if (soap->error) + return NULL; + } + if (soap_element_end_in(soap, tag)) + return NULL; + } + else + { a = (struct ns1__prune_USCOREobjectResponse *)soap_id_forward(soap, soap->href, (void*)a, 0, SOAP_TYPE_ns1__prune_USCOREobjectResponse, 0, sizeof(struct ns1__prune_USCOREobjectResponse), 0, NULL); + if (soap->body && soap_element_end_in(soap, tag)) + return NULL; + } + return a; +} + SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__chmod_USCOREobject(struct soap *soap, struct ns1__chmod_USCOREobject *a) { Index: branches/eam_branches/20090715/Nebulous/nebclient/src/soapClient.c =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/soapClient.c (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/soapClient.c (revision 25022) @@ -10,5 +10,5 @@ #endif -SOAP_SOURCE_STAMP("@(#) soapClient.c ver 2.7.12 2009-06-01 23:55:50 GMT") +SOAP_SOURCE_STAMP("@(#) soapClient.c ver 2.7.12 2009-07-25 01:30:29 GMT") @@ -870,4 +870,61 @@ } +SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__prune_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, int *result) +{ struct ns1__prune_USCOREobject soap_tmp_ns1__prune_USCOREobject; + struct ns1__prune_USCOREobjectResponse *soap_tmp_ns1__prune_USCOREobjectResponse; + if (!soap_endpoint) + soap_endpoint = "http://localhost:80/nebulous"; + if (!soap_action) + soap_action = "urn:Nebulous/Server/SOAP#prune_object"; + soap->encodingStyle = "http://schemas.xmlsoap.org/soap/encoding/"; + soap_tmp_ns1__prune_USCOREobject.key = key; + soap_begin(soap); + soap_serializeheader(soap); + soap_serialize_ns1__prune_USCOREobject(soap, &soap_tmp_ns1__prune_USCOREobject); + if (soap_begin_count(soap)) + return soap->error; + if (soap->mode & SOAP_IO_LENGTH) + { if (soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__prune_USCOREobject(soap, &soap_tmp_ns1__prune_USCOREobject, "ns1:prune_object", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap)) + return soap->error; + } + if (soap_end_count(soap)) + return soap->error; + if (soap_connect(soap, soap_endpoint, soap_action) + || soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__prune_USCOREobject(soap, &soap_tmp_ns1__prune_USCOREobject, "ns1:prune_object", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap) + || soap_end_send(soap)) + return soap_closesock(soap); + if (!result) + return soap_closesock(soap); + soap_default_int(soap, result); + if (soap_begin_recv(soap) + || soap_envelope_begin_in(soap) + || soap_recv_header(soap) + || soap_body_begin_in(soap)) + return soap_closesock(soap); + soap_tmp_ns1__prune_USCOREobjectResponse = soap_get_ns1__prune_USCOREobjectResponse(soap, NULL, "ns1:prune_objectResponse", ""); + if (soap->error) + { if (soap->error == SOAP_TAG_MISMATCH && soap->level == 2) + return soap_recv_fault(soap); + return soap_closesock(soap); + } + if (soap_body_end_in(soap) + || soap_envelope_end_in(soap) + || soap_end_recv(soap)) + return soap_closesock(soap); + if (result && soap_tmp_ns1__prune_USCOREobjectResponse->result) + *result = *soap_tmp_ns1__prune_USCOREobjectResponse->result; + return soap_closesock(soap); +} + #ifdef __cplusplus } Index: branches/eam_branches/20090715/Nebulous/nebclient/src/soapH.h =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/soapH.h (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/soapH.h (revision 25022) @@ -42,5 +42,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Fault -#define SOAP_TYPE_SOAP_ENV__Fault (67) +#define SOAP_TYPE_SOAP_ENV__Fault (70) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Fault(struct soap*, struct SOAP_ENV__Fault *); @@ -56,5 +56,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Reason -#define SOAP_TYPE_SOAP_ENV__Reason (66) +#define SOAP_TYPE_SOAP_ENV__Reason (69) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Reason(struct soap*, struct SOAP_ENV__Reason *); @@ -70,5 +70,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Detail -#define SOAP_TYPE_SOAP_ENV__Detail (63) +#define SOAP_TYPE_SOAP_ENV__Detail (66) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Detail(struct soap*, struct SOAP_ENV__Detail *); @@ -84,5 +84,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Code -#define SOAP_TYPE_SOAP_ENV__Code (61) +#define SOAP_TYPE_SOAP_ENV__Code (64) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Code(struct soap*, struct SOAP_ENV__Code *); @@ -98,5 +98,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Header -#define SOAP_TYPE_SOAP_ENV__Header (60) +#define SOAP_TYPE_SOAP_ENV__Header (63) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Header(struct soap*, struct SOAP_ENV__Header *); @@ -109,4 +109,24 @@ #endif +#ifndef SOAP_TYPE_ns1__prune_USCOREobject +#define SOAP_TYPE_ns1__prune_USCOREobject (62) +#endif +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__prune_USCOREobject(struct soap*, struct ns1__prune_USCOREobject *); +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__prune_USCOREobject(struct soap*, const struct ns1__prune_USCOREobject *); +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__prune_USCOREobject(struct soap*, const struct ns1__prune_USCOREobject *, const char*, const char*); +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__prune_USCOREobject(struct soap*, const char*, int, const struct ns1__prune_USCOREobject *, const char*); +SOAP_FMAC3 struct ns1__prune_USCOREobject * SOAP_FMAC4 soap_get_ns1__prune_USCOREobject(struct soap*, struct ns1__prune_USCOREobject *, const char*, const char*); +SOAP_FMAC3 struct ns1__prune_USCOREobject * SOAP_FMAC4 soap_in_ns1__prune_USCOREobject(struct soap*, const char*, struct ns1__prune_USCOREobject *, const char*); + +#ifndef SOAP_TYPE_ns1__prune_USCOREobjectResponse +#define SOAP_TYPE_ns1__prune_USCOREobjectResponse (61) +#endif +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__prune_USCOREobjectResponse(struct soap*, struct ns1__prune_USCOREobjectResponse *); +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__prune_USCOREobjectResponse(struct soap*, const struct ns1__prune_USCOREobjectResponse *); +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__prune_USCOREobjectResponse(struct soap*, const struct ns1__prune_USCOREobjectResponse *, const char*, const char*); +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__prune_USCOREobjectResponse(struct soap*, const char*, int, const struct ns1__prune_USCOREobjectResponse *, const char*); +SOAP_FMAC3 struct ns1__prune_USCOREobjectResponse * SOAP_FMAC4 soap_get_ns1__prune_USCOREobjectResponse(struct soap*, struct ns1__prune_USCOREobjectResponse *, const char*, const char*); +SOAP_FMAC3 struct ns1__prune_USCOREobjectResponse * SOAP_FMAC4 soap_in_ns1__prune_USCOREobjectResponse(struct soap*, const char*, struct ns1__prune_USCOREobjectResponse *, const char*); + #ifndef SOAP_TYPE_ns1__chmod_USCOREobject #define SOAP_TYPE_ns1__chmod_USCOREobject (59) @@ -422,5 +442,5 @@ #ifndef SOAP_TYPE_PointerToSOAP_ENV__Reason -#define SOAP_TYPE_PointerToSOAP_ENV__Reason (69) +#define SOAP_TYPE_PointerToSOAP_ENV__Reason (72) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_PointerToSOAP_ENV__Reason(struct soap*, struct SOAP_ENV__Reason *const*); @@ -435,5 +455,5 @@ #ifndef SOAP_TYPE_PointerToSOAP_ENV__Detail -#define SOAP_TYPE_PointerToSOAP_ENV__Detail (68) +#define SOAP_TYPE_PointerToSOAP_ENV__Detail (71) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_PointerToSOAP_ENV__Detail(struct soap*, struct SOAP_ENV__Detail *const*); @@ -448,5 +468,5 @@ #ifndef SOAP_TYPE_PointerToSOAP_ENV__Code -#define SOAP_TYPE_PointerToSOAP_ENV__Code (62) +#define SOAP_TYPE_PointerToSOAP_ENV__Code (65) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_PointerToSOAP_ENV__Code(struct soap*, struct SOAP_ENV__Code *const*); Index: branches/eam_branches/20090715/Nebulous/nebclient/src/soapServer.c =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/soapServer.c (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/soapServer.c (revision 25022) @@ -10,5 +10,5 @@ #endif -SOAP_SOURCE_STAMP("@(#) soapServer.c ver 2.7.12 2009-06-01 23:55:50 GMT") +SOAP_SOURCE_STAMP("@(#) soapServer.c ver 2.7.12 2009-07-25 01:30:29 GMT") @@ -107,4 +107,6 @@ if (!soap_match_tag(soap, soap->tag, "ns1:chmod_object")) return soap_serve_ns1__chmod_USCOREobject(soap); + if (!soap_match_tag(soap, soap->tag, "ns1:prune_object")) + return soap_serve_ns1__prune_USCOREobject(soap); return soap->error = SOAP_NO_METHOD; } @@ -759,4 +761,48 @@ } +SOAP_FMAC5 int SOAP_FMAC6 soap_serve_ns1__prune_USCOREobject(struct soap *soap) +{ struct ns1__prune_USCOREobject soap_tmp_ns1__prune_USCOREobject; + struct ns1__prune_USCOREobjectResponse soap_tmp_ns1__prune_USCOREobjectResponse; + int soap_tmp_int; + soap_default_ns1__prune_USCOREobjectResponse(soap, &soap_tmp_ns1__prune_USCOREobjectResponse); + soap_default_int(soap, &soap_tmp_int); + soap_tmp_ns1__prune_USCOREobjectResponse.result = &soap_tmp_int; + soap_default_ns1__prune_USCOREobject(soap, &soap_tmp_ns1__prune_USCOREobject); + soap->encodingStyle = "http://schemas.xmlsoap.org/soap/encoding/"; + if (!soap_get_ns1__prune_USCOREobject(soap, &soap_tmp_ns1__prune_USCOREobject, "ns1:prune_object", NULL)) + return soap->error; + if (soap_body_end_in(soap) + || soap_envelope_end_in(soap) + || soap_end_recv(soap)) + return soap->error; + soap->error = ns1__prune_USCOREobject(soap, soap_tmp_ns1__prune_USCOREobject.key, &soap_tmp_int); + if (soap->error) + return soap->error; + soap_serializeheader(soap); + soap_serialize_ns1__prune_USCOREobjectResponse(soap, &soap_tmp_ns1__prune_USCOREobjectResponse); + if (soap_begin_count(soap)) + return soap->error; + if (soap->mode & SOAP_IO_LENGTH) + { if (soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__prune_USCOREobjectResponse(soap, &soap_tmp_ns1__prune_USCOREobjectResponse, "ns1:prune_objectResponse", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap)) + return soap->error; + }; + if (soap_end_count(soap) + || soap_response(soap, SOAP_OK) + || soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__prune_USCOREobjectResponse(soap, &soap_tmp_ns1__prune_USCOREobjectResponse, "ns1:prune_objectResponse", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap) + || soap_end_send(soap)) + return soap->error; + return soap_closesock(soap); +} + #ifdef __cplusplus } Index: branches/eam_branches/20090715/Nebulous/nebclient/src/soapStub.h =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/src/soapStub.h (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/src/soapStub.h (revision 25022) @@ -325,6 +325,24 @@ #endif +#ifndef SOAP_TYPE_ns1__prune_USCOREobjectResponse +#define SOAP_TYPE_ns1__prune_USCOREobjectResponse (61) +/* ns1:prune_objectResponse */ +struct ns1__prune_USCOREobjectResponse +{ + int *result; /* SOAP 1.2 RPC return element (when namespace qualified) */ /* optional element of type xsd:int */ +}; +#endif + +#ifndef SOAP_TYPE_ns1__prune_USCOREobject +#define SOAP_TYPE_ns1__prune_USCOREobject (62) +/* ns1:prune_object */ +struct ns1__prune_USCOREobject +{ + char *key; /* optional element of type xsd:string */ +}; +#endif + #ifndef SOAP_TYPE_SOAP_ENV__Header -#define SOAP_TYPE_SOAP_ENV__Header (60) +#define SOAP_TYPE_SOAP_ENV__Header (63) /* SOAP Header: */ struct SOAP_ENV__Header @@ -337,5 +355,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Code -#define SOAP_TYPE_SOAP_ENV__Code (61) +#define SOAP_TYPE_SOAP_ENV__Code (64) /* SOAP Fault Code: */ struct SOAP_ENV__Code @@ -347,5 +365,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Detail -#define SOAP_TYPE_SOAP_ENV__Detail (63) +#define SOAP_TYPE_SOAP_ENV__Detail (66) /* SOAP-ENV:Detail */ struct SOAP_ENV__Detail @@ -358,5 +376,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Reason -#define SOAP_TYPE_SOAP_ENV__Reason (66) +#define SOAP_TYPE_SOAP_ENV__Reason (69) /* SOAP-ENV:Reason */ struct SOAP_ENV__Reason @@ -367,5 +385,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Fault -#define SOAP_TYPE_SOAP_ENV__Fault (67) +#define SOAP_TYPE_SOAP_ENV__Fault (70) /* SOAP Fault: */ struct SOAP_ENV__Fault @@ -458,4 +476,6 @@ SOAP_FMAC5 int SOAP_FMAC6 ns1__chmod_USCOREobject(struct soap*, char *key, int mode, int *result); +SOAP_FMAC5 int SOAP_FMAC6 ns1__prune_USCOREobject(struct soap*, char *key, int *result); + /******************************************************************************\ * * @@ -495,4 +515,6 @@ SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__chmod_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, int mode, int *result); +SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__prune_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, int *result); + /******************************************************************************\ * * @@ -534,4 +556,6 @@ SOAP_FMAC5 int SOAP_FMAC6 soap_serve_ns1__chmod_USCOREobject(struct soap*); + +SOAP_FMAC5 int SOAP_FMAC6 soap_serve_ns1__prune_USCOREobject(struct soap*); #ifdef __cplusplus Index: branches/eam_branches/20090715/Nebulous/nebclient/tests/tests.c =================================================================== --- branches/eam_branches/20090715/Nebulous/nebclient/tests/tests.c (revision 25020) +++ branches/eam_branches/20090715/Nebulous/nebclient/tests/tests.c (revision 25022) @@ -20,5 +20,5 @@ char *key = "foobarbaz"; - plan_tests(29); + plan_tests(31); if (getenv("NEB_SERVER")) { @@ -98,10 +98,9 @@ char *copies = nebGetXattr(server, key, "user.copies"); - neb_ok(server, strcmp(copies, "2") == 0, "get user.copies xattr"); + neb_ok(server, copies != NULL && strcmp(copies, "2") == 0, "get user.copies xattr"); char **xattrs = NULL; int n_xattrs = nebListXattr(server, key, &xattrs); - printf("# %s\n\n", xattrs[0]); - neb_ok(server, n_xattrs == 1, "count of xattrs"); - neb_ok(server, strcmp(xattrs[0], "user.copies") == 0, "user.copies xattr exists"); + neb_ok(server, xattrs != NULL && n_xattrs == 1, "count of xattrs"); + neb_ok(server, xattrs != NULL && strcmp(xattrs[0], "user.copies") == 0, "user.copies xattr exists"); neb_ok(server, nebRemoveXattr(server, key, "user.copies"), "remove user.copies xattr"); @@ -138,4 +137,10 @@ neb_ok(server, nebChmod(server, "movedfile", 0440) == 0, "chmod object"); + // no dead instances to remove + neb_ok(server, nebPrune(server, "movedfile") == 0, "prune object"); + + nebReplicate(server, key, NULL, NULL); + neb_ok(server, nebThereCanBeOnlyOne(server, "movedfile", NULL) == 0, "reduce instances to only 1"); + if (!nebDelete(server, "movedfile")) { diag( "cleanup failed %s\n", nebErr(server)); Index: branches/eam_branches/20090715/Nebulous/t/69_client_prune.t =================================================================== --- branches/eam_branches/20090715/Nebulous/t/69_client_prune.t (revision 25022) +++ branches/eam_branches/20090715/Nebulous/t/69_client_prune.t (revision 25022) @@ -0,0 +1,72 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2009 Joshua Hoblitt + +use strict; +use warnings FATAL => qw( all ); + +use Apache::Test qw( -withtestmore ); + +plan tests => 5; + +use lib qw( ./t ./lib ); + + +use Nebulous::Client; +use Nebulous::Util qw( :standard ); +use Test::Nebulous; + +my $hostport = Apache::Test->config->{ 'hostport' }; + +my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", +); + +# note: Test::DBUnit::expected_dataset_ok() does not seem to work under +# Apache::Test for some reason +# +use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $uri1 = $neb->create($key, 'node01'); + my $uri2 = $neb->replicate($key, 'node02'); + + # make one of the instances unavailable + my $dbh = test_dbh(); + $dbh->do("UPDATE mountedvol SET available = 0 WHERE name = 'node01'"); + + ok($neb->prune($key), "prune object"); + + is(scalar @{$neb->find_instances($key)}, 1, "# of instances"); +} + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $uri1 = $neb->create($key); + + is($neb->prune($key), 0, "no dead instances to remove"); +} + +Test::Nebulous->setup; + +eval { + $neb->prune(); +}; +like($@, qr/1 was expected/, "no params"); + +Test::Nebulous->setup; + +eval { + my $key = "foo"; + my $uri1 = $neb->create($key); + + $neb->prune("foo", 2); +}; +like($@, qr/1 was expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090715/Nebulous/t/71_client_there_can_be_only_one.t =================================================================== --- branches/eam_branches/20090715/Nebulous/t/71_client_there_can_be_only_one.t (revision 25022) +++ branches/eam_branches/20090715/Nebulous/t/71_client_there_can_be_only_one.t (revision 25022) @@ -0,0 +1,89 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 56_client_find_instances.t,v 1.5 2008-09-10 23:50:26 bills Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Apache::Test qw( -withtestmore ); + +plan tests => 8; + +use lib qw( ./t ./lib ); + +use Nebulous::Client; +use Nebulous::Util qw( :standard ); +use Test::Nebulous; + +my $hostport = Apache::Test->config->{ 'hostport' }; + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + $neb->create($key); + $neb->replicate($key,); + + is($neb->there_can_be_only_one($key), 1, "there can be only one!"); + + my $locations = $neb->find_instances( "foo" ); + + is( scalar @$locations, 1, "found 1" ); +} + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + $neb->create($key, "node01"); + my $uri = $neb->replicate($key, "node02"); + + is($neb->there_can_be_only_one($key, "node02"), 1, "there can be only one!"); + + my $locations = $neb->find_instances( "foo" ); + + is( scalar @$locations, 1, "found 1" ); + is($locations->[0], $uri, "instance on correct volume" ); +} + +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + is( $neb->there_can_be_only_one( "foo" ), undef, "storage object does not exist" ); +} + +Test::Nebulous->setup; + +eval { + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + $neb->there_can_be_only_one(); +}; +like( $@, qr/1 - 2 were expected/, "no params" ); + +Test::Nebulous->setup; + +eval { + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + $neb->there_can_be_only_one( "foo", 'read', 3 ); +}; +like( $@, qr/1 - 2 were expected/, "too many params" ); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090715/Nebulous/t/72_neb-cull.t =================================================================== --- branches/eam_branches/20090715/Nebulous/t/72_neb-cull.t (revision 25022) +++ branches/eam_branches/20090715/Nebulous/t/72_neb-cull.t (revision 25022) @@ -0,0 +1,181 @@ +#!/usr/bin/env perl + +# Copyright (C) 2009 Joshua Hoblitt + +=head1 NAME + +t/72_neb-cull.t - tests neb-cull + +=head1 SYNOPSIS + + prove t/72_neb-cull.t + +=cut + +use strict; +use warnings; + +use Apache::Test qw( -withtestmore ); +plan tests => 29; + +use lib qw( ./lib ./t ); + +use Test::Cmd; +use Nebulous::Client; +use Test::Nebulous; + +my $hostport = Apache::Test->config->{ 'hostport' }; +my $neb_url = "http://$hostport/nebulous"; + + +my $cmd = 'bin/neb-cull'; + +# last ditch effort to make sure $cmd executable +chmod 0755, $cmd; + +my $test = Test::Cmd->new(prog => $cmd, workdir => ''); +isa_ok($test, 'Test::Cmd'); + +# NEB_SERVER env var not set +undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'}; +Test::Nebulous->setup; + +{ + $test->run(args => ''); + missing_args(2, "Required options: --server"); +} + +# NEB_SERVER set +Test::Nebulous->setup; + +{ + $ENV{NEB_SERVER} = $neb_url; + + $test->run(args => ''); + missing_args(2, "missing key operand"); +} + +Test::Nebulous->setup; + +{ + my $key = 'foo'; + + my $neb = Nebulous::Client->new( + proxy => $neb_url, + ); + $neb->create($key); + + $test->run(args => $key); + + is($neb->stat($key)->[6], 1, "correct # of instances"); + is($? >> 8, 255, "exit code"); + like($test->stdout, qr/^$/, "stdout"); + like($test->stderr, qr/not enough instances/, "stderr"); +} + +Test::Nebulous->setup; + +{ + my $key = 'foo'; + + my $neb = Nebulous::Client->new( + proxy => $neb_url, + ); + $neb->create($key); + $neb->replicate($key); + + $test->run(args => $key); + + is($neb->stat($key)->[6], 1, "correct # of instances"); + is($? >> 8, 0, "exit code"); + like($test->stdout, qr/^$/, "stdout"); + like($test->stderr, qr/^$/, "stderr"); +} + +Test::Nebulous->setup; + +{ + my $key = 'foo'; + + my $neb = Nebulous::Client->new( + proxy => $neb_url, + ); + $neb->create($key); + $neb->replicate($key); + $neb->replicate($key); + + $test->run(args => $key); + + is($neb->stat($key)->[6], 2, "correct # of instances"); + is($? >> 8, 0, "exit code"); + like($test->stdout, qr/^$/, "stdout"); + like($test->stderr, qr/^$/, "stderr"); +} + +Test::Nebulous->setup; + +{ + my $key = 'foo'; + + my $neb = Nebulous::Client->new( + proxy => $neb_url, + ); + $neb->create($key); + + $test->run(args => "--one_only " . $key); + + is($neb->stat($key)->[6], 1, "correct # of instances"); + is($? >> 8, 255, "exit code"); + like($test->stdout, qr/^$/, "stdout"); + like($test->stderr, qr/not enough instances/, "stderr"); +} + +Test::Nebulous->setup; + +{ + my $key = 'foo'; + + my $neb = Nebulous::Client->new( + proxy => $neb_url, + ); + $neb->create($key); + $neb->replicate($key); + + $test->run(args => "--one_only " . $key); + + is($neb->stat($key)->[6], 1, "correct # of instances"); + is($? >> 8, 0, "exit code"); + like($test->stdout, qr/^$/, "stdout"); + like($test->stderr, qr/^$/, "stderr"); +} + +Test::Nebulous->setup; + +{ + my $key = 'foo'; + + my $neb = Nebulous::Client->new( + proxy => $neb_url, + ); + $neb->create($key); + $neb->replicate($key); + $neb->replicate($key); + + $test->run(args => "--one_only " . $key); + + is($neb->stat($key)->[6], 1, "correct # of instances"); + is($? >> 8, 0, "exit code"); + like($test->stdout, qr/^$/, "stdout"); + like($test->stderr, qr/^$/, "stderr"); +} + + +Test::Nebulous->cleanup; + +sub missing_args +{ + my ($exit, $errstr) = @_; + + is($? >> 8, $exit, "error code is: $exit"); + like($test->stderr, qr/$errstr/, "error string is: $errstr"); +} Index: branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25020) +++ branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 25022) @@ -24,4 +24,5 @@ use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config qw( :standard ); +use Carp; ### my @images = locate_images($image_db, $req_type, $img_type, $id, $component, @@ -121,4 +122,5 @@ my $command; my $id_opt; # option for the lookup + my $image_name; my $mask_name; my $weight_name; @@ -130,5 +132,5 @@ # special class_id value "null" means ignore - if ($component eq "null") { + if ($component and ($component eq "null")) { $component = undef; } @@ -146,4 +148,5 @@ $command .= " -class_id $class_id" if $class_id; $id_opt = "-chip_id"; + $image_name = "PPIMAGE.CHIP"; $mask_name = "PPIMAGE.CHIP.MASK"; $weight_name = "PPIMAGE.CHIP.VARIANCE"; @@ -156,4 +159,5 @@ $command .= " -skycell_id $skycell_id" if $skycell_id; $id_opt = "-warp_id"; + $image_name = "PSWARP.OUTPUT"; $mask_name = "PSWARP.OUTPUT.MASK"; $weight_name = "PSWARP.OUTPUT.VARIANCE"; @@ -164,4 +168,5 @@ $command .= " -skycell_id $skycell_id" if $skycell_id; $id_opt = "-diff_id"; + $image_name = "PPSUB.OUTPUT"; $mask_name = "PPSUB.OUTPUT.MASK"; $weight_name = "PPSUB.OUTPUT.VARIANCE"; @@ -171,4 +176,5 @@ $id_opt = "-stack_id"; + $image_name = "PPSTACK.OUTPUT"; $mask_name = "PPSTACK.OUTPUT.MASK"; $weight_name = "PPSTACK.OUTPUT.VARIANCE"; @@ -210,5 +216,6 @@ my $base; - next if ($img_type eq "warp" and $image->{ignored}); + next if $image->{fault}; + next if ($img_type ne "raw") and $image->{quality}; if ($base_name) { @@ -224,21 +231,20 @@ $ipprc->define_camera($camera); } - my $out = {}; - $out->{exp_id} = $image->{exp_id}; - $out->{exp_name} = $image->{exp_name}; + my $out = $image; + + # if uri is nil this will get overridded below + # (this is for raw stage) $out->{image} = $image->{uri}; - $out->{state} = $image->{state}; # state is undef for rawExp, but that's ok if ($set_class_id) { $class_id = $image->{class_id}; - $out->{class_id} = $class_id; + $out->{component} = $class_id; } # find the mask and weight images if ($base) { + $out->{image} = $ipprc->filename($image_name, $base, $class_id) if $image_name; $out->{mask} = $ipprc->filename($mask_name, $base, $class_id) if $mask_name; $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name; } - $out->{camera} = $camera; - $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom; @@ -432,4 +438,9 @@ } +# cache of results of ppConfigDump +my %astromSources; +my $last_exp_id = 0; +my $lastAstromFile; + # find the astrometry file for a given exposure # return undef if no completed camRun exists @@ -440,4 +451,12 @@ my $verbose = shift; + my $exp_id = $image->{exp_id}; + if (($exp_id eq $last_exp_id) and $lastAstromFile) { + # running camtool 60 times is really expensive when the answer is the same + return $lastAstromFile; + } + $last_exp_id = 0; + $lastAstromFile = undef; + my $missing_tools; my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1); @@ -448,5 +467,5 @@ } - my $command = "$camtool -dbname $image_db -processedexp -exp_id $image->{exp_id}"; + my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id"; # run the tool and parse the output my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -478,6 +497,6 @@ my $camRoot = $camdata->{path_base}; my $camera = $image->{camera}; - my $astromSource; - { + my $astromSource = $astromSources{$camera}; + if (!$astromSource) { my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -490,7 +509,12 @@ die("Unable to parse metadata config doc"); $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + $astromSources{$camera} = $astromSource; } my $astromFile = $ipprc->filename($astromSource, $camRoot); + if ($astromFile) { + $lastAstromFile = $astromFile; + $last_exp_id = $exp_id; + } return $astromFile; @@ -538,9 +562,20 @@ # resolve_project() # get project specific information +my $last_project = ""; sub resolve_project { my $ipprc = shift; my $project_name = shift; + + if (!$project_name) { + carp ("project is not defined"); + return undef; + } + + if ($project_name eq $last_project) { + return $last_project; + } + my $dbname = shift; - die "project is not defined" if !$project_name; + my $dbserver = shift; my $verbose = 0; @@ -555,4 +590,5 @@ my $command = "$pstamptool -project -name $project_name"; $command .= " -dbname $dbname" if defined $dbname; + $command .= " -dbserver $dbserver" if defined $dbserver; # run the tool and parse the output my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -572,5 +608,7 @@ my $proj_hash = parse_md_fast($mdcParser, $output); - return $proj_hash->[0]; + $last_project = $proj_hash->[0]; + + return $last_project; } 1; Index: branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 25020) +++ branches/eam_branches/20090715/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 25022) @@ -26,4 +26,5 @@ +# these values need to match the ones in pstamp/src/pstamp.h our $PSTAMP_CENTER_IN_PIXELS = 1; our $PSTAMP_RANGE_IN_PIXELS = 2; Index: branches/eam_branches/20090715/archive/ducttape/fix_compress.c =================================================================== --- branches/eam_branches/20090715/archive/ducttape/fix_compress.c (revision 25022) +++ branches/eam_branches/20090715/archive/ducttape/fix_compress.c (revision 25022) @@ -0,0 +1,117 @@ +// Program to fix bug in FITS images written with psLib prior to trunk@r25002. For these images, funpack +// won't put a single compressed image back in the PHU because they are missing ZSIMPLE. This program simply +// reads the FITS file and writes it back out the way it came in, but since psLib is fixed, the output should +// behave well with funpack. +#include +#include +#include + +int main(int argc, char *argv[]) +{ + if (argc != 3) { + fprintf(stderr, "Insufficient arguments. Please supply input FITS file and output FITS file."); + return PS_EXIT_CONFIG_ERROR; + } + const char *inName = argv[1]; + const char *outName = argv[2]; + + if (strcmp(inName, outName) == 0) { + fprintf(stderr, "Can't overwrite input image."); + return PS_EXIT_CONFIG_ERROR; + } + + psFits *inFits = psFitsOpen(inName, "r"); + psFits *outFits = psFitsOpen(outName, "w"); + outFits->options = psFitsOptionsAlloc(); + + int numExt = psFitsGetSize(inFits); + for (int i = 0; i < numExt; i++) { + if (!psFitsMoveExtNum(inFits, i, false)) { + psErrorStackPrint(stderr, "Can't move to extension %d\n", i); + return PS_EXIT_SYS_ERROR; + } + + if (i == 0 && psFitsCheckCompressedImagePHU(inFits, NULL)) { + // We're now positioned on the image + i++; + } + + psMetadata *header = psFitsReadHeader(NULL, inFits); + psString extname = psFitsGetExtName(inFits); + psFitsType type = psFitsGetExtType(inFits); + switch (type) { + case PS_FITS_TYPE_IMAGE: { + int naxis = psMetadataLookupS32(NULL, header, "NAXIS"); + if (naxis == 0) { + if (!psFitsWriteBlank(outFits, header, extname)) { + psErrorStackPrint(stderr, "Can't write blank extension %d\n", i); + return PS_EXIT_SYS_ERROR; + } + } else { + psString compType = psMetadataLookupStr(NULL, header, "ZCMPTYPE"); + if (compType) { + psVector *tile = psVectorAlloc(3, PS_TYPE_S32); + psVectorInit(tile, 0); + for (int i = 0; i < naxis; i++) { + psString name = NULL; + psStringAppend(&name, "ZTILE%d", i + 1); + tile->data.S32[i] = psMetadataLookupS32(NULL, header, name); + psFree(name); + } + psFitsCompressionType type = psFitsCompressionTypeFromString(compType); + psFitsSetCompression(outFits, type, tile, 8, 0, 0); + psFree(tile); + } else { + psFitsSetCompression(outFits, PS_FITS_COMPRESS_NONE, NULL, 8, 0, 0); + } + + outFits->options->bitpix = psMetadataLookupS32(NULL, header, "BITPIX"); + outFits->options->bscale = psMetadataLookupF64(NULL, header, "BSCALE"); + outFits->options->bzero = psMetadataLookupF64(NULL, header, "BZERO"); + if (isfinite(outFits->options->bscale) && isfinite(outFits->options->bzero)) { + outFits->options->scaling = PS_FITS_SCALE_MANUAL; + } else { + outFits->options->scaling = PS_FITS_SCALE_NONE; + } + outFits->options->fuzz = false; + + psArray *cube = psFitsReadImageCube(inFits, psRegionSet(0,0,0,0)); + if (!cube) { + psErrorStackPrint(stderr, "Can't read image(s) from extension %d\n", i); + return PS_EXIT_SYS_ERROR; + } + if (!psFitsWriteImageCube(outFits, header, cube, extname)) { + psErrorStackPrint(stderr, "Can't write image(s) to extension %d\n", i); + return PS_EXIT_SYS_ERROR; + } + psFree(cube); + } + break; + } + case PS_FITS_TYPE_BINARY_TABLE: + case PS_FITS_TYPE_ASCII_TABLE: { + psArray *table = psFitsReadTable(inFits); + if (!table) { + psErrorStackPrint(stderr, "Can't read table from extension %d\n", i); + return PS_EXIT_SYS_ERROR; + } + if (!psFitsWriteTable(outFits, header, table, extname)) { + psErrorStackPrint(stderr, "Can't write table to extension %d\n", i); + return PS_EXIT_SYS_ERROR; + } + psFree(table); + break; + } + default: + psAbort("Unrecognised FITS extension type: %x\n", type); + } + + psFree(header); + psFree(extname); + } + + psFitsClose(inFits); + psFitsClose(outFits); + + return PS_EXIT_SUCCESS; +} Index: branches/eam_branches/20090715/arclog/arclog.pl =================================================================== --- branches/eam_branches/20090715/arclog/arclog.pl (revision 25020) +++ branches/eam_branches/20090715/arclog/arclog.pl (revision 25022) @@ -19,8 +19,9 @@ use Pod::Usage qw( pod2usage ); -my ($email); +my ($debug, $email); GetOptions( - 'email|s=s' => \$email, + 'debug|d' => \$debug, + 'email|s=s' => \$email, ) || pod2usage( 2 ); @@ -40,5 +41,5 @@ die "can not open logdb" unless $dbh; -$dbh->do("CREATE TABLE IF NOT EXISTS log(record TEXT, PRIMARY KEY(record))") +$dbh->do("CREATE TABLE IF NOT EXISTS log(stamp TEXT, record TEXT, PRIMARY KEY(record))") or die "db error"; @@ -162,5 +163,5 @@ $a->{time} cmp $b->{time} } @records; -my $query = $dbh->prepare("INSERT OR IGNORE INTO log VALUES(?)") +my $query = $dbh->prepare("INSERT OR IGNORE INTO log(stamp, record) VALUES(DATETIME('now'), ?)") or die "database error: $!"; @@ -176,6 +177,8 @@ = @$rec{qw( host time device event elapse_time errors )}; # pack record - $query->execute("$host|$time|$device|$event|$elapse_time|$errors") + my $rec = "$host|$time|$device|$event|$elapse_time|$errors"; + $query->execute($rec) or die "database error: $!"; + warn "inserting $rec\n" if $debug; # do not print filtered records Index: branches/eam_branches/20090715/arclog/arclog_readdb.pl =================================================================== --- branches/eam_branches/20090715/arclog/arclog_readdb.pl (revision 25020) +++ branches/eam_branches/20090715/arclog/arclog_readdb.pl (revision 25022) @@ -19,5 +19,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -my $filter_records = 1; +my $filter_records = 0; my $rcfile = "$ENV{HOME}/.arclogrc"; @@ -30,5 +30,5 @@ die "can not open logdb" unless $dbh; -my $query = $dbh->prepare("SELECT * FROM log") +my $query = $dbh->prepare("SELECT record, stamp FROM log") or die "database error: $!"; $query->execute or die "database error: $!"; @@ -52,4 +52,5 @@ @rec{qw( host time device event elapse_time errors )} = split(/\|/, $row->[0]); + $rec{stamp} = $row->[1]; $rec{time} = $strptime->parse_datetime($rec{time}); push @records, \%rec; @@ -60,6 +61,6 @@ RECORDS: foreach my $rec (@records) { - my ($host, $time, $device, $event, $elapse_time, $errors) - = @$rec{qw( host time device event elapse_time errors )}; + my ($host, $stamp, $time, $device, $event, $elapse_time, $errors) + = @$rec{qw( host stamp time device event elapse_time errors )}; # do not print filtered records @@ -71,5 +72,5 @@ no warnings qw( uninitialized ); - print "$host $time $device $event $elapse_time $errors\n"; + print "$stamp - $host $time $device $event $elapse_time $errors\n"; use warnings; } Index: branches/eam_branches/20090715/console/scripts/console =================================================================== --- branches/eam_branches/20090715/console/scripts/console (revision 25020) +++ branches/eam_branches/20090715/console/scripts/console (revision 25022) @@ -63,73 +63,12 @@ =over 4 -=item * --db|-d +=item * --user|-u -Name of database (C) to write too. +The C username to use when connecting to the console port. -Optional if defined in the F<.nebdiskdrc> file or the appropriate environment -variable is set. - -=item * --user|-u - -Username to authenticate with. - -Optional if defined in the F<.nebdiskdrc> file or the appropriate environment -variable is set. - -=item * --pass|-p - -Password to authenticate with. - -Optional if defined in the F<.nebdiskdrc> file or the appropriate environment -variable is set. - -=item * --debug - -This flag prevents the program from "daemonizing" so output can be sent to -C/C and a debugger used on the running process. - -=item * --pidfile - -Filename to log the running daemon's PID too. - -Optional, if this option is omitted and no value is defined in the -F<.nebdiskrc> then no pidfile is created. - -=item * --stop|-s - -Uses the pidfile to kill an already running daemon. - -=item * --restart|-r - -Uses the pidfile to kill an already running daemon and then starts a new -instance. - -=item * --verbose|-r +=item * --verbose|-v Turns on informational/debugging messages to be sent to the -C/C. This option is probably not very usefully unless combined -with C<--debug>. - -=back - -=head1 ENVIRONMENT - -These environment variables may be used in place of the specified command line -options. All command line option will override the corresponding environment -value. - -=over 4 - -=item * C - -Equivalent to --db|-d - -=item * C - -Equivalent to --user|-u - -=item * C - -Equivalent to --pass|-p +C/C. =back @@ -137,41 +76,5 @@ =head1 RCFILE -This program will attempt to read an C from F<$HOME/.nebdiskrc> and if -found will use it's values to override program defaults. Please not that the -precedence for values is: command line, rcfile, environment variables. - -The format of the C is in L format and uses the following values: - - --- - db: nebulous - dbpass: '@neb@' - dbuser: nebulous - mounts: - - /mnt - - /tmp - - /usr - pidfile: /var/tmp/nebdiskd - poll_interval: 5 - -The values C, C, C, and C have the same semantics -as their command line and/or environment variable equivalents. - -=over 4 - -=item * C - -A list of "paths" to C before mounted volumes are polled. The propose -of this option is to attempt to keep "automounted" volumes mounted while this -program is running. - -This value may be omitted or left blank. - -=item * C - -The number of seconds to wait between checks for mounted volumes. - -This value may be omitted or left blank. The default value is C<60>s. - -=back +This program will attempt to read an C from F<$HOME/.Consolerc>. =head1 CREDITS @@ -209,5 +112,5 @@ =head1 SEE ALSO -L +L, L, L, L =cut Index: branches/eam_branches/20090715/console/scripts/console-config =================================================================== --- branches/eam_branches/20090715/console/scripts/console-config (revision 25020) +++ branches/eam_branches/20090715/console/scripts/console-config (revision 25022) @@ -152,39 +152,5 @@ =head1 RCFILE -This program will attempt to read an C from F<$HOME/.Consolerc> - -The format of the C is in L format and uses the following values: - - --- - db: nebulous - dbpass: '@neb@' - dbuser: nebulous - mounts: - - /mnt - - /tmp - - /usr - pidfile: /var/tmp/nebdiskd - poll_interval: 5 - -The values C, C, C, and C have the same semantics -as their command line and/or environment variable equivalents. - -=over 4 - -=item * C - -A list of "paths" to C before mounted volumes are polled. The propose -of this option is to attempt to keep "automounted" volumes mounted while this -program is running. - -This value may be omitted or left blank. - -=item * C - -The number of seconds to wait between checks for mounted volumes. - -This value may be omitted or left blank. The default value is C<60>s. - -=back +This program will attempt to read an C from F<$HOME/.Consolerc>. =head1 CREDITS @@ -222,5 +188,5 @@ =head1 SEE ALSO -L +L, L, L, L =cut Index: branches/eam_branches/20090715/console/scripts/console-ipp-defaults =================================================================== --- branches/eam_branches/20090715/console/scripts/console-ipp-defaults (revision 25020) +++ branches/eam_branches/20090715/console/scripts/console-ipp-defaults (revision 25022) @@ -139,5 +139,5 @@ =over 4 -=item * --verbose|-r +=item * --verbose|-v Turns on informational/debugging messages to be sent to the @@ -180,5 +180,5 @@ =head1 SEE ALSO -L, L +L, L, L, L =cut Index: branches/eam_branches/20090715/dbconfig/changes.txt =================================================================== --- branches/eam_branches/20090715/dbconfig/changes.txt (revision 25020) +++ branches/eam_branches/20090715/dbconfig/changes.txt (revision 25022) @@ -1201,2 +1201,7 @@ ALTER TABLE distTarget DROP KEY clean; ALTER TABLE distTarget ADD CONSTRAINT UNIQUE(label, filter, stage, clean); + +ALTER TABLE pstampJob DROP COLUMN args; +ALTER TABLE pstampJob DROP COLUMN uri; +ALTER TABLE magicDSRun ADD COLUMN inv_magic_id BIGINT AFTER magic_id; +ALTER TABLE magicDSRun ADD FOREIGN KEY (inv_magic_id) REFERENCES magicRun(magic_id); Index: branches/eam_branches/20090715/dbconfig/magic.md =================================================================== --- branches/eam_branches/20090715/dbconfig/magic.md (revision 25020) +++ branches/eam_branches/20090715/dbconfig/magic.md (revision 25022) @@ -47,4 +47,5 @@ magic_ds_id S64 0 # Primary Key magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) + inv_magic_id S64 0 # Primary Key fkey(inverse_magic_id) ref magicRun(magic_id) state STR 0 # Key stage STR 64 Index: branches/eam_branches/20090715/dbconfig/pstamp.md =================================================================== --- branches/eam_branches/20090715/dbconfig/pstamp.md (revision 25020) +++ branches/eam_branches/20090715/dbconfig/pstamp.md (revision 25022) @@ -36,7 +36,5 @@ jobType STR 16 fault S32 0 - uri STR 255 exp_id S64 0 outputBase STR 255 - args STR 511 END Index: branches/eam_branches/20090715/doc/docgen.pl =================================================================== --- branches/eam_branches/20090715/doc/docgen.pl (revision 25020) +++ (revision ) @@ -1,36 +1,0 @@ -#!/usr/bin/perl -w - -#Iterate through ipp src dir to find Doxygen configuration files -#Run Doxygen and copy the html files produced to alala web server -#Called daily by cron -#Bill Giebink 12/08 - -use strict; -use warnings; -use File::Find; -use File::Copy; - -my $srcDir = "/data/ipp003.0/ippTests/src/ipp"; -my $destDir = "/data/alala.0/ippDocs"; -system `rm -rf $destDir/*`; - -finddepth(\&wanted, $srcDir); - -sub wanted { - #don't want the archive or DB src - return if ($File::Find::name =~ /archive$/); - return if ($File::Find::name =~ /ippdb\.src/); - - if ($File::Find::name =~ /Doxyfile\.in$/) { - my $file = $File::Find::name; - my $subDir = $1 if ($file =~ /\/([^\/]+)\/Doxyfile/); - my $currDir = $File::Find::dir; - system ("$currDir/autogen.sh"); - system ("doxygen"); - system ("mkdir $destDir/$subDir"); - system ("cp $currDir/docs/html/* $destDir/$subDir"); - } -} - - - Index: branches/eam_branches/20090715/doc/misc/docgen.pl =================================================================== --- branches/eam_branches/20090715/doc/misc/docgen.pl (revision 25022) +++ branches/eam_branches/20090715/doc/misc/docgen.pl (revision 25022) @@ -0,0 +1,37 @@ +#!/usr/bin/perl -w + +#Iterate through ipp src dir to find Doxygen configuration files +#Run Doxygen and copy the html files produced to alala web server +#Called daily by cron + +use strict; +use warnings; +use File::Find; +use File::Copy; + +my $srcDir = "/data/ipp002.0/ippDoxygen/ipp"; +system `rm -rf /data/ipp002.0/ippDoxygen/ipp`; +system `svn co http://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/ /data/ipp002.0/ippDoxygen/ipp`; +my $destDir = "/data/alala.0/ippDocs"; +system `rm -rf $destDir/*`; + +finddepth(\&wanted, $srcDir); +chdir $srcDir; +system `/usr/bin/doxygen`; +system ("cp -r $srcDir/docs/html/* $destDir"); + +sub wanted { + #don't want the archive or DB src + return if ($File::Find::name =~ /archive$/); + return if ($File::Find::name =~ /ippdb\.src/); + + if ($File::Find::name =~ /Doxyfile\.in$/) { + my $file = $File::Find::name; + my $subDir = $1 if ($file =~ /\/([^\/]+)\/Doxyfile/); + my $currDir = $File::Find::dir; + system ("$currDir/autogen.sh"); + } +} + + + Index: branches/eam_branches/20090715/ippMonitor/Makefile.in =================================================================== --- branches/eam_branches/20090715/ippMonitor/Makefile.in (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/Makefile.in (revision 25022) @@ -78,4 +78,5 @@ $(DESTWWW)/detStackedImfile_failure.php \ $(DESTWWW)/detRun.php \ +$(DESTWWW)/detRunActive.php \ $(DESTWWW)/detRunSummary.php \ $(DESTWWW)/newExp.php \ Index: branches/eam_branches/20090715/ippMonitor/def/camPendingExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camPendingExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camPendingExp.d (revision 25022) @@ -25,6 +25,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/camProcessedExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camProcessedExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camProcessedExp.d (revision 25022) @@ -33,6 +33,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_Images.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_Images.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_Images.d (revision 25022) @@ -37,6 +37,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_NoImages.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_NoImages.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_NoImages.d (revision 25022) @@ -33,6 +33,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_failure.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_failure.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camProcessedExp_failure.d (revision 25022) @@ -33,6 +33,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/camProcessedImfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camProcessedImfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camProcessedImfile.d (revision 25022) @@ -28,6 +28,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/camStageExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/camStageExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/camStageExp.d (revision 25022) @@ -24,6 +24,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 10, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/chipPendingExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/chipPendingExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/chipPendingExp.d (revision 25022) @@ -23,6 +23,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/chipPendingImfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/chipPendingImfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/chipPendingImfile.d (revision 25022) @@ -28,6 +28,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/chipProcessedExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/chipProcessedExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/chipProcessedExp.d (revision 25022) @@ -23,6 +23,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/chipProcessedImfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/chipProcessedImfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/chipProcessedImfile.d (revision 25022) @@ -33,6 +33,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/chipProcessedImfile_failure.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/chipProcessedImfile_failure.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/chipProcessedImfile_failure.d (revision 25022) @@ -34,6 +34,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/chipStageExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/chipStageExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/chipStageExp.d (revision 25022) @@ -21,6 +21,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/detResidExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/detResidExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/detResidExp.d (revision 25022) @@ -41,4 +41,6 @@ FIELD (detResidExp.bg/detResidExp.bin_stdev), 8, %.3f, S/N, value FIELD (detResidExp.bg/rawExp.exp_time), 8, %.3f, flux, value +FIELD rawExp.exp_time, 8, %.2f, exptime, value +FIELD rawExp.ccd_temp, 8, %.2f, ccd temp, value # FIELD *, 8, %s, S/N, op=OP1 # FIELD detResidExp.fringe_0, 8, %.3f, fringe, value @@ -50,7 +52,5 @@ FIELD rawExp.dateobs, 19, %T, obs date, value FIELD rawExp.filter, 10, %s, filter, value -FIELD rawExp.pon_time, 10, %d, pon_time, value -FIELD rawExp.exp_time, 8, %.2f, exptime, value -FIELD rawExp.ccd_temp, 8, %.2f, ccd temp, value +FIELD rawExp.pon_time, 10, %.1f, pon_time, value FIELD rawExp.airmass, 8, %.4f, airmass, value FIELD rawExp.sun_alt, 8, %.4f, sun_alt, value Index: branches/eam_branches/20090715/ippMonitor/def/detRunActive.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/detRunActive.d (revision 25022) +++ branches/eam_branches/20090715/ippMonitor/def/detRunActive.d (revision 25022) @@ -0,0 +1,49 @@ +TABLE detRun +TITLE Active Detrend Runs +FILE detRunActive.php +MENU ipp.detrend.dat + +# the following WHERE clauses are added to all queries joined by AND +WHERE detRun.state != 'drop' + +ARGS ARG1 detRunSummary.det_id=$det_id + +ARGS ARG2 detInputExp.det_id=$det_id +ARGS ARG2 detInputExp.iteration=$iteration + +ARGS ARG3 detProcessedExp.det_id=$det_id + +ARGS ARG4 detResidExp.det_id=$det_id +ARGS ARG4 detResidExp.iteration=$iteration + +ARGS ARG5 detStackedImfile.det_id=$det_id +ARGS ARG6 detNormalizedImfile.det_id=$det_id +ARGS ARG7 detRegisteredImfile.det_id=$det_id + +# field size format name show link to extras +FIELD det_id, 5, %s, det ID, value, detRunSummary.php, ARG1 +FIELD iteration, 3, %s, iter +FIELD det_type, 12, %s, type +FIELD state, 5, %s, state +FIELD *, 5, %s, IN, value=IN, detInputExp.php, ARG2 +FIELD *, 5, %s, P, value=P, detProcessedExp.php, ARG3 +FIELD *, 5, %s, D, value=D, detResidExp.php, ARG4 +FIELD *, 5, %s, S, value=S, detStackedImfile.php, ARG5 +FIELD *, 5, %s, N, value=N, detNormalizedImfile.php, ARG6 +FIELD *, 5, %s, R, value=R, detRegisteredImfile.php, ARG7 +FIELD mode, 5, %s, mode +FIELD filter, 5, %s, filter +FIELD registered, 19, %s, registered +FIELD use_begin, 19, %s, use_begin +FIELD use_end, 19, %s, use_end +FIELD airmass_min, 5, %.2f, min_airmass +FIELD airmass_max, 5, %.2f, max +FIELD exp_time_min, 5, %.2f, min_exp_time +FIELD exp_time_max, 5, %.2f, max +FIELD ccd_temp_min, 5, %.2f, min_ccd_temp +FIELD ccd_temp_max, 5, %.2f, max +FIELD posang_min, 5, %.2f, min_posang +FIELD posang_max, 5, %s, max + +TD_CLASS list_off $state == 'drop' +TD_CLASS list_run $state == 'run' Index: branches/eam_branches/20090715/ippMonitor/def/fakePendingExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/fakePendingExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/fakePendingExp.d (revision 25022) @@ -29,6 +29,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/fakePendingImfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/fakePendingImfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/fakePendingImfile.d (revision 25022) @@ -30,6 +30,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/fakeProcessedExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/fakeProcessedExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/fakeProcessedExp.d (revision 25022) @@ -29,6 +29,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/fakeProcessedImfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/fakeProcessedImfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/fakeProcessedImfile.d (revision 25022) @@ -37,6 +37,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/fakeProcessedImfile_failure.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/fakeProcessedImfile_failure.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/fakeProcessedImfile_failure.d (revision 25022) @@ -37,6 +37,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/fakeStageExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/fakeStageExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/fakeStageExp.d (revision 25022) @@ -27,6 +27,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/rawExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/rawExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/rawExp.d (revision 25022) @@ -16,6 +16,6 @@ FIELD exp_type, 8, %s, Type FIELD dateobs, 19, %T, Date/Time -FIELD ra, 8, %10.6f, RA, op=OP1 -FIELD decl, 8, %10.6f, DEC, op=OP2 +FIELD ra, 8, %C, RA, op=OP1 +FIELD decl, 8, %C, DEC, op=OP2 FIELD object, 10, %s, object FIELD filter, 10, %s, FILTER @@ -25,6 +25,9 @@ FIELD pon_time, 5, %.2f, pon_time FIELD sat_pixel_frac, 5, %.4f, f(sat pixels) -FIELD bg, 8, %.2f, backgnd -FIELD bg_stdev, 8, %.2f, stdev +FIELD bg, 8, %.2f, backgnd +FIELD bg_stdev, 8, %.2f, stdev FIELD bg_mean_stdev, 8, %.2f, [stdev] +FIELD moon_angle, 5, %.2f, moon angle +FIELD moon_phase, 5, %.2f, moon phase +FIELD sun_alt, 5, %.2f, sun altitude FIELD comment, 65, %s, Comment Index: branches/eam_branches/20090715/ippMonitor/def/rawExp_failed.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/rawExp_failed.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/rawExp_failed.d (revision 25022) @@ -18,6 +18,6 @@ FIELD telescope, 10, %s, Telescope FIELD camera, 10, %s, Camera -FIELD ra, 8, %10.6f, RA, op=OP1 -FIELD decl, 8, %10.6f, DEC, op=OP2 +FIELD ra, 8, %C, RA, op=OP1 +FIELD decl, 8, %C, DEC, op=OP2 FIELD filter, 10, %s, FILTER FIELD dateobs, 19, %T, Date/Time Index: branches/eam_branches/20090715/ippMonitor/def/rawImfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/rawImfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/rawImfile.d (revision 25022) @@ -17,6 +17,6 @@ FIELD rawImfile.exp_type, 8, %s, exp_type FIELD rawImfile.dateobs, 19, %T, Date/Time -FIELD rawImfile.ra, 8, %10.6f, RA, op=OP1 -FIELD rawImfile.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawImfile.ra, 8, %C, RA, op=OP1 +FIELD rawImfile.decl, 8, %C, DEC, op=OP2 FIELD rawImfile.object, 8, %s, object FIELD rawImfile.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/rawImfile_failed.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/rawImfile_failed.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/rawImfile_failed.d (revision 25022) @@ -14,6 +14,6 @@ FIELD exp_name, 5, %s, Exposure FIELD class_id, 8, %s, Chip ID -FIELD ra, 8, %10.6f, RA, op=OP1 -FIELD decl, 8, %10.6f, DEC, op=OP2 +FIELD ra, 8, %C, RA, op=OP1 +FIELD decl, 8, %C, DEC, op=OP2 FIELD bg, 8, %s, backgnd FIELD bg_stdev, 8, %s, stdev Index: branches/eam_branches/20090715/ippMonitor/def/rawUnknownExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/rawUnknownExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/rawUnknownExp.d (revision 25022) @@ -19,6 +19,6 @@ FIELD camera, 10, %s, Camera FIELD dateobs, 19, %T, Date/Time -FIELD ra, 8, %10.6f, RA, op=OP1 -FIELD decl, 8, %10.6f, DEC, op=OP2 +FIELD ra, 8, %C, RA, op=OP1 +FIELD decl, 8, %C, DEC, op=OP2 FIELD object, 8, %s, object FIELD filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/stackInputSkyfile.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/stackInputSkyfile.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/stackInputSkyfile.d (revision 25022) @@ -37,6 +37,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, Filter Index: branches/eam_branches/20090715/ippMonitor/def/warpFailedSkyfiles.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpFailedSkyfiles.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpFailedSkyfiles.d (revision 25022) @@ -43,6 +43,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/warpProcessedSkyfiles.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpProcessedSkyfiles.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpProcessedSkyfiles.d (revision 25022) @@ -46,6 +46,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/warpProcessedSkyfiles_Images.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 25022) @@ -50,6 +50,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/warpSkycellMap.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpSkycellMap.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpSkycellMap.d (revision 25022) @@ -26,6 +26,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, Filter Index: branches/eam_branches/20090715/ippMonitor/def/warpStageExp.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpStageExp.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpStageExp.d (revision 25022) @@ -30,6 +30,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/warpStageSkyfileInputs.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpStageSkyfileInputs.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpStageSkyfileInputs.d (revision 25022) @@ -43,6 +43,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/def/warpStageSkyfiles.d =================================================================== --- branches/eam_branches/20090715/ippMonitor/def/warpStageSkyfiles.d (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/def/warpStageSkyfiles.d (revision 25022) @@ -35,6 +35,6 @@ FIELD rawExp.camera, 10, %s, Camera FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.ra, 8, %10.6f, RA, op=OP1 -FIELD rawExp.decl, 8, %10.6f, DEC, op=OP2 +FIELD rawExp.ra, 8, %C, RA, op=OP1 +FIELD rawExp.decl, 8, %C, DEC, op=OP2 FIELD rawExp.object, 8, %s, Object FIELD rawExp.filter, 10, %s, FILTER Index: branches/eam_branches/20090715/ippMonitor/raw/ipp.detrend.dat =================================================================== --- branches/eam_branches/20090715/ippMonitor/raw/ipp.detrend.dat (revision 25020) +++ branches/eam_branches/20090715/ippMonitor/raw/ipp.detrend.dat (revision 25022) @@ -28,4 +28,5 @@ menulink | menuselect | link | Detrend Frames | masterDetrendFrames.php menulink | menuselect | link | Detrend Runs | detRun.php +menulink | menuselect | link | Detrend Runs (Active) | detRunActive.php menulink | menuselect | link | Detrend Iterations | detRunSummary.php Index: branches/eam_branches/20090715/ippScripts/scripts/dist_component.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/dist_component.pl (revision 25020) +++ branches/eam_branches/20090715/ippScripts/scripts/dist_component.pl (revision 25022) @@ -157,6 +157,6 @@ my $image_type = get_image_type($stage, $file_rule); - # if we are building a clean bundle skip this rule - next if $clean && $image_type; + # if this is an image and we are building a clean bundle or if quality is bad skip this rule + next if $image_type and ($clean or $poor_quality); # if magic is required, don't ship jpegs or binned fits images Index: branches/eam_branches/20090715/ippScripts/scripts/magic_destreak.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/magic_destreak.pl (revision 25020) +++ branches/eam_branches/20090715/ippScripts/scripts/magic_destreak.pl (revision 25022) @@ -38,5 +38,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base); +my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base); my ($outroot, $recoveryroot); my ($replace, $release); @@ -47,4 +47,5 @@ 'camera=s' => \$camera, # camera for evaluating file rules 'streaks=s' => \$streaks, # file containing the list of streaks + 'inv_streaks=s' => \$inv_streaks,# file containing the list of streaks from the inverse diff 'stage=s' => \$stage, # raw, chip, warp, or diff 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id @@ -52,5 +53,4 @@ 'uri=s' => \$uri, # uri of the input image 'path_base=s' => \$path_base, # path_base of the input - 'inverse' => \$inverse, # Inverse subtraction? 'cam_path_base=s'=> \$cam_path_base, # path_base from camera stage (for chip and raw) 'outroot=s' => \$outroot, # "directory" for temporary images (may be nebulous) @@ -94,5 +94,6 @@ } elsif ($stage ne "camera") { &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); -} +} +$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL"); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -234,8 +235,7 @@ $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $path_base); } elsif ($stage eq "diff") { - my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images - $image = $ipprc->filename($name, $path_base); - $mask = $ipprc->filename("$name.MASK", $path_base); - $weight = $ipprc->filename("$name.VARIANCE", $path_base); + $image = $ipprc->filename("PPSUB.OUTPUT", $path_base); + $mask = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base); + $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base); $sources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base); } @@ -266,4 +266,30 @@ } } + if (($stage eq "diff") and $inv_streaks) { + $image = $ipprc->filename("PPSUB.INVERSE", $path_base); + $mask = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base); + $weight = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $path_base); + $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base); + + my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $inv_streaks -image $image"; + + $command .= " -recovery $recoveryroot" if defined $recoveryroot; + $command .= " -mask $mask" if defined $mask; + $command .= " -weight $weight" if defined $weight; + $command .= " -sources $sources" if defined $sources; + $command .= " -replace" if $replace; + $command .= " -release" if $release; + $command .= " -dbname $dbname" if defined $dbname; + unless (defined $no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code); + } + } else { + print "skipping command $command\n"; + } + } } else { # camera stage. The only work to do is to censor the detections file Index: branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl (revision 25020) +++ branches/eam_branches/20090715/ippScripts/scripts/magic_destreak_revert.pl (revision 25022) @@ -60,5 +60,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot", +pod2usage( -msg => "Required options: --magic_ds_id --camera --stage --stage_id --component --path_base --outroot", -exitval => 3) unless defined $magic_ds_id and @@ -95,5 +95,5 @@ } elsif ($stage eq "diff") { $skycell_id = $component; -} else { +} elsif ($stage ne "camera") { &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } @@ -142,6 +142,6 @@ -my ($image, $mask, $ch_mask, $weight); -my ($bimage, $bmask, $bch_mask, $bweight); +my ($image, $mask, $ch_mask, $weight, $sources, $astrom); +my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); if ($stage eq "raw") { @@ -167,11 +167,16 @@ $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $class_id); $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id); +} elsif ($stage eq "camera") { + $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base); + $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base); } elsif ($stage eq "warp") { $image = $ipprc->filename("PSWARP.OUTPUT", $path_base); $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base); $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base); + $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $path_base); $bimage = $ipprc->filename("PSWARP.OUTPUT", $backup_path_base); $bmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $backup_path_base); $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base); } elsif ($stage eq "diff") { my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images @@ -179,21 +184,22 @@ $mask = $ipprc->filename("$name.MASK", $path_base); $weight = $ipprc->filename("$name.VARIANCE", $path_base); + $sources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base); $bimage = $ipprc->filename($name, $backup_path_base); $bmask = $ipprc->filename("$name.MASK", $backup_path_base); $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); -} - -revert_file($image, $bimage) or + $bsources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $backup_path_base); +} + +if ($image) { + revert_file($image, $bimage) or &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); - -# don't commit next the next couple of lines and see xxx below -# change to if 0 to revert chip runs without masks -if (0) { +} if ($mask) { if (!revert_file($mask, $bmask)) { if ($stage eq 'chip') { - # XXX: don't fail if the mask file fails to revert. It probably doesn't exist - # Handle this properly + # don't fail if the mask file (from the camera stage) fails to revert. + # It probably doesn't exist + # XXX: Handle this properly print STDERR "failure to revert mask file, ignored\n"; } else { @@ -204,17 +210,23 @@ if ($ch_mask) { - # Handle this properly if (!revert_file($ch_mask, $bch_mask)) { - # print STDERR "failure to revert chip mask file, ignored\n"; &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } } -# xxx don't commmit next line -} if ($weight) { revert_file($weight, $bweight) or &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); +} + +if ($sources) { + revert_file($sources, $bsources) or + &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); +} + +if ($astrom) { + revert_file($astrom, $bastrom) or + &my_die("failed to restore astrometry file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } Index: branches/eam_branches/20090715/ippScripts/scripts/publish_file.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/publish_file.pl (revision 25020) +++ branches/eam_branches/20090715/ippScripts/scripts/publish_file.pl (revision 25022) @@ -134,5 +134,5 @@ my $file = $ipprc->file_resolve( $files{$comp} ) or &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR); - my_die("Unable to find file $file", $pub_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists( $file ); + my_die("Unable to find file $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $file ); my $zp = $zp{$comp}; Index: branches/eam_branches/20090715/ippScripts/scripts/receive_file.pl =================================================================== --- branches/eam_branches/20090715/ippScripts/scripts/receive_file.pl (revision 25020) +++ branches/eam_branches/20090715/ippScripts/scripts/receive_file.pl (revision 25022) @@ -19,5 +19,5 @@ use PS::IPP::Metadata::List qw( parse_md_list ); use PS::IPP::Config 1.01 qw( :standard ); -use File::Temp qw( tempfile ); +use File::Temp qw( tempfile tempdir ); use File::Basename qw( basename ); use Carp; @@ -35,5 +35,5 @@ } -my $tempdir = "/tmp"; +my $temproot = "/tmp"; # Parse the command-line arguments @@ -72,5 +72,5 @@ defined $workdir; -$tempdir .= "/$file_id"; +my $tempdir = tempdir( "$temproot/receive.$file_id.XXXX", CLEANUP => !$save_temps); &my_die( "dirinfo is NULL for $component", $file_id, $PS_EXIT_CONFIG_ERROR ) @@ -329,9 +329,5 @@ } -if (!$save_temps and -e $filename) { - unlink $filename or &my_die( "Unable to unlink $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR); -} my $mjd_extract = DateTime->now->mjd; # MJD of finishing extract - # All done Index: branches/eam_branches/20090715/ippTasks/Makefile.am =================================================================== --- branches/eam_branches/20090715/ippTasks/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/ippTasks/Makefile.am (revision 25022) @@ -19,4 +19,5 @@ stack.pro \ summit.copy.pro \ + survey.pro \ replicate.pro \ dist.pro \ Index: branches/eam_branches/20090715/ippTasks/calibration.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/calibration.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/calibration.pro (revision 25022) @@ -7,5 +7,5 @@ $LOGSUBDIR = $LOGDIR/calibration -exec mkdir -p $LOGSUBDIR +mkdir $LOGSUBDIR book init calBook Index: branches/eam_branches/20090715/ippTasks/dist.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/dist.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/dist.pro (revision 25022) @@ -5,5 +5,5 @@ #$LOGSUBDIR = $LOGDIR/dist -#exec mkdir -p $LOGSUBDIR +mkdir $LOGSUBDIR ### Initialise the books containing the tasks to do Index: branches/eam_branches/20090715/ippTasks/magic.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/magic.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/magic.pro (revision 25022) @@ -380,6 +380,7 @@ periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 20 + # this query can take a long time + periods -exec 10 + periods -timeout 120 npending 1 @@ -388,5 +389,5 @@ task.exec - $run = magicdstool -todestreak -limit 32 + $run = magicdstool -todestreak -limit 120 if ($DB:n == 0) option DEFAULT @@ -449,4 +450,5 @@ book getword magicToDS $pageName camera -var CAMERA book getword magicToDS $pageName streaks_uri -var STREAKS + book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS book getword magicToDS $pageName stage -var STAGE book getword magicToDS $pageName stage_id -var STAGE_ID @@ -454,5 +456,4 @@ book getword magicToDS $pageName uri -var URI book getword magicToDS $pageName path_base -var PATH_BASE - book getword magicToDS $pageName inverse -var INVERSE book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE book getword magicToDS $pageName outroot -var OUTROOT @@ -473,9 +474,5 @@ # TODO: do not add recoveryroot or replace if they are null or zero - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE - if ("$INVERSE" == "T") - # Inverse subtraction - $run = $run --inverse - end + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE add_standard_args run Index: branches/eam_branches/20090715/ippTasks/pantasks.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/pantasks.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/pantasks.pro (revision 25022) @@ -255,5 +255,4 @@ module detrend.reject.pro module detrend.correct.pro - module flatcorr.pro end Index: branches/eam_branches/20090715/ippTasks/pstamp.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/pstamp.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/pstamp.pro (revision 25022) @@ -8,4 +8,13 @@ $pstampJob_DB = 0 $pstampFin_DB = 0 + +# set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config +# warning: no quotes around the two words. it cause it to get passed to pstamptool as one word +# tricky to debug problem ensues +# example: +# $PSDBSERVER = -dbserver hostname +if ($?PSDBSERVER == 0) + $PS_DBSERVER = "" +end macro pstamp.reset @@ -115,13 +124,15 @@ task.exec + $run = pstamptool -pendingreq if ($DB:n == 0) option DEFAULT - command pstamptool -pendingreq else option $DB:$pstampReq_DB - command pstamptool -pendingreq -dbname $DB:$pstampReq_DB + $run = $run $PS_DBSERVER -dbname $DB:$pstampReq_DB $pstampReq_DB ++ if ($pstampReq_DB >= $DB:n) set pstampReq_DB = 0 end + add_poll_args run + command $run end @@ -213,5 +224,5 @@ else option $DB:$pstampFin_DB - command pstamptool -completedreq -dbname $DB:$pstampFin_DB + command pstamptool -completedreq -dbname $DB:$pstampFin_DB $PS_DBSERVER $pstampFin_DB ++ if ($pstampFin_DB >= $DB:n) set pstampFin_DB = 0 @@ -307,5 +318,5 @@ else option $DB:$pstampJob_DB - command pstamptool -pendingjob -limit 10 -dbname $DB:$pstampJob_DB + command pstamptool -pendingjob -limit 10 -dbname $DB:$pstampJob_DB $PS_DBSERVER $pstampJob_DB ++ if ($pstampJob_DB >= $DB:n) set pstampJob_DB = 0 @@ -354,4 +365,6 @@ book setword pstampJob $pageName pantaskState RUN book getword pstampJob $pageName job_id -var JOB_ID + book getword pstampJob $pageName jobType -var JOB_TYPE + book getword pstampJob $pageName rownum -var ROWNUM book getword pstampJob $pageName dbname -var DBNAME book getword pstampJob $pageName outputBase -var OUTPUT_BASE @@ -362,5 +375,5 @@ - $run = pstamp_job_run.pl --job_id $JOB_ID --redirect-output --output_base $OUTPUT_BASE + $run = pstamp_job_run.pl --job_id $JOB_ID --job_type $JOB_TYPE --rownum $ROWNUM --output_base $OUTPUT_BASE --redirect-output add_standard_args run Index: branches/eam_branches/20090715/ippTasks/rcserver.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/rcserver.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/rcserver.pro (revision 25022) @@ -5,5 +5,5 @@ #$LOGSUBDIR = $LOGDIR/rcserver -#exec mkdir -p $LOGSUBDIR +mkdir $LOGSUBDIR ### Initialise the books containing the tasks to do Index: branches/eam_branches/20090715/ippTasks/replicate.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/replicate.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/replicate.pro (revision 25022) @@ -10,5 +10,5 @@ $LOGSUBDIR = $LOGDIR/replicate -exec mkdir -p $LOGSUBDIR +mkdir $LOGSUBDIR book init replicatePending Index: branches/eam_branches/20090715/ippTasks/survey.pro =================================================================== --- branches/eam_branches/20090715/ippTasks/survey.pro (revision 25020) +++ branches/eam_branches/20090715/ippTasks/survey.pro (revision 25022) @@ -4,39 +4,123 @@ check.globals -list DIFF_WW - ThreePi.Run2.g.v0 - ThreePi.Run2.r.v0 - ThreePi.Run2.i.v0 -end - -$DIFF_WW_CNT = 0 - -task diff.check +book create SURVEY_DIFF +book create SURVEY_MAGIC + +$SURVEY_DIFF_DB = 0 +$SURVEY_MAGIC_DB = 0 + +$SURVEY_EXEC = 120 +$SURVEY_POLL = 10 +$SURVEY_TIMEOUT = 600 + + +# user functions to manipulate diff labels +macro survey.add.diff + if ($0 != 3) + echo "USAGE: survey.add.diff (label) (workdir base)" + break + end + book newpage SURVEY_DIFF $1 + book setword SURVEY_DIFF $1 WORKDIR $2 + book setword SURVEY_DIFF $1 STATE PENDING +end + +macro survey.del.diff + if ($0 != 2) + echo "USAGE: survey.del.diff (label)" + break + end + book delpage SURVEY_DIFF $1 +end + +macro survey.show.diff + if ($0 != 1) + echo "USAGE: survey.show.diff" + break + end + book listbook SURVEY_DIFF +end + +# user functions to manipulate magic labels +macro survey.add.magic + if ($0 != 3) + echo "USAGE: survey.add.magic (label) (workdir base)" + break + end + book newpage SURVEY_MAGIC $1 + book setword SURVEY_MAGIC $1 WORKDIR $2 + book setword SURVEY_MAGIC $1 STATE PENDING +end + +macro survey.del.magic + if ($0 != 2) + echo "USAGE: survey.del.magic (label)" + break + end + book delpage SURVEY_MAGIC $1 +end + +macro survey.show.magic + if ($0 != 1) + echo "USAGE: survey.show.magic" + break + end + book listbook SURVEY_MAGIC +end + + +task survey.diff host local - periods -poll 10 - periods -exec 120 - periods -timeout 600 + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT npending 1 - stdout $LOGDIR/diffcheck.log - stderr $LOGDIR/diffcheck.log + stdout $LOGDIR/survey.diff.log + stderr $LOGDIR/survey.diff.log # generate diff warp-warp runs task.exec - if ($DIFF_WW:n == 0) break - $run = difftool -dbname gpc1 -definewarpwarp -distance 0.1 -good_frac 0.1 - $run = $run -input_label $DIFF_WW:$DIFF_WW_CNT - $run = $run -template_label $DIFF_WW:$DIFF_WW_CNT - $run = $run -label $DIFF_WW:$DIFF_WW_CNT - $run = $run -workdir neb://@HOST@.0/gpc1/$DIFF_WW:$DIFF_WW_CNT - $run = $run -reduction WARPWARP - - $DIFF_WW_CNT ++ - if ($DIFF_WW_CNT >= $DIFF_WW:n) - echo "reset DIFF_WW_CNT (was $DIFF_WW_CNT)" - set DIFF_WW_CNT = 0 - end - + book npages SURVEY_DIFF -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_DIFF 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_DIFF $i -var label + book setword SURVEY_DIFF $label STATE NEW + end + book getpage SURVEY_DIFF 0 -var label -key STATE NEW + + # Select different database + $SURVEY_DIFF_DB ++ + if ($SURVEY_DIFF_DB >= $DB:n) + $SURVEY_DIFF_DB = 0 + end + end + + book setword SURVEY_DIFF $label STATE DONE + book getword SURVEY_DIFF $label WORKDIR -var workdir + + $run = difftool -definewarpwarp -distance 0.1 -good_frac 0.1 -timediff 3600 + $run = $run -input_label $label + $run = $run -template_label $label + $run = $run -label $label + $run = $run -workdir $workdir/$label/ + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_DIFF_DB + option $DB:$SURVEY_DIFF_DB + end + +# echo $run command $run end @@ -44,5 +128,5 @@ # success task.exit 0 - echo "checked for new diff runs" +# echo "Success" end @@ -61,2 +145,77 @@ end end + + + +task survey.magic + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.magic.log + stderr $LOGDIR/survey.magic.log + + # generate magic warp-warp runs + task.exec + book npages SURVEY_MAGIC -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_MAGIC 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_MAGIC $i -var label + book setword SURVEY_MAGIC $label STATE NEW + end + book getpage SURVEY_MAGIC 0 -var label -key STATE NEW + + # Select different database + $SURVEY_MAGIC_DB ++ + if ($SURVEY_MAGIC_DB >= $DB:n) + set $SURVEY_MAGIC_DB = 0 + end + end + + book setword SURVEY_MAGIC $label STATE DONE + book getword SURVEY_MAGIC $label WORKDIR -var workdir + + $run = magictool -definebyquery -label $label -diff_label $label -workdir $workdir/$label/ + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_MAGIC_DB + option $DB:$SURVEY_MAGIC_DB + end + +# echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: branches/eam_branches/20090715/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/20090715/ippTools/share/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/Makefile.am (revision 25022) @@ -137,4 +137,9 @@ flatcorr_dropcamera.sql \ magictool_addmask.sql \ + magictool_censor_raw.sql \ + magictool_censor_chip.sql \ + magictool_censor_camera.sql \ + magictool_censor_warp.sql \ + magictool_censor_diff.sql \ magictool_create_tmp_warpcomplete.sql \ magictool_definebyquery_insert.sql \ @@ -145,4 +150,5 @@ magictool_tomask.sql \ magictool_toprocess_inputs.sql \ + magictool_toprocess_runs.sql \ magictool_toprocess_tree.sql \ magictool_toskyfilemask.sql \ @@ -154,4 +160,9 @@ magictool_revertnode.sql \ magicdstool_completed_runs.sql \ + magicdstool_definebyquery_raw.sql \ + magicdstool_definebyquery_chip.sql \ + magicdstool_definebyquery_camera.sql \ + magicdstool_definebyquery_warp.sql \ + magicdstool_definebyquery_diff.sql \ magicdstool_getrunids.sql \ magicdstool_getskycells.sql \ @@ -213,5 +224,4 @@ stacktool_pendingcleanupskyfile.sql \ stacktool_revertsumskyfile_delete.sql \ - stacktool_revertsumskyfile_update.sql \ stacktool_sumskyfile.sql \ stacktool_tosum.sql \ Index: branches/eam_branches/20090715/ippTools/share/chiptool_processedimfile.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/chiptool_processedimfile.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/chiptool_processedimfile.sql (revision 25022) @@ -8,4 +8,5 @@ chipProcessedImfile.bg_mean_stdev, chipProcessedImfile.path_base, + chipProcessedImfile.magicked, chipRun.state, chipRun.workdir, Index: branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_camera.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_camera.sql (revision 25022) @@ -0,0 +1,30 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'camera' AS stage, + camRun.cam_id AS stage_id, + camRun.cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'camera' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n 1 " if rerun else "\n 0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND camRun.magicked = 0 + AND chipRun.magicked + AND chipRun.state = 'full' + AND camRun.state = 'full' + Index: branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_chip.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_chip.sql (revision 25022) @@ -0,0 +1,29 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'chip' AS stage, + chip_id AS stage_id, + camRun.cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'chip' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND chipRun.magicked = 0 + AND chipRun.state = 'full' + AND camRun.state = 'full' + Index: branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_diff.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_diff.sql (revision 25022) @@ -0,0 +1,68 @@ +SELECT * +FROM ( +SELECT + magicRun.magic_id, + exp_id, + 'diff' AS stage, + diff_id AS stage_id, + 0 AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'diff' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND diffRun.bothways = 0 + AND diffRun.magicked = 0 + AND diffRun.state = 'full' + -- WHERE hook %s +UNION +SELECT + forwardRun.magic_id, + forwardRun.exp_id, + 'diff' AS stage, + forwardRun.diff_id AS stage_id, + 0 AS cam_id, + forwardRun.label, + forwardRun.workdir, + inverseRun.magic_id AS inv_magic_id, + inverseRun.exp_id AS inv_exp_id +FROM ( + SELECT + magicRun.magic_id, + exp_id, + diff_id, + magicRun.label, + magicRun.workdir + FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'diff' + WHERE magicRun.state = 'full' + AND magicRun.inverse = 0 + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND diffRun.state = 'full' + AND diffRun.bothways + AND diffRun.magicked = 0 + -- WHERE hook %s +) AS forwardRun +JOIN ( + SELECT + magic_id, + exp_id, + diff_id + FROM magicRun + JOIN magicMask USING(magic_id) + WHERE magicRun.state = 'full' + AND magicRun.inverse = 1 +) AS inverseRun USING(diff_id) + +) AS Foo Index: branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_raw.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_raw.sql (revision 25022) @@ -0,0 +1,28 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'raw' AS stage, + exp_id AS stage_id, + 0 AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage_id = exp_id + AND magicDSRun.stage = 'raw' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND rawExp.magicked = 0 + Index: branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_warp.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_definebyquery_warp.sql (revision 25022) @@ -0,0 +1,28 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + exp_id, + 'warp' AS stage, + warp_id AS stage_id, + 0 AS cam_id, + magicRun.label, + magicRun.workdir, + CAST(NULL AS SIGNED) AS inv_magic_id, + CAST(NULL AS SIGNED) AS inv_exp_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun ON (!magicRun.inverse AND warp1 = warp_id) + OR ( magicRun.inverse AND warp2 = warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id, exp_id) + JOIN rawExp USING(exp_id) + LEFT JOIN magicDSRun ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'warp' +WHERE magicRun.state = 'full' + AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s + OR magicDSRun.magic_ds_id IS NULL) + AND warpRun.magicked = 0 + AND warpRun.state = 'full' + Index: branches/eam_branches/20090715/ippTools/share/magicdstool_getrunids.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_getrunids.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_getrunids.sql (revision 25022) @@ -5,5 +5,6 @@ chip_id, cam_id, - warp_id + warp_id, + diffSkyfile.diff_id FROM magicRun JOIN magicInputSkyfile USING(magic_id) Index: branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_todestreak.sql (revision 25022) @@ -8,4 +8,5 @@ camera, magicMask.uri as streaks_uri, + NULL AS inv_streaks_uri, stage, stage_id, @@ -40,4 +41,5 @@ camera, magicMask.uri as streaks_uri, + NULL AS inv_streaks_uri, stage, stage_id, @@ -77,4 +79,5 @@ camera, magicMask.uri as streaks_uri, + NULL AS inv_streaks_uri, stage, stage_id, @@ -115,4 +118,5 @@ camera, magicMask.uri as streaks_uri, + NULL AS inv_streaks_uri, stage, stage_id, @@ -143,5 +147,5 @@ AND magicDSFile.component IS NULL UNION --- diffSkyfiles +-- regular diffSkyfiles SELECT DISTINCT magicDSRun.magic_ds_id, @@ -151,4 +155,5 @@ rawExp.camera, magicMask.uri as streaks_uri, + NULL AS inv_streaks_uri, stage, magicRun.diff_id as stage_id, @@ -167,4 +172,5 @@ JOIN magicDSRun USING(magic_id) JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) JOIN diffSkyfile ON magicRun.diff_id = diffSkyfile.diff_id @@ -176,6 +182,47 @@ magicDSRun.state = 'new' AND magicDSRun.stage = 'diff' + AND diffRun.bothways = 0 AND diffSkyfile.fault = 0 AND diffSkyfile.quality = 0 AND magicDSFile.component IS NULL +-- bothways diffSkyfiles +UNION +SELECT DISTINCT + magicDSRun.magic_ds_id, + magicRun.magic_id, + magicRun.exp_id, + magicDSRun.label, + rawExp.camera, + magicMask.uri as streaks_uri, + (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri, + stage, + magicRun.diff_id as stage_id, + diffSkyfile.skycell_id as component, + NULL AS uri, + diffSkyfile.path_base, + magicRun.inverse, + NULL as cam_path_base, + outroot, + recoveryroot, + re_place, + remove +FROM rawExp +JOIN magicRun USING (exp_id) +JOIN magicMask USING (magic_id) +JOIN magicDSRun USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) +JOIN diffRun USING(diff_id) +JOIN diffSkyfile + ON magicRun.diff_id = diffSkyfile.diff_id + AND magicInputSkyfile.node = diffSkyfile.skycell_id +LEFT JOIN magicDSFile + ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = diffSkyfile.skycell_id +WHERE + magicDSRun.state = 'new' + AND magicDSRun.stage = 'diff' + AND diffRun.bothways + AND diffSkyfile.fault = 0 + AND diffSkyfile.quality = 0 + AND magicDSFile.component IS NULL ) as Foo Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_camera.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_camera.sql (revision 25022) @@ -19,4 +19,4 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'camera' - AND magicDSRun.state = 'new' + AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') AND magicDSFile.fault > 0 Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_chip.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_chip.sql (revision 25022) @@ -18,4 +18,4 @@ JOIN rawExp using(exp_id) WHERE magicDSRun.stage = 'chip' - AND magicDSRun.state = 'new' + AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') AND magicDSFile.fault > 0 Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_diff.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_diff.sql (revision 25022) @@ -9,5 +9,5 @@ outroot, diffSkyfile.path_base, - NULL AS cam_path_base, + "NULL" AS cam_path_base, 0 AS bytes, 0 AS md5sum @@ -18,4 +18,4 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'diff' - AND magicDSRun.state = 'new' + AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') AND magicDSFile.fault > 0 Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_raw.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_raw.sql (revision 25022) @@ -18,4 +18,4 @@ AND rawImfile.tmp_class_id = summitImfile.class_id) WHERE magicDSRun.stage = 'raw' - AND magicDSRun.state = 'new' + AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') AND magicDSFile.fault > 0 Index: branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_warp.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/magicdstool_torevert_warp.sql (revision 25022) @@ -21,4 +21,4 @@ JOIN rawExp USING(exp_id) WHERE magicDSRun.stage = 'warp' - AND magicDSRun.state = 'new' + AND (magicDSRun.state = 'new' OR magicDSRun.state = 'censored') AND magicDSFile.fault > 0 Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_camera.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_camera.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magictool_censor_camera.sql (revision 25022) @@ -0,0 +1,8 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN magicDSFile USING(magic_ds_id) + JOIN camRun ON magic_ds_id = magicked +SET camRun.magicked = 0, + magicDSRun.state = 'censored', + magicDSFile.fault = 42 + Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_chip.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_chip.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magictool_censor_chip.sql (revision 25022) @@ -0,0 +1,9 @@ +UPDATE chipRun + JOIN chipProcessedImfile USING(chip_id, exp_id) + JOIN magicDSRun ON chipRun.magicked = magicDSRun.magic_ds_id + JOIN magicDSFile USING(magic_ds_id) + JOIN magicRun USING(magic_id, exp_id) +SET chipProcessedImfile.magicked = 0, + chipRun.magicked = 0, + magicDSRun.state = 'censored', + magicDSFile.fault = 42 Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_diff.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_diff.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magictool_censor_diff.sql (revision 25022) @@ -0,0 +1,9 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN magicDSFile USING(magic_ds_id) + JOIN diffRun ON magic_ds_id = magicked + JOIN diffSkyfile ON diffRun.diff_id = diffSkyfile.diff_id +SET diffRun.magicked = 0, + diffSkyfile.magicked = 0, + magicDSRun.state = 'censored', + magicDSFile.fault = 42 Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_raw.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_raw.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magictool_censor_raw.sql (revision 25022) @@ -0,0 +1,10 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN magicDSFile USING(magic_ds_id) + JOIN rawExp ON rawExp.magicked = magic_ds_id + JOIN rawImfile ON rawExp.exp_id = rawImfile.exp_id +SET rawImfile.magicked = 0, + rawExp.magicked = 0, + magicDSRun.state = 'censored', + magicDSFile.fault = 42 + Index: branches/eam_branches/20090715/ippTools/share/magictool_censor_warp.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_censor_warp.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magictool_censor_warp.sql (revision 25022) @@ -0,0 +1,9 @@ +UPDATE magicRun + JOIN magicDSRun USING(magic_id) + JOIN magicDSFile USING(magic_ds_id) + JOIN warpRun ON magic_ds_id = magicked + JOIN warpSkyfile using(warp_id) +SET warpRun.magicked = 0, + warpSkyfile.magicked = 0, + magicDSRun.state = 'censored', + magicDSFile.fault = 42 Index: branches/eam_branches/20090715/ippTools/share/magictool_toprocess_runs.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/magictool_toprocess_runs.sql (revision 25022) +++ branches/eam_branches/20090715/ippTools/share/magictool_toprocess_runs.sql (revision 25022) @@ -0,0 +1,11 @@ +SELECT DISTINCT + magic_id +FROM magicTree +JOIN magicRun USING(magic_id) +LEFT JOIN magicNodeResult USING(magic_id, node) +WHERE + magicRun.state = 'new' +AND magicNodeResult.magic_id IS NULL +-- WHERE hook %s +ORDER BY + magicRun.magic_id Index: branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_otherjob.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_otherjob.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_otherjob.sql (revision 25022) @@ -1,3 +1,3 @@ INSERT INTO pstampJob - (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault) - VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s) + (req_id, rownum, state, jobType, exp_id, outputBase, fault) + VALUES( %lld, '%s', '%s', '%s', %lld, '%s', %s) Index: branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_stampjob.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_stampjob.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/pstamptool_addjob_stampjob.sql (revision 25022) @@ -1,3 +1,3 @@ INSERT INTO pstampJob - (req_id, rownum, state, jobType, uri, exp_id, outputBase, fault, args) - VALUES( %lld, '%s', '%s', '%s', '%s', %lld, '%s', %s, '%s') + (req_id, rownum, state, jobType, exp_id, outputBase, fault) + VALUES( %lld, '%s', '%s', '%s', %lld, '%s', %s) Index: branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/pxadmin_create_tables.sql (revision 25022) @@ -1071,4 +1071,5 @@ magic_ds_id BIGINT AUTO_INCREMENT, magic_id BIGINT, + inv_magic_id BIGINT, state VARCHAR(64), stage VARCHAR(64), @@ -1086,4 +1087,5 @@ KEY(label), FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) + FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1203,8 +1205,6 @@ jobType VARCHAR(16), fault SMALLINT, - uri VARCHAR(255), exp_id BIGINT, outputBase VARCHAR(255), - args VARCHAR(511), PRIMARY KEY(job_id, req_id), KEY(job_id), Index: branches/eam_branches/20090715/ippTools/share/stacktool_revertsumskyfile_update.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/stacktool_revertsumskyfile_update.sql (revision 25020) +++ (revision ) @@ -1,5 +1,0 @@ -UPDATE stackRun -JOIN stackSumSkyfile USING(stack_id) -SET stackRun.state = 'new' -WHERE - fault != 0 Index: branches/eam_branches/20090715/ippTools/share/stacktool_sumskyfile.sql =================================================================== --- branches/eam_branches/20090715/ippTools/share/stacktool_sumskyfile.sql (revision 25020) +++ branches/eam_branches/20090715/ippTools/share/stacktool_sumskyfile.sql (revision 25022) @@ -2,6 +2,9 @@ stackSumSkyfile.*, stackRun.state, + stackRun.tess_id, stackRun.skycell_id, + stackRun.filter, stackRun.workdir, + stackRun.label, (SELECT rawExp.camera FROM stackInputSkyfile Index: branches/eam_branches/20090715/ippTools/src/dettool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/dettool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/dettool.c (revision 25022) @@ -276,6 +276,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-sun_angle_max", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); PXOPT_LOOKUP_TIME(time_end, config->args, "-time_end", false, false); @@ -521,6 +521,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-sun_angle_max", false, false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -557,6 +557,8 @@ PXOPT_COPY_F64(config->args, where, "-select_posang_min", "posang", ">="); PXOPT_COPY_F64(config->args, where, "-select_posang_max", "posang", "<="); - PXOPT_COPY_F64(config->args, where, "-select_solang_min", "solang", ">="); - PXOPT_COPY_F64(config->args, where, "-select_solang_max", "solang", "<="); + PXOPT_COPY_F64(config->args, where, "-select_sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F64(config->args, where, "-select_sun_angle_max", "sun_angle", "<="); + PXOPT_COPY_F64(config->args, where, "-select_sun_alt_min", "sun_alt", ">="); + PXOPT_COPY_F64(config->args, where, "-select_sun_alt_max", "sun_alt", "<="); PXOPT_COPY_F64(config->args, where, "-select_moon_angle_min", "moon_angle", ">="); @@ -775,6 +777,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-set_posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-set_posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-set_solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-set_solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-set_sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-set_sun_angle_max", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-set_time_begin", false, false); @@ -1613,6 +1615,6 @@ PXOPT_LOOKUP_F64(posang_min, config->args, "-posang_min", false, false); PXOPT_LOOKUP_F64(posang_max, config->args, "-posang_max", false, false); - PXOPT_LOOKUP_F64(solang_min, config->args, "-solang_min", false, false); - PXOPT_LOOKUP_F64(solang_max, config->args, "-solang_max", false, false); + PXOPT_LOOKUP_F64(solang_min, config->args, "-sun_angle_min", false, false); + PXOPT_LOOKUP_F64(solang_max, config->args, "-sun_angle_max", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_TIME(time_begin, config->args, "-time_begin", false, false); Index: branches/eam_branches/20090715/ippTools/src/dettoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/dettoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/dettoolConfig.c (revision 25022) @@ -68,6 +68,6 @@ psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -100,6 +100,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); @@ -125,6 +125,8 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_alt_min", 0, "define min solar altitude", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_alt_max", 0, "define max solar altitude", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_min", 0, "define min moon angle", NAN); @@ -164,6 +166,6 @@ psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_min", 0, "define rotator position angle", NAN); psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_posang_max", 0, "define rotator position angle", NAN); - psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_min", 0, "define solar angle", NAN); - psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_solang_max", 0, "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_sun_angle_min", 0, "define solar angle", NAN); + psMetadataAddF64(definebydetrunArgs, PS_LIST_TAIL, "-set_sun_angle_max", 0, "define solar angle", NAN); psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddTime(definebydetrunArgs, PS_LIST_TAIL, "-set_time_begin", 0, "start of period to apply detrend too", NULL); @@ -840,6 +842,6 @@ psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(register_detrendArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddTime(register_detrendArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); Index: branches/eam_branches/20090715/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/difftool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/difftool.c (revision 25022) @@ -582,5 +582,5 @@ PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); @@ -661,4 +661,10 @@ PXOPT_COPY_STR(config->args, where, "-label", "diffRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } if (!psDBTransaction(config->dbh)) { @@ -1288,4 +1294,35 @@ "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<="); + // other where restrictions: + PXOPT_COPY_TIME(config->args, selectWhere, "-dateobs_begin", "inputRawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, selectWhere, "-dateobs_end", "inputRawExp.dateobs", "<="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_min", "inputRawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-ra_max", "inputRawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_min", "inputRawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, selectWhere, "-decl_max", "inputRawExp.decl", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-sat_pixel_frac_min", "inputRawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-sat_pixel_frac_max", "inputRawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-airmass_min", "inputRawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-airmass_max", "inputRawExp.airmass", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-exp_time_min", "inputRawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-exp_time_max", "inputRawExp.exp_time", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_min", "inputRawExp.bg", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_max", "inputRawExp.bg", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_stdev_min", "inputRawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_stdev_max", "inputRawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_mean_stdev_min", "inputRawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-bg_mean_stdev_max", "inputRawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-alt_min", "inputRawExp.alt", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-alt_max", "inputRawExp.alt", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-az_min", "inputRawExp.az", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-az_max", "inputRawExp.az", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-ccd_temp_min", "inputRawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-ccd_temp_max", "inputRawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, selectWhere, "-posang_min", "inputRawExp.posang", ">="); + PXOPT_COPY_F64(config->args, selectWhere, "-posang_max", "inputRawExp.posang", "<"); + PXOPT_COPY_F32(config->args, selectWhere, "-sun_angle_min", "inputRawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, selectWhere, "-sun_angle_max", "inputRawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, selectWhere, "-comment", "inputRawExp.comment", "LIKE"); + // Haversine formula for great circle distance PXOPT_COPY_F32(config->args, selectWhere, "-distance", Index: branches/eam_branches/20090715/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/difftoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/difftoolConfig.c (revision 25022) @@ -133,4 +133,5 @@ psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdiffskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); // -definepoprun @@ -189,4 +190,33 @@ psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(definewarpwarpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); // -pendingcleanuprun Index: branches/eam_branches/20090715/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/faketool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/faketool.c (revision 25022) @@ -157,6 +157,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); @@ -325,6 +325,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); @@ -669,6 +669,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); if (!psListLength(where->list) Index: branches/eam_branches/20090715/ippTools/src/faketoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/faketoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/faketoolConfig.c (revision 25022) @@ -84,6 +84,6 @@ psMetadataAddF64(queueArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(queueArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -137,6 +137,6 @@ psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); @@ -235,6 +235,6 @@ psMetadataAddF64(revertprocessedimfileArgs, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(revertprocessedimfileArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); Index: branches/eam_branches/20090715/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magicdstool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/magicdstool.c (revision 25022) @@ -2,5 +2,5 @@ * magicdstool.c * - * Copyright (C) 2006-2007 IfA + * Copyright (C) 2006-2009 IfA * * This program is free software; you can redistribute it and/or modify it @@ -27,4 +27,5 @@ #include #include +#include #include "pxtools.h" @@ -97,119 +98,114 @@ static bool definebyqueryMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_UNKNOWN, true, "definebyquery not implelmented yet"); - - return false; -#ifdef notyet // Required + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + + // Optional PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); - - // Optional - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); + PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - // Create temporary table of the best diffs - { - psString query = pxDataGet("magictool_definebyquery_temp_create.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // search args + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); + + pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); + + ippStage stageNum = ippStringToStage(stage); + + psString query = NULL; + switch (stageNum) { + case IPP_STAGE_RAW: + query = pxDataGet("magicdstool_definebyquery_raw.sql"); + break; + case IPP_STAGE_CHIP: + query = pxDataGet("magicdstool_definebyquery_chip.sql"); + break; + case IPP_STAGE_CAMERA: + query = pxDataGet("magicdstool_definebyquery_camera.sql"); + break; + case IPP_STAGE_WARP: + query = pxDataGet("magicdstool_definebyquery_warp.sql"); + break; + case IPP_STAGE_DIFF: + query = pxDataGet("magicdstool_definebyquery_diff.sql"); + break; + case IPP_STAGE_FAKE: + case IPP_STAGE_STACK: + psError(PXTOOLS_ERR_DATA, true, "%sRuns do not need to be destreaked", stage); + return false; + case IPP_STAGE_NONE: + psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage); + return false; + default: + psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s", + stageNum, stage); + return false; + } + + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + char *rerun_flag = rerun ? "\n1 " : "\n0" ; + + if (stageNum != IPP_STAGE_DIFF) { + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag)) { psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); - } - - // Insert list of best diffs into temporary table - { - psString query = pxDataGet("magictool_definebyquery_temp_insert.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - psString whereClause = NULL; // WHERE conditions - if (psListLength(where->list)) { - whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&whereClause, "\n AND "); - } - psFree(where); - - if (!p_psDBRunQueryF(config->dbh, query, whereClause)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(whereClause); psFree(query); return false; } - psFree(whereClause); - psFree(query); - } - - // Get list of exposures ready to magic - { - psString query = pxDataGet("magictool_definebyquery_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - psString magicSkyCellNumsWhere = NULL; // WHERE conditions for magicSkyCellNums - { - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); - - if (psListLength(where->list)) { - magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL); - psStringPrepend(&magicSkyCellNumsWhere, "\n AND "); - } - psFree(where); - } - - // "available" means only concern ourselves with exposures that have all diffs completed, unless we're - // told to only take what's available. - // "new" means we want a new run even if there's already a magic run defined - PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); - PXOPT_LOOKUP_BOOL(new, config->args, "-new", false); - - psString queryWhere = NULL; // WHERE conditions for entire query - if (available) { - psStringAppend(&queryWhere, " WHERE num_done = num_todo"); - } - if (new) { - const char *newWhere = " magic_id IS NULL"; // String to add - if (queryWhere) { - psStringAppend(&queryWhere, " AND %s", newWhere); - } else { - psStringAppend(&queryWhere, " WHERE %s", newWhere); - } - } - if (queryWhere) { - psStringAppend(&query, " %s", queryWhere); - psFree(queryWhere); - } - - - if (!p_psDBRunQueryF(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) { + } else { + // diff stage query has two types bothways and !bothways + // so we need to send the rerun flag and the where data twice + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, rerun_flag, whereString, rerun_flag, whereString)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(magicSkyCellNumsWhere); + psFree(whereString); psFree(query); return false; } - psFree(magicSkyCellNumsWhere); - psFree(query); - } + psFree(whereString); + } + psFree(query); psArray *output = p_psDBFetchResult(config->dbh); @@ -233,72 +229,71 @@ } - // Parse the list of exposures ready to magic - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query + // Parse the list of runs ready to be destreaked + + if (!dry_run && !psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } psArray *list = psArrayAllocEmpty(16); // List of runs, to print for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; // Row of interest - psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier + psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); + psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id"); + psS64 inv_magic_id = psMetadataLookupS64(NULL, row, "inv_magic_id"); + psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); + psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); + + // if workdir is not supplied use the magicRun's + if (!workdir) { + workdir = magicRunWorkdir; + } + + psString outroot = NULL; + // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip + psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage); // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, "new", workdir, "dirty", label, dvodb, registered, 0); + magicDSRunRow *run = magicDSRunRowAlloc( + 0, // magic_ds_id + magic_id, + inv_magic_id, + "new", + stage, + stage_id, + cam_id, + set_label ? set_label : magicRunLabel, + outroot, + recoveryroot, + re_place, + 0); // remove + + psFree(outroot); if (!run) { - psAbort("failed to alloc magicRun object"); - } - - if (!magicRunInsertObject(config->dbh, run)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(run); - psFree(insert); - psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { + psAbort("failed to alloc magicDSRun object"); + } + + if (!dry_run) { + if (!magicDSRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + psFree(output); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; } - return false; - } - - psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier - run->magic_id = magic_id; + psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier + run->magic_ds_id = magic_ds_id; + } psArrayAdd(list, list->n, run); psFree(run); - - // Create a suitable insertion query for this run - psString thisInsert = psStringCopy(insert); - { - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, magic_id); - psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@"); - psFree(idString); - } - { - psString idString = NULL; - psStringAppend(&idString, "%" PRId64, exp_id); - psStringSubstitute(&thisInsert, idString, "@EXP_ID@"); - psFree(idString); - } - - if (!p_psDBRunQueryF(config->dbh, thisInsert, magic_id, exp_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(thisInsert); - psFree(insert); - psFree(output); - psFree(list); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(thisInsert); - } - - if (!psDBCommit(config->dbh)) { + } + + if (!dry_run && !psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -306,5 +301,5 @@ psFree(output); - if (!magicRunPrintObjects(stdout, list, !simple)) { + if (!magicDSRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -315,5 +310,4 @@ return true; -#endif // notyet } @@ -344,4 +338,5 @@ 0, // ID magic_id, + 0, // inv_magic_id "new", // state stage, @@ -405,4 +400,5 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -501,27 +497,33 @@ psMetadata *row = output->data[0]; - psString stage = psMetadataLookupStr(NULL, row, "stage"); + psString stage= psMetadataLookupStr(NULL, row, "stage"); psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); - if (!strcmp(stage, "camera")) { - // no there is no magicked column in camProcessedExp + ippStage stageNum = ippStringToStage(stage); + + // chose the appropriate query based on the stage + switch (stageNum) { + case IPP_STAGE_RAW: + query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; + break; + case IPP_STAGE_CHIP: + query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; + break; + case IPP_STAGE_CAMERA: + // no there is no magicked column in camProcessedExp so we have nothing to do psFree(output); return true; - } - - // chose the appropriate query based on the stage - if (!strcmp(stage, "raw")) { - query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; - } else if (!strcmp(stage, "chip")) { - query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; - } else if (!strcmp(stage, "warp")) { + case IPP_STAGE_WARP: query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'"; - } else if (!strcmp(stage, "diff")) { + break; + case IPP_STAGE_DIFF: query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'"; - } else { + break; + default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); psFree(output); return false; } + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -571,17 +573,24 @@ psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id"); + ippStage stageNum = ippStringToStage(stage); // chose the appropriate query based on the stage - if (!strcmp(stage, "raw")) { + switch (stageNum) { + case IPP_STAGE_RAW: query = "UPDATE rawExp SET magicked = %" PRId64 " where exp_id = %" PRId64; - } else if (!strcmp(stage, "chip")) { + break; + case IPP_STAGE_CHIP: query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64; - } else if (!strcmp(stage, "camera")) { + break; + case IPP_STAGE_CAMERA: query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64; - } else if (!strcmp(stage, "warp")) { + break; + case IPP_STAGE_WARP: query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64; - } else if (!strcmp(stage, "diff")) { + break; + case IPP_STAGE_DIFF: query = "UPDATE diffRun SET magicked = %" PRId64 " where diff_id = %" PRId64; - } else { + break; + default: psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); psFree(output); @@ -671,25 +680,9 @@ PS_ASSERT_PTR_NON_NULL(cam_id, false); - if (!strcmp(stage, "diff")) { - // don't need these ids for diff stage because diff_id is in the magicRun - *stage_id = 0; - *cam_id = 0; - return true; - } - - int stageNum; - if (!strcmp(stage, "raw")) { - stageNum = 0; - } else if (!strcmp(stage, "chip")) { - stageNum = 1; - } else if (!strcmp(stage, "camera")) { - stageNum = 2; - } else if (!strcmp(stage, "warp")) { - stageNum = 3; - } else { - psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage); - return false; - } - + int stageNum = ippStringToStage(stage);; + if (stageNum == IPP_STAGE_NONE) { + psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage); + return false; + } psString query = pxDataGet("magicdstool_getrunids.sql"); @@ -724,12 +717,20 @@ *cam_id = psMetadataLookupS64(NULL, row, "cam_id"); - if (stageNum == 0) { + switch (stageNum) { + case IPP_STAGE_RAW: *stage_id = psMetadataLookupS64(NULL, row, "exp_id"); - } else if (stageNum == 1) { + break; + case IPP_STAGE_CHIP: *stage_id = psMetadataLookupS64(NULL, row, "chip_id"); - } else if (stageNum == 2) { + break; + case IPP_STAGE_CAMERA: *stage_id = *cam_id; - } else if (stageNum == 3) { + return true; + case IPP_STAGE_WARP: *stage_id = psMetadataLookupS64(NULL, row, "warp_id"); + break; + case IPP_STAGE_DIFF: + *stage_id = psMetadataLookupS64(NULL, row, "diff_id"); + break; } Index: branches/eam_branches/20090715/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magicdstoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/magicdstoolConfig.c (revision 25022) @@ -49,21 +49,28 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); -#ifdef notyet - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -#endif + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0); + psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-streaks_max", 0, "maximum number of streaks", 0); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new run even if one arleady exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be queued", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -definerun psMetadata *definerunArgs = psMetadataAlloc(); psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define output directory (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for run (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); @@ -89,4 +96,5 @@ psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); Index: branches/eam_branches/20090715/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magictool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/magictool.c (revision 25022) @@ -47,4 +47,5 @@ static bool revertmaskMode(pxConfig *config); static bool maskMode(pxConfig *config); +static bool censorrunMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -85,4 +86,5 @@ MODECASE(MAGICTOOL_MODE_REVERTMASK, revertmaskMode); MODECASE(MAGICTOOL_MODE_MASK, maskMode); + MODECASE(MAGICTOOL_MODE_CENSORRUN, censorrunMode); default: psAbort("invalid option (this should not happen)"); @@ -580,4 +582,5 @@ PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0"); @@ -801,5 +804,7 @@ // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); + // cut limit in half + // hack to prevent pending leaf nodes from blocking branch nodes + psString limitString = psDBGenerateLimitSQL((limit + 1) / 2); psStringAppend(&query, " %s", limitString); psFree(limitString); @@ -847,7 +852,5 @@ // look for tree nodes that need to be processed - // XXX: This gets all nodes from all magicRuns that are in 'new'state - // That doens't seem particularly efficient - query = pxDataGet("magictool_toprocess_tree.sql"); + query = pxDataGet("magictool_toprocess_runs.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -855,4 +858,9 @@ } + { + psString limitString = psDBGenerateLimitSQL( 100 ); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { @@ -865,6 +873,6 @@ psFree(query); - psArray *magicTree = p_psDBFetchResult(config->dbh); - if (!magicTree) { + psArray *magicRuns = p_psDBFetchResult(config->dbh); + if (!magicRuns) { psErrorCode err = psErrorCodeLast(); switch (err) { @@ -879,42 +887,60 @@ return false; } - if (!psArrayLength(magicTree)) { + if (!psArrayLength(magicRuns)) { psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(magicTree); + psFree(magicRuns); return true; } - // entries are ordered by magic_id - long index = 0; - while (index < psArrayLength(magicTree)) { - bool status; + query = pxDataGet("magictool_toprocess_tree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + for (psS64 index = 0; index < psArrayLength(magicRuns); index++) { if (limit && (psArrayLength(output) >= limit)) { break; } - psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id"); + bool status; + psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id"); if (!status) { psAbort("failed to lookup value for magic_id column"); } - - // find the end of this block - long first = index; - long last = index; - for (long i = index + 1; i < psArrayLength(magicTree); i++) { - psS64 magic_id = psMetadataLookupS64(&status, magicTree->data[i], "magic_id"); - if (!status) { - psAbort("failed to lookup value for magic_id column"); + + whereString = NULL; + psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id); + if (!p_psDBRunQueryF(config->dbh, query, whereString )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + psFree(query); + return false; + } + psFree(whereString); + psArray *magicTree = p_psDBFetchResult(config->dbh); + if (!magicTree) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); } - if (magic_id != current_magic_id) { - break; - } - last = i; - } - - index = last + 1; - - psHash *forest = psHashAlloc(last - first + 1); + + return false; + } + psS64 length = psArrayLength(magicTree); + if (!length) { + psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id); + psFree(magicTree); + continue; + } + + psHash *forest = psHashAlloc(length); // convert the array of metadata into a pxTree structure - for (long i = first; i <= last; i++) { + for (long i = 0; i < length; i++) { bool status; psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); @@ -941,7 +967,6 @@ pxTreeCrawl(root, findReadyNodes, output); psFree(root); - - } - psFree(magicTree); + psFree(magicTree); + } if (psArrayLength(output)) { @@ -1367,2 +1392,111 @@ return true; } + +static bool censorStage(pxConfig *config, psString stage, psString whereClause) +{ + psString queryFile = NULL; + psStringAppend(&queryFile, "magictool_censor_%s.sql", stage); + psString query = pxDataGet(queryFile); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psFree(queryFile); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(queryFile); + + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + return true; +} + +static bool censorrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet"); + return false; + + psMetadata *where = psMetadataAlloc(); + + // at least one of these required + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + if (!psListLength(where->list)) { + psError(PS_ERR_UNKNOWN, true, "either -exp_id or -magic_id is required"); + psFree(where); + return false; + } + + psString query = psStringCopy("UPDATE magicRun SET state = 'censored'"); + + psString whereClause = psDBGenerateWhereConditionSQL(where, "magicRun"); + psFree(where); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(query); + + psS32 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated == 0) { + psError(PS_ERR_UNKNOWN, false, "failed to censor magicRun"); + psFree(whereClause); + return false; + } + + // Now queue any destreaked files to be re-verted + + // note: on failure censorStage issues the rollback + if (!censorStage(config, "raw", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "chip", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "camera", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "warp", whereClause)) { + psFree(whereClause); + return false; + } + if (!censorStage(config, "diff", whereClause)) { + psFree(whereClause); + return false; + } + + psFree(whereClause); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} Index: branches/eam_branches/20090715/ippTools/src/magictool.h =================================================================== --- branches/eam_branches/20090715/ippTools/src/magictool.h (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/magictool.h (revision 25022) @@ -41,8 +41,5 @@ MAGICTOOL_MODE_REVERTMASK, MAGICTOOL_MODE_MASK, - MAGICTOOL_MODE_DIFFSKYFILE, - MAGICTOOL_MODE_WARPSKYFILE, - MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, - MAGICTOOL_MODE_RAWIMFILE, + MAGICTOOL_MODE_CENSORRUN, } MAGICtoolMode; Index: branches/eam_branches/20090715/ippTools/src/magictoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/magictoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/magictoolConfig.c (revision 25022) @@ -108,4 +108,5 @@ psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0); psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(reverttreeArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); // -inputs @@ -160,30 +161,8 @@ psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -#ifdef notdef - // -diffskyfile - psMetadata *diffskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -warpskyfile - psMetadata *warpskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(warpskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddBool(warpskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddStr(warpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); - - // -chipprocessedimfile - psMetadata *chipprocessedimfileArgs = psMetadataAlloc(); - psMetadataAddS64(chipprocessedimfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(chipprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); - psMetadataAddBool(chipprocessedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -rawimfile - psMetadata *rawimfileArgs = psMetadataAlloc(); - psMetadataAddS64(rawimfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(rawimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); - psMetadataAddBool(rawimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -#endif // notdef + // -censorun + psMetadata *censorrunArgs = psMetadataAlloc(); + psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); + psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); psFree(now); @@ -208,10 +187,5 @@ PXOPT_ADD_MODE("-revertmask", "", MAGICTOOL_MODE_REVERTMASK, revertmaskArgs); PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); -#ifdef notdef - PXOPT_ADD_MODE("-diffskyfile", "", MAGICTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); - PXOPT_ADD_MODE("-warpskyfile", "", MAGICTOOL_MODE_WARPSKYFILE, warpskyfileArgs); - PXOPT_ADD_MODE("-chipprocessedimfile", "", MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileArgs); - PXOPT_ADD_MODE("-rawimfile", "", MAGICTOOL_MODE_RAWIMFILE, rawimfileArgs); -#endif + PXOPT_ADD_MODE("-censorrun", "", MAGICTOOL_MODE_CENSORRUN, censorrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090715/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pstamptool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/pstamptool.c (revision 25022) @@ -34,17 +34,17 @@ static bool datastoreMode(pxConfig *config); static bool moddatastoreMode(pxConfig *config); -static bool addReqMode(pxConfig *config); -static bool completedReqMode(pxConfig *config); -static bool listReqMode(pxConfig *config); -static bool pendingReqMode(pxConfig *config); -static bool updateReqMode(pxConfig *config); -static bool revertReqMode(pxConfig *config); -static bool addJobMode(pxConfig *config); -static bool listJobMode(pxConfig *config); -static bool pendingJobMode(pxConfig *config); -static bool updateJobMode(pxConfig *config); -static bool addProjectMode(pxConfig *config); +static bool addreqMode(pxConfig *config); +static bool completedreqMode(pxConfig *config); +static bool listreqMode(pxConfig *config); +static bool pendingreqMode(pxConfig *config); +static bool updatereqMode(pxConfig *config); +static bool revertreqMode(pxConfig *config); +static bool addjobMode(pxConfig *config); +static bool listjobMode(pxConfig *config); +static bool pendingjobMode(pxConfig *config); +static bool updatejobMode(pxConfig *config); +static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); -static bool modProjectMode(pxConfig *config); +static bool modprojectMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -70,16 +70,16 @@ MODECASE(PSTAMPTOOL_MODE_DATASTORE, datastoreMode); MODECASE(PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreMode); - MODECASE(PSTAMPTOOL_MODE_ADDREQ, addReqMode); - MODECASE(PSTAMPTOOL_MODE_COMPLETEDREQ, completedReqMode); - MODECASE(PSTAMPTOOL_MODE_LISTREQ, listReqMode); - MODECASE(PSTAMPTOOL_MODE_PENDINGREQ, pendingReqMode); - MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updateReqMode); - MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertReqMode); - MODECASE(PSTAMPTOOL_MODE_ADDJOB, addJobMode); - MODECASE(PSTAMPTOOL_MODE_LISTJOB, listJobMode); - MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingJobMode); - MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updateJobMode); - MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addProjectMode); - MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modProjectMode); + MODECASE(PSTAMPTOOL_MODE_ADDREQ, addreqMode); + MODECASE(PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqMode); + MODECASE(PSTAMPTOOL_MODE_LISTREQ, listreqMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGREQ, pendingreqMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updatereqMode); + MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertreqMode); + MODECASE(PSTAMPTOOL_MODE_ADDJOB, addjobMode); + MODECASE(PSTAMPTOOL_MODE_LISTJOB, listjobMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode); + MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); + MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); default: @@ -220,5 +220,5 @@ } -static bool addReqMode(pxConfig *config) +static bool addreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -251,5 +251,5 @@ } -static bool pendingReqMode(pxConfig *config) +static bool pendingreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -311,5 +311,5 @@ } -static bool listReqMode(pxConfig *config) +static bool listreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -359,5 +359,5 @@ } -static bool completedReqMode(pxConfig *config) +static bool completedreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -408,5 +408,5 @@ } -static bool updateReqMode(pxConfig *config) +static bool updatereqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -472,5 +472,5 @@ } -static bool revertReqMode(pxConfig *config) +static bool revertreqMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -493,5 +493,5 @@ } -static bool addJobMode(pxConfig *config) +static bool addjobMode(pxConfig *config) { bool stampJob = false; @@ -500,10 +500,8 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(uri, config->args, "-uri", true, false); PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); PXOPT_LOOKUP_STR(rownum, config->args, "-rownum", true, false); PXOPT_LOOKUP_STR(job_type, config->args, "-job_type", false, false); PXOPT_LOOKUP_STR(outputBase, config->args, "-outputBase", true, false); - PXOPT_LOOKUP_STR(argString, config->args, "-args", false, false); PXOPT_LOOKUP_STR(stateString, config->args, "-state", false, false); PXOPT_LOOKUP_STR(fault, config->args, "-fault", false, false); @@ -519,10 +517,4 @@ return false; } - - if (stampJob && !argString) { - psError(PS_ERR_UNKNOWN, true, "-args is required for stamp job"); - return false; - } - if (stampJob) { query = pxDataGet("pstamptool_addjob_stampjob.sql"); @@ -535,5 +527,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, uri, exp_id, outputBase, fault, argString)) { + if (!p_psDBRunQueryF(config->dbh, query, req_id, rownum, stateString, job_type, exp_id, outputBase, fault)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -554,5 +546,5 @@ } -static bool listJobMode(pxConfig *config) +static bool listjobMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -626,5 +618,5 @@ } -static bool pendingJobMode(pxConfig *config) +static bool pendingjobMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -688,5 +680,5 @@ } -static bool updateJobMode(pxConfig *config) +static bool updatejobMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -723,5 +715,5 @@ return true; } -static bool addProjectMode(pxConfig *config) +static bool addprojectMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -801,5 +793,5 @@ return true; } -static bool modProjectMode(pxConfig *config) +static bool modprojectMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); Index: branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/pstamptoolConfig.c (revision 25022) @@ -100,10 +100,8 @@ // -addjob psMetadata *addjobArgs = psMetadataAlloc(); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-uri", 0, "define job file uri", NULL); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-args", 0, "define job args", NULL); psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); @@ -128,5 +126,4 @@ psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0, "req_id for which to change state", 0); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); - // psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-result", 0, "new result", NULL); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); Index: branches/eam_branches/20090715/ippTools/src/pxcam.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxcam.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/pxcam.c (revision 25022) @@ -68,6 +68,6 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); return true; @@ -113,6 +113,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); return true; Index: branches/eam_branches/20090715/ippTools/src/pxchip.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/pxchip.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/pxchip.c (revision 25022) @@ -69,6 +69,6 @@ psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max", 0, "search by max solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); @@ -122,6 +122,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); Index: branches/eam_branches/20090715/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/regtool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/regtool.c (revision 25022) @@ -923,6 +923,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); Index: branches/eam_branches/20090715/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/regtoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/regtoolConfig.c (revision 25022) @@ -138,4 +138,5 @@ ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); + ADD_OPT(Bool, processedimfileArgs, "-ordered_by_date", "force output to be in DATE order", false); // -revertprocessedimfile @@ -273,6 +274,6 @@ psMetadataAddF64(processedexpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_min", 0, "define min solar angle", NAN); - psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-solang_max", 0, "define max solar angle", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF32(processedexpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/eam_branches/20090715/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/stacktool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/stacktool.c (revision 25022) @@ -133,6 +133,6 @@ PXOPT_COPY_F32(config->args, where, "-select_posang_min", "rawExp.posang", ">="); PXOPT_COPY_F32(config->args, where, "-select_posang_max", "rawExp.posang", "<="); - PXOPT_COPY_F32(config->args, where, "-select_solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-select_solang_max", "rawExp.solang", "<="); + PXOPT_COPY_F32(config->args, where, "-select_sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-select_sun_angle_max", "rawExp.sun_angle", "<="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_major_min", "camProcessedExp.fwhm_major", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_major_max", "camProcessedExp.fwhm_major", "<="); @@ -857,4 +857,6 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -931,94 +933,33 @@ PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - return false; - } - - int numStacks; // Number of stacks affected - - // Update state to 'new' - { - psString update = pxDataGet("stacktool_revertsumskyfile_update.sql"); - if (!update) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&update, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, update)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(update); - psFree(where); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(update); - - numStacks = psDBAffectedRows(config->dbh); - - if (numStacks < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; } // Delete product - { - psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); - if (!delete) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&delete, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, delete)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(delete); - psFree(where); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } + psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql"); + if (!delete) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(delete); - - if (psDBAffectedRows(config->dbh) != numStacks) { - psError(PS_ERR_UNKNOWN, true, "Updated and deleted different number of entries!"); - psFree(where); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - } - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows); psFree(where); Index: branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/stacktoolConfig.c (revision 25022) @@ -71,6 +71,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_min", 0, "define min solar angle", NAN); - psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_solang_max", 0, "define max solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_major_min", 0, "define min fwhm (major axis)", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_major_max", 0, "define max fwhm (major axis)", NAN); @@ -159,4 +159,6 @@ psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); @@ -169,4 +171,5 @@ psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertsumskyfileArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", 0); // -pendingcleanuprun Index: branches/eam_branches/20090715/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/warptool.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/warptool.c (revision 25022) @@ -234,6 +234,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); - PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); @@ -1190,4 +1190,6 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_skyfile_id", "warpImfile.warp_skyfile_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); Index: branches/eam_branches/20090715/ippTools/src/warptoolConfig.c =================================================================== --- branches/eam_branches/20090715/ippTools/src/warptoolConfig.c (revision 25020) +++ branches/eam_branches/20090715/ippTools/src/warptoolConfig.c (revision 25022) @@ -89,6 +89,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_min", 0, "search by min solar angle", NAN); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_max", 0, "search by max solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by fakeRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on fakeRun label", NULL); @@ -208,12 +208,13 @@ // -warped psMetadata *warpedArgs = psMetadataAlloc(); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure tag", 0); - psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure tag", 0); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "define phase 3 version of exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-warp_skyfile_id", 0, "search by skyfile ID", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertwarped Index: branches/eam_branches/20090715/ippconfig/gpc1/format_20090220.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/format_20090220.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/gpc1/format_20090220.config (revision 25022) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -501,6 +501,4 @@ # How to translation PS concepts into database lookups DATABASE METADATA - # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: branches/eam_branches/20090715/ippconfig/gpc1/ppImage.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/ppImage.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/gpc1/ppImage.config (revision 25022) @@ -132,4 +132,7 @@ END +# only apply the fringe for these filters +FRINGE.FILTERS MULTI +FRINGE.FILTERS STR y.00000 # Standard chip processing @@ -148,4 +151,5 @@ FLAT BOOL TRUE # Flat-field normalisation MASK BOOL TRUE # Mask bad pixels + PATTERN BOOL TRUE # Subtract pattern noise FRINGE BOOL FALSE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -852,4 +856,108 @@ END + +PR_DARKTEST_NOSUB METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? + + DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + DARK METADATA + DETTYPE STR DARKTEST + EXPTIME STR FPA.EXPOSURE + END + FLAT METADATA + FILTER STR FPA.FILTERID + END + FLATCORR METADATA + FILTER STR FPA.FILTERID + END + FRINGE METADATA + FILTER STR FPA.FILTERID + END + SHUTTER METADATA + END + MASK METADATA + END + ASTROM METADATA + END + NOISEMAP METADATA + END + END +END + + PR METADATA -END + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + PATTERN BOOL TRUE # Subtract pattern noise + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? +END + + +# Standard chip processing with pattern subtraction activated +CHIP_PATTERN METADATA + BASE.FITS BOOL TRUE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? + TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool + PATTERN BOOL TRUE # Subtract pattern noise + NOISEMAP BOOL TRUE # Apply read noise map +END + Index: branches/eam_branches/20090715/ippconfig/gpc1/ppMerge.config =================================================================== --- branches/eam_branches/20090715/ippconfig/gpc1/ppMerge.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/gpc1/ppMerge.config (revision 25022) @@ -18,8 +18,10 @@ ROWS S32 32 # Number of rows to read at once -FRINGE.NUM S32 10000 # Number of fringe regions -FRINGE.SIZE S32 5 # Half-size of fringe regions -FRINGE.XSMOOTH S32 5 # Number of smoothing regions in x -FRINGE.YSMOOTH S32 5 # Number of smoothing regions in y +FRINGE.NUM S32 1000 # Number of fringe regions per cell +FRINGE.SIZE S32 5 # Half-size of fringe regions +FRINGE.XSMOOTH S32 3 # Number of smoothing regions per cell in x +FRINGE.YSMOOTH S32 3 # Number of smoothing regions per cell in y +FRINGE.SMOOTH BOOL TRUE # Smooth the output image? +FRINGE.SMOOTH.SIGMA F32 2.0 # Sigma of smoothing Gaussian # Ordinates for fitting dark current @@ -30,11 +32,28 @@ # Ordinates for fitting dark current as function of darktime and fpa temp: +# DARK.ORDINATES METADATA +# CELL.DARKTIME S32 1 +# FPA.TEMP METADATA +# ORDER S32 1 +# SCALE BOOL TRUE +# RULE STR FPA.TEMP * CELL.DARKTIME +# MIN F32 -95 +# MAX F32 -50 +# END +# END + +# Ordinates for fitting dark current as function of darktime and fpa temp: +# Counts = C0 + (CT0 + CT1*Temp + CT2*Temp^2)*exptime +# the 0th order term is implied, right? +# Note: GPC1 has bad values for DARKTIME in unpredictable cells; use EXPOSURE for now DARK.ORDINATES METADATA - CELL.DARKTIME S32 1 - FPA.TEMP METADATA + CELL.EXPOSURE S32 1 + TEMP METADATA ORDER S32 1 - SCALE BOOL TRUE - MIN F32 -95 - MAX F32 -50 + RULE STR CHIP.TEMP * CELL.EXPOSURE + END + TEMP2 METADATA + ORDER S32 1 + RULE STR CHIP.TEMP * CHIP.TEMP * CELL.EXPOSURE END END Index: branches/eam_branches/20090715/ippconfig/megacam/ppImage.config =================================================================== --- branches/eam_branches/20090715/ippconfig/megacam/ppImage.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/megacam/ppImage.config (revision 25022) @@ -9,6 +9,6 @@ # for a test, turn off selected detrend types OVERSCAN BOOL FALSE # Overscan subtraction -BIAS BOOL FALSE # Bias subtraction -DARK BOOL TRUE # Dark subtraction +BIAS BOOL TRUE # Bias subtraction +DARK BOOL FALSE # Dark subtraction FLAT BOOL TRUE # Flat-field normalisation @@ -72,5 +72,5 @@ SHUTTER BOOL FALSE # Shutter correction FLAT BOOL TRUE # Flat-field normalisation - MASK BOOL TRUE # Mask bad pixels + MASK BOOL FALSE # Mask bad pixels FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -111,5 +111,5 @@ CHIP.MASK.FITS BOOL FALSE # Save chip-mosaic-ed image? CHIP.VARIANCE.FITS BOOL FALSE # Save chip-mosaic-ed image? - OVERSCAN BOOL TRUE # Overscan subtraction + OVERSCAN BOOL TRUE # Overscan subtraction BIAS BOOL TRUE # Bias subtraction DARK BOOL FALSE # Dark subtraction Index: branches/eam_branches/20090715/ippconfig/megacam/psphot.config =================================================================== --- branches/eam_branches/20090715/ippconfig/megacam/psphot.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/megacam/psphot.config (revision 25022) @@ -41,2 +41,4 @@ USE_FOOTPRINTS BOOL TRUE # use new pmFootprint peak packaging + +PEAKS_NSIGMA_LIMIT F32 10.0 Index: branches/eam_branches/20090715/ippconfig/recipes/ppImage.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppImage.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/ppImage.config (revision 25022) @@ -17,4 +17,5 @@ MASK.LOW BOOL TRUE # Mask pixels below valid range VARIANCE.BUILD BOOL FALSE # Build internal variance image +PATTERN BOOL FALSE # Fit and remove pattern noise? FRINGE BOOL FALSE # Fringe subtraction PHOTOM BOOL FALSE # Source identification and photometry @@ -25,4 +26,5 @@ USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool + TILTYSTREAK.BY.CLASS METADATA # apply the 'tiltystreak' tool @@ -78,4 +80,12 @@ OVERSCAN.CONSTANT BOOL FALSE # Apply a known, fixed value? OVERSCAN.VALUE F32 0.0 # value to apply, if requested + +# Pattern subtraction options +PATTERN.ORDER S32 2 # Polynomial order +PATTERN.ITER S32 4 # Rejection iterations +PATTERN.REJ F32 2.5 # Rejection threshold +PATTERN.THRESH F32 20.0 # Ignore threshold +PATTERN.MEAN STR SAMPLE_MEAN # Statistic for mean +PATTERN.STDEV STR SAMPLE_STDEV # Statistic for standard deviation # Fringe subtraction options @@ -1641,4 +1651,77 @@ #For PR images -PR METADATA -END +PR_DARKTEST_NOSUB METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + + DETREND.CONSTRAINTS METADATA + DARK METADATA + DETTYPE STR DARKTEST + EXPTIME STR FPA.EXPOSURE + END + END +END + +PR METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? +END + + +# Standard chip processing with pattern subtraction +CHIP_PATTERN METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + PATTERN BOOL TRUE # Subtract pattern noise? + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? +END Index: branches/eam_branches/20090715/ippconfig/recipes/ppMerge.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppMerge.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/ppMerge.config (revision 25022) @@ -10,8 +10,12 @@ NKEEP S32 5 # Minimum number of pixels in stack to keep VARIANCES BOOL FALSE # Use image variances in combination? + FRINGE.NUM S32 10000 # Number of fringe regions FRINGE.SIZE S32 5 # Half-size of fringe regions FRINGE.XSMOOTH S32 5 # Number of smoothing regions in x FRINGE.YSMOOTH S32 11 # Number of smoothing regions in y +FRINGE.SMOOTH BOOL FALSE # Smooth the output image? +FRINGE.SMOOTH.SIGMA F32 2.0 # Sigma of smoothing Gaussian + CTE.MIN F32 0.5 # regions lower than this in the CTE image are masked SHUTTER.SIZE S32 128 # Size for shutter measurement regions Index: branches/eam_branches/20090715/ippconfig/recipes/ppSim.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppSim.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/ppSim.config (revision 25022) @@ -13,6 +13,20 @@ BIAS.ORDER S32 7 # Order for overscan polynomial +# dark signal = (DARK.RATE + DARK.TEMP1*TEMP + DARK.TEMP2*TEMP^2)*EXPTIME DARK BOOL TRUE DARK.RATE F32 5.0 # Dark current (e/s) +DARK.TEMP F32 0.0 # Dark temperature trend (e/s/C) +DARK.TEMP2 F32 0.0 # 2nd order Dark temperature trend (e/s/C^2) +DET.TEMP F32 -20.0 # Detector temperature + +# dark signal = (DARK.RATE + DARK.TEMP1*TEMP + DARK.TEMP2*TEMP^2)*EXPTIME +# the following dark model can be used to test ppMerge with DARK.ORDINATES +TEST.DARKTEMP METADATA + DARK BOOL TRUE + DARK.RATE F32 25.0 # Dark current (e/s) + DARK.TEMP F32 0.5 # Dark temperature trend (e/s/C) + DARK.TEMP2 F32 0.05 # 2nd order Dark temperature trend (e/s/C^2) + DET.TEMP F32 -20.0 # Detector temperature +END FLAT BOOL TRUE Index: branches/eam_branches/20090715/ippconfig/recipes/ppStack.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppStack.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/ppStack.config (revision 25022) @@ -19,4 +19,5 @@ BIN1 S32 4 # Binning factor for first level BIN2 S32 4 # Binning factor for second level +SKIP.BG.SUB BOOL FALSE # Bypass background subtraction? RENORM BOOL FALSE # Renormalise variance maps? @@ -79,3 +80,4 @@ CONVOLVE BOOL FALSE IMAGE.REJ F32 0.5 + SKIP.BG.SUB BOOL FALSE END Index: branches/eam_branches/20090715/ippconfig/recipes/ppSub.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/ppSub.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/ppSub.config (revision 25022) @@ -18,4 +18,5 @@ MASK.POOR STR CONV.POOR # Mask value to give poor pixels +LOW.THRESHOLD F32 4.0 # Mask pixels below this threshold (std dev) POOR.FRACTION F32 0.10 # Maximum fraction of bad weight for poor pixels BADFRAC F32 0.95 # Maximum fraction of bad pixels Index: branches/eam_branches/20090715/ippconfig/recipes/psphot.config =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/psphot.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/psphot.config (revision 25022) @@ -216,4 +216,5 @@ FOOTPRINT_CULL_NSIGMA_DELTA F32 4 # Cull peaks that aren't nsigma above coll to neighbour FOOTPRINT_CULL_NSIGMA_MIN F32 1 # Minimum height of colls in units of skyStdev +FOOTPRINT_CULL_NSIGMA_PAD F32 0.01 # Fractional Padding for stdev # alternate PSPHOT-type recipes Index: branches/eam_branches/20090715/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/20090715/ippconfig/recipes/reductionClasses.mdc (revision 25020) +++ branches/eam_branches/20090715/ippconfig/recipes/reductionClasses.mdc (revision 25022) @@ -305,5 +305,5 @@ # Intended for fast processing to make PR images PR METADATA - CHIP_PPIMAGE STR CHIP + CHIP_PPIMAGE STR PR CHIP_PSPHOT STR PR WARP_PSWARP STR WARP @@ -322,5 +322,5 @@ # Intended for fast processing to make PR images PR_DARKTEST METADATA - CHIP_PPIMAGE STR CHIP_DARKTEST + CHIP_PPIMAGE STR PR_DARKTEST_NOSUB CHIP_PSPHOT STR PR WARP_PSWARP STR WARP @@ -336,2 +336,19 @@ PSASTRO STR PR_RECIPE END + +# Testing pattern subtraction +TEST_PATTERN METADATA + CHIP_PPIMAGE STR CHIP_PATTERN + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE +END Index: branches/eam_branches/20090715/ippconfig/simtest/Makefile.am =================================================================== --- branches/eam_branches/20090715/ippconfig/simtest/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/ippconfig/simtest/Makefile.am (revision 25022) @@ -14,4 +14,5 @@ ppSim.config \ ppSub.config \ + ppMerge.config \ ppStack.config Index: branches/eam_branches/20090715/ippconfig/simtest/camera.config =================================================================== --- branches/eam_branches/20090715/ippconfig/simtest/camera.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/simtest/camera.config (revision 25022) @@ -48,4 +48,5 @@ PPSTACK STR simtest/ppStack.config # ppStack recipe overrides PSASTRO STR simtest/psastro.config # psastro details + PPMERGE STR simtest/ppMerge.config # ppMerge details REJECTIONS STR simtest/rejections.config # Rejection for detrend creation END Index: branches/eam_branches/20090715/ippconfig/simtest/format.config =================================================================== --- branches/eam_branches/20090715/ippconfig/simtest/format.config (revision 25020) +++ branches/eam_branches/20090715/ippconfig/simtest/format.config (revision 25022) @@ -43,4 +43,5 @@ FPA.TIME STR DATE-OBS UTC-OBS # Date and time FPA.EXPOSURE STR EXPTIME + FPA.TEMP STR DETTEMP CELL.EXPOSURE STR EXPTIME CELL.DARKTIME STR DARKTIME Index: branches/eam_branches/20090715/ippconfig/simtest/ppMerge.config =================================================================== --- branches/eam_branches/20090715/ippconfig/simtest/ppMerge.config (revision 25022) +++ branches/eam_branches/20090715/ippconfig/simtest/ppMerge.config (revision 25022) @@ -0,0 +1,31 @@ + +# Ordinates for fitting dark current as function of darktime and fpa temp: +# Counts = C0 + (CT0 + CT1*Temp + CT2*Temp^2)*exptime +# the 0th order term is implied, right? +# DARK.ORDINATES METADATA +# CELL.DARKTIME S32 1 +# FPA.TEMP METADATA +# ORDER S32 1 +# RULE STR FPA.TEMP * CELL.DARKTIME +# END +# # FPA.TEMP2 METADATA +# # ORDER S32 1 +# # RULE STR FPA.TEMP * FPA.TEMP * CELL.DARKTIME +# # END +# END + + +# Ordinates for fitting dark current as function of darktime and fpa temp: +# Counts = C0 + (CT0 + CT1*Temp + CT2*Temp^2)*exptime +# the 0th order term is implied, right? +# DARK.ORDINATES METADATA +# CELL.DARKTIME S32 1 +# FPA.TEMP METADATA +# ORDER S32 1 +# RULE STR FPA.TEMP * CELL.DARKTIME +# END +# FPA.TEMP2 METADATA +# ORDER S32 1 +# RULE STR FPA.TEMP * FPA.TEMP * CELL.DARKTIME +# END +# END Index: branches/eam_branches/20090715/magic/Makefile.in =================================================================== --- branches/eam_branches/20090715/magic/Makefile.in (revision 25020) +++ branches/eam_branches/20090715/magic/Makefile.in (revision 25022) @@ -27,5 +27,5 @@ if [ -d magic ]; then (cd magic && make -f Makefile.magic); fi -magic.install: +magic.install: ssa-core-cpp if [ -d magic ]; then (cd magic && make -f Makefile.magic install); fi Index: branches/eam_branches/20090715/magic/remove/src/Line.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/Line.c (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/Line.c (revision 25022) @@ -444,9 +444,5 @@ if (y >= 0 && y < numRows) { - pixel = psAlloc (sizeof(PixelPos)); - pixel->x = (unsigned int) x; - pixel->y = (unsigned int) y; - psArrayAdd (pixels, 1024, pixel); - psFree (pixel); + psImageSet(pixels, x, y, 1); } } @@ -494,9 +490,5 @@ if (x >=0 && x < numCols) { - pixel = psAlloc (sizeof(PixelPos)); - pixel->x = (unsigned int) x; - pixel->y = (unsigned int) y; - psArrayAdd (pixels, 1024, pixel); - psFree (pixel); + psImageSet(pixels, x, y, 1); } } Index: branches/eam_branches/20090715/magic/remove/src/streaksextern.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksextern.c (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/streaksextern.c (revision 25022) @@ -34,5 +34,6 @@ int i; Line line; - StreakPixels *pixels = psArrayAllocEmpty (1024); + StreakPixels *pixels = psImageAlloc(numCols, numRows, PS_TYPE_U8); + psImageInit(pixels, 0); int streaksOnComponent = 0; Index: branches/eam_branches/20090715/magic/remove/src/streaksextern.h =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksextern.h (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/streaksextern.h (revision 25022) @@ -4,5 +4,5 @@ /** psLib includes */ -#include "psArray.h" +#include "psImage.h" /** streakremove includes */ @@ -36,5 +36,5 @@ /** For now, use psArray to define a list of PixelPos pointers */ -typedef psArray StreakPixels; +typedef psImage StreakPixels; /** Create a list of pixel positions from a Streaks pointer list Index: branches/eam_branches/20090715/magic/remove/src/streaksio.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksio.c (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/streaksio.c (revision 25022) @@ -9,4 +9,9 @@ static nebServer *ourNebServer = NULL; + +// Assumptions about the file structure of non-raw files +// The 'image' for each file (image, mask weight) is contained in the first +// extension. + // open the files required for streaks procesing. @@ -285,4 +290,6 @@ // copy header from fpu sfile->header = (psMetadata*) psMemIncrRefCounter(sfile->pmfile->fpa->hdu->header); + + sfile->nHDU = psFitsGetSize(sfile->pmfile->fits); return sfile; @@ -1198,2 +1205,59 @@ *maskMask = ~convPoor; } + +static void +doCopyExtraExtensions(streakFiles *sf, sFile *in, sFile *out) +{ + for (int extnum = sf->extnum; extnum < in->nHDU; extnum++) { + moveExt(in, extnum); + readImage(in, extnum, sf->stage, false); + + if (SFILE_IS_IMAGE(in)) { + // Turn off compression (code adapted from pmFPAWrite) +#ifdef SAVE_AND_RESTORE_COMPRESSION + int bitpix = out->fits->options ? out->fits->options->bitpix : 0; // Desired bits per pixel + psFitsCompression *compress = psFitsCompressionGet(out->fits); // Current compression options +#endif + if (!psFitsSetCompression(out->fits, PS_FITS_COMPRESS_NONE, NULL, 0, 0, 0)) { + psError(PM_ERR_UNKNOWN, false, "failed to turn off compression for extension %d\n", extnum); + streaksExit("", PS_EXIT_UNKNOWN_ERROR); + } +#ifdef SAVE_AND_RESTORE_COMPRESSION + if (out->fits->options) { + out->fits->options->bitpix = 0; + } + if (!psFitsWriteImage(out->fits, in->header, in->image, 0, in->extname)) { + psError(PM_ERR_UNKNOWN, false, "failed to write image for extension %d\n", extnum); + streaksExit("", PS_EXIT_UNKNOWN_ERROR); + } + if (out->fits->options) { + out->fits->options->bitpix = bitpix; + } + if (!psFitsCompressionApply(out->fits, compress)) { + psError(PM_ERR_UNKNOWN, false, "failed to rest compression image for extension %d\n", extnum); + streaksExit("", PS_EXIT_UNKNOWN_ERROR); + } +#endif + } else { + copyTable(out, in, extnum); + } + } +} + +void +copyExtraExtensions(streakFiles *sf) +{ + // XXX: Bad assumption, the begining of the mask and weight files have the same structure as the image + // So we are currently at the image portion + sf->extnum = sf->inImage->nHDU; + + if (sf->inWeight && sf->outWeight) { + doCopyExtraExtensions(sf, sf->inWeight, sf->outWeight); + } + if (sf->inMask && sf->outMask) { + doCopyExtraExtensions(sf, sf->inMask, sf->outMask); + } + if (sf->inChMask && sf->outChMask) { + doCopyExtraExtensions(sf, sf->inChMask, sf->outChMask); + } +} Index: branches/eam_branches/20090715/magic/remove/src/streaksio.h =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksio.h (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/streaksio.h (revision 25022) @@ -17,4 +17,5 @@ void setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll); void strkGetMaskValues(streakFiles *sfiles, psU32 *maskStreak, psU32 *maskMask); +void copyExtraExtensions(streakFiles *sfiles); void writeImage(sFile *sfile, psString extname, int extnum); Index: branches/eam_branches/20090715/magic/remove/src/streaksrelease.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksrelease.c (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/streaksrelease.c (revision 25022) @@ -67,4 +67,10 @@ printf("time to process component %d: %f\n", sfiles->extnum, psTimerClear("PROCESS_COMPONENT")); } while (streakFilesNextExtension(sfiles)); + + // check the weight and mask files for extra extensions that might be in files + // (covariance matrix for example) + copyExtraExtensions(sfiles); + + psTimerStart("CLOSE_IMAGES"); Index: branches/eam_branches/20090715/magic/remove/src/streaksremove.c =================================================================== --- branches/eam_branches/20090715/magic/remove/src/streaksremove.c (revision 25020) +++ branches/eam_branches/20090715/magic/remove/src/streaksremove.c (revision 25022) @@ -13,5 +13,5 @@ static bool readAndCopyToOutput(streakFiles *sf, bool exciseAll); static void exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue); -static bool warpedPixel(streakFiles *sfiles, PixelPos *cellCoord); +static bool warpedPixel(streakFiles *sfiles, int x, int y); static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue); static void writeImages(streakFiles *sf, bool exciseImageCube); @@ -146,21 +146,20 @@ } - totalStreakPixels += psArrayLength(pixels); psTimerStart("REMOVE_STREAKS"); - // for each pixel covered by the streak - for (int i = 0; i < psArrayLength (pixels); ++i) { - PixelPos *pixelPos = psArrayGet (pixels, i); - - // if this pixel was not part of the warp, skip because it has already been - // excised (unless we weren't asked to check) - if (!checkNonWarpedPixels || warpedPixel(sfiles, pixelPos)) { - - excisePixel(sfiles, pixelPos->x, pixelPos->y, true, maskStreak); - - } else { - // This pixel was not included in any warp and has thus already excised - // by exciseNonWarpedPixels + for (int y=0 ; y < sfiles->inImage->numRows; y++) { + for (int x = 0; x < sfiles->inImage->numCols; x++) { + if (psImageGet(pixels, x, y)) { + ++totalStreakPixels; + if (!checkNonWarpedPixels || warpedPixel(sfiles, x, y)) { + + excisePixel(sfiles, x, y, true, maskStreak); + + } else { + // This pixel was not included in any warp and has thus already excised + // by exciseNonWarpedPixels + } + } } } @@ -178,5 +177,4 @@ exciseImageCube = true; } - psArrayElementsFree (pixels); psFree(pixels); } @@ -198,7 +196,12 @@ } while (streakFilesNextExtension(sfiles)); + psFree(streaks); psLogMsg("streaksremove", PS_LOG_INFO, "pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, 100. * totalStreakPixels / totalPixels); + + // check the weight and mask files for extra extensions that might be in files + // (covariance matrix for example) + copyExtraExtensions(sfiles); // all done close the files. This is where the files are written so it can take a long time. @@ -751,5 +754,5 @@ static bool -warpedPixel(streakFiles *sfiles, PixelPos *cellCoord) +warpedPixel(streakFiles *sfiles, int x, int y) { PixelPos chipCoord; @@ -762,10 +765,10 @@ // we clip so that the streak calculation code doesn't have to // clipping here insures that we don't touch the overscan regions - if ((cellCoord->x < 0) || (cellCoord->x >= sfiles->inImage->numCols) || - (cellCoord->y < 0) || (cellCoord->y >= sfiles->inImage->numRows)) { + if ((x < 0) || (x >= sfiles->inImage->numCols) || + (y < 0) || (y >= sfiles->inImage->numRows)) { return false; } - cellToChipInt(&chipCoord.x, &chipCoord.y, sfiles->astrom, cellCoord->x, cellCoord->y); + cellToChipInt(&chipCoord.x, &chipCoord.y, sfiles->astrom, x, y); if (chipCoord.x < 0 || chipCoord.x >= sfiles->warpedPixels->numCols) { @@ -834,13 +837,20 @@ } - printf("Censored %d sources\n", numCensored); - // get rid of unused elements (don't know if this is necessary) psArrayRealloc(outTable, j); addDestreakKeyword(in->header); - if (! psFitsWriteTable(out->fits, in->header, outTable, extname)) { - psError(PS_ERR_IO, false, "failed to write table to %s", out->resolved_name); - streaksExit("", PS_EXIT_DATA_ERROR); + if (psArrayLength(outTable) > 0) { + printf("Censored %d sources\n", numCensored); + if (! psFitsWriteTable(out->fits, in->header, outTable, extname)) { + psError(PS_ERR_IO, false, "failed to write table to %s", out->resolved_name); + streaksExit("", PS_EXIT_DATA_ERROR); + } + } else { + printf("Censored ALL %d sources\n", numCensored); + if (! psFitsWriteTableEmpty(out->fits, in->header, inTable->data[0], extname)) { + psError(PS_ERR_IO, false, "failed to write empty table to %s", out->resolved_name); + streaksExit("", PS_EXIT_DATA_ERROR); + } } Index: branches/eam_branches/20090715/ppImage/src/ppImage.h =================================================================== --- branches/eam_branches/20090715/ppImage/src/ppImage.h (revision 25020) +++ branches/eam_branches/20090715/ppImage/src/ppImage.h (revision 25022) @@ -37,4 +37,5 @@ bool doShutter; // Shutter correction bool doFlat; // Flat-field normalisation + bool doPattern; // Pattern noise subtraction bool doFringe; // Fringe subtraction bool doPhotom; // Source identification and photometry @@ -44,5 +45,5 @@ bool doStats; // call ppStats on the image bool checkCTE; // measure pixel-based variance - bool applyParity; // Apply Cell parities + bool applyParity; // Apply Cell parities bool doCrosstalkMeasure; // measure crosstalk signal @@ -93,4 +94,12 @@ float fringeKeep; // Fringe keep fraction + // Pattern noise subtraction + int patternOrder; // Polynomial order + int patternIter; // Clipping iterations + float patternRej; // Clipping threshold + float patternThresh; // Ignore threshold + psStatsOptions patternMean; // Statistic for mean + psStatsOptions patternStdev; // Statistic for stdev + int remnanceSize; // Size for remnance detection float remnanceThresh; // Threshold for remnance detection Index: branches/eam_branches/20090715/ppImage/src/ppImageDetrendFringe.c =================================================================== --- branches/eam_branches/20090715/ppImage/src/ppImageDetrendFringe.c (revision 25020) +++ branches/eam_branches/20090715/ppImage/src/ppImageDetrendFringe.c (revision 25022) @@ -11,15 +11,9 @@ PS_ASSERT_PTR_NON_NULL(options, false); - // Reference fringe measurements - psArray *references = psMemIncrRefCounter(psMetadataLookupPtr(NULL, fringe->analysis, - "FRINGE.MEASUREMENTS")); + // Reference fringe measurements (stored on the reference cell->analysis) + psArray *references = psMemIncrRefCounter(psMetadataLookupPtr(NULL, fringe->analysis, "FRINGE.MEASUREMENTS")); if (!references) { - references = pmFringesParse(fringe); // Reference fringes - if (!references) { - psError(PS_ERR_IO, false, "Unable to find fringe references.\n"); - return false; - } - psMetadataAdd(fringe->analysis, PS_LIST_TAIL, "FRINGE.MEASUREMENTS", PS_DATA_UNKNOWN, - "Fringe measurements", references); + psError(PS_ERR_IO, false, "Unable to find fringe references.\n"); + return false; } @@ -75,6 +69,5 @@ } - psMetadataAdd(readout->parent->analysis, PS_LIST_TAIL, scienceFringes, - PS_DATA_UNKNOWN | PS_META_REPLACE, "Fringe measurements", measurements); + psMetadataAdd(readout->parent->analysis, PS_LIST_TAIL, scienceFringes, PS_DATA_UNKNOWN | PS_META_REPLACE, "Fringe measurements", measurements); psFree(measurements); psFree(scienceFringes); @@ -92,10 +85,12 @@ psArray *fringes = psArrayAlloc(cells->n); // Fringes, to return for (int i = 0; i < cells->n; i++) { + fringes->data[i] = NULL; + pmCell *cell = cells->data[i]; // Cell of interest // XXX for now, skip the video cells (cell->readouts->n > 1) if (cell->readouts->n > 1) { - psWarning ("Skipping Video Cell for ppImageDetrendFringe.c:getFringes"); - continue; + psWarning ("Skipping Video Cell (%d) for ppImageDetrendFringe.c:getFringes", i); + continue; } @@ -197,4 +192,6 @@ } assert(fringes->readouts->n == solution->nFringeFrames); + + psImageMaskType maskVal = options->flatMask; bool mdok; // Status of MD lookup @@ -262,9 +259,19 @@ } + // subtract fringe and update mask if fringe value is NAN + for (int iy = 0; iy < readout->image->numRows; iy++) { + for (int ix = 0; ix < readout->image->numCols; ix++) { + readout->image->data.F32[iy][ix] -= sumFringe->data.F32[iy][ix]; + if (!isfinite(sumFringe->data.F32[iy][ix]) && readout->mask) { + readout->mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= maskVal; + } + } + } + // XXX: Make generic, so subregions may be subtracted as well - if (!psBinaryOp(readout->image, readout->image, "-", sumFringe)) { - psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to subtract fringe.\n"); - return false; - } + // if (!psBinaryOp(readout->image, readout->image, "-", sumFringe)) { + // psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to subtract fringe.\n"); + // return false; + // } // measure residual fringe amplitude. results go to FRINGE.RESIDUALS Index: branches/eam_branches/20090715/ppImage/src/ppImageDetrendReadout.c =================================================================== --- branches/eam_branches/20090715/ppImage/src/ppImageDetrendReadout.c (revision 25020) +++ branches/eam_branches/20090715/ppImage/src/ppImageDetrendReadout.c (revision 25022) @@ -64,27 +64,27 @@ if (options->doVarianceBuild) { // create the target mask and variance images - psImage *noiseImage = NULL; - if (options->doNoiseMap) { - // XXX convert the noiseMap image to a binned image - pmReadout *noiseMap = NULL; - noiseMap = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.NOISEMAP"); - noiseImage = psImageCopy (NULL, input->image, PS_TYPE_F32); - psImageInit (noiseImage, 0.0); - - // XXX this works, but is not really quite right: the model shoud include the - // offset information, we are not really getting exactly the right mapping from the - // original file. - psImageBinning *binning = psImageBinningAlloc(); - binning->nXruff = noiseMap->image->numCols; - binning->nYruff = noiseMap->image->numRows; - binning->nXfine = input->image->numCols; - binning->nYfine = input->image->numRows; - psImageBinningSetScale(binning, PS_IMAGE_BINNING_LEFT); - - psImageUnbin (noiseImage, noiseMap->image, binning); - psFree (binning); - } + psImage *noiseImage = NULL; + if (options->doNoiseMap) { + // XXX convert the noiseMap image to a binned image + pmReadout *noiseMap = NULL; + noiseMap = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.NOISEMAP"); + noiseImage = psImageCopy (NULL, input->image, PS_TYPE_F32); + psImageInit (noiseImage, 0.0); + + // XXX this works, but is not really quite right: the model shoud include the + // offset information, we are not really getting exactly the right mapping from the + // original file. + psImageBinning *binning = psImageBinningAlloc(); + binning->nXruff = noiseMap->image->numCols; + binning->nYruff = noiseMap->image->numRows; + binning->nXfine = input->image->numCols; + binning->nYfine = input->image->numRows; + psImageBinningSetScale(binning, PS_IMAGE_BINNING_LEFT); + + psImageUnbin (noiseImage, noiseMap->image, binning); + psFree (binning); + } pmReadoutGenerateVariance(input, noiseImage, true); - psFree (noiseImage); + psFree (noiseImage); } @@ -119,4 +119,14 @@ pmReadout *flat = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.FLAT"); if (!pmFlatField(input, flat, options->flatMask)) { + psFree(detview); + return false; + } + } + + // Pattern noise correction + if (options->doPattern) { + if (!pmPatternRow(input, options->patternOrder, options->patternIter, options->patternRej, + options->patternThresh, options->patternMean, options->patternStdev, + options->maskValue, options->darkMask)) { psFree(detview); return false; Index: branches/eam_branches/20090715/ppImage/src/ppImageOptions.c =================================================================== --- branches/eam_branches/20090715/ppImage/src/ppImageOptions.c (revision 25020) +++ branches/eam_branches/20090715/ppImage/src/ppImageOptions.c (revision 25022) @@ -31,4 +31,5 @@ options->doShutter = false; // Shutter correction options->doFlat = false; // Flat-field normalisation + options->doPattern = false; // Pattern noise subtraction options->doFringe = false; // Fringe subtraction options->doPhotom = false; // Source identification and photometry @@ -86,4 +87,13 @@ options->fringeIter = 0; // Fringe iterations options->fringeKeep = 1.0; // Fringe keep fraction + + // Pattern noise values + + options->patternOrder = 0; // Polynomial order + options->patternIter = 0; // Clipping iterations + options->patternRej = NAN; // Clipping rejection threshold + options->patternThresh = NAN; // Threshold for ignoring pixels + options->patternMean = PS_STAT_NONE; // Statistic for mean + options->patternStdev = PS_STAT_NONE; // Statistic for standard deviation // Remnance values @@ -225,4 +235,5 @@ options->doFringe = psMetadataLookupBool(NULL, recipe, "FRINGE"); options->doShutter = psMetadataLookupBool(NULL, recipe, "SHUTTER"); + options->doPattern = psMetadataLookupBool(NULL, recipe, "PATTERN"); options->doStats = false; @@ -298,4 +309,12 @@ options->fringeKeep = psMetadataLookupF32(NULL, recipe, "FRINGE.KEEP"); + // Pattern noise + options->patternOrder = psMetadataLookupS32(NULL, recipe, "PATTERN.ORDER"); + options->patternIter = psMetadataLookupS32(NULL, recipe, "PATTERN.ITER"); + options->patternRej = psMetadataLookupF32(NULL, recipe, "PATTERN.REJ"); + options->patternThresh = psMetadataLookupF32(NULL, recipe, "PATTERN.THRESH"); + options->patternMean = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "PATTERN.MEAN")); + options->patternStdev = psStatsOptionFromString(psMetadataLookupStr(NULL, recipe, "PATTERN.STDEV")); + // Remnance options options->remnanceSize = psMetadataLookupF32(NULL, recipe, "REMNANCE.SIZE"); Index: branches/eam_branches/20090715/ppMerge/src/ppMerge.c =================================================================== --- branches/eam_branches/20090715/ppMerge/src/ppMerge.c (revision 25020) +++ branches/eam_branches/20090715/ppMerge/src/ppMerge.c (revision 25022) @@ -144,4 +144,5 @@ // Cleaning up + pmDarkVisualCleanup(); psFree(data); psFree(options); @@ -152,4 +153,5 @@ psTimerStop(); + pmDarkVisualCleanup(); psFree(config); pmConfigDone(); Index: branches/eam_branches/20090715/ppMerge/src/ppMergeArguments.c =================================================================== --- branches/eam_branches/20090715/ppMerge/src/ppMergeArguments.c (revision 25020) +++ branches/eam_branches/20090715/ppMerge/src/ppMergeArguments.c (revision 25022) @@ -170,4 +170,6 @@ psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-xsmooth", 0, "Number of smoothing regions in x", 0); psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-ysmooth", 0, "Number of smoothing regions in y", 0); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-fringe-smooth", 0, "Smooth output image", false); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-fringe-smooth-sigma", 0, "Size of smoothing Gaussian", NAN); /** CTEMASK construction parameters */ @@ -201,4 +203,9 @@ psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CELL_SELECTIONS", PS_DATA_STRING, "", argv[argnum]); psArgumentRemove (argnum, &argc, argv); + } + + if ((argnum = psArgumentGet (argc, argv, "-visual"))) { + psArgumentRemove (argnum, &argc, argv); + pmVisualSetVisual(true); } @@ -324,4 +331,6 @@ VALUE_ARG_RECIPE_INT("-fringe-xsmooth", "FRINGE.XSMOOTH", S32, 0); VALUE_ARG_RECIPE_INT("-fringe-ysmooth", "FRINGE.YSMOOTH", S32, 0); + VALUE_ARG_RECIPE_BOOL("-fringe-smooth", "FRINGE.SMOOTH"); + VALUE_ARG_RECIPE_FLOAT("-fringe-smooth-sigma", "FRINGE.SMOOTH.SIGMA", F32); /** CTEMASK construction parameters */ @@ -371,4 +380,5 @@ bool scale = false; ///< Scale values? float min = NAN, max = NAN; ///< Minimum and maximum values for scaling + char *rule = NULL; switch (item->type) { case PS_TYPE_S32: @@ -381,4 +391,5 @@ min = psMetadataLookupF32(&mdok, item->data.md, "MIN"); max = psMetadataLookupF32(&mdok, item->data.md, "MAX"); + rule = psMetadataLookupStr(&mdok, item->data.md, "RULE"); break; default: @@ -398,4 +409,5 @@ ord->min = min; ord->max = max; + ord->rule = psStringCopy(rule); psArrayAdd(translated, translated->n, ord); psFree(ord); Index: branches/eam_branches/20090715/ppMerge/src/ppMergeLoop.c =================================================================== --- branches/eam_branches/20090715/ppMerge/src/ppMergeLoop.c (revision 25020) +++ branches/eam_branches/20090715/ppMerge/src/ppMergeLoop.c (revision 25022) @@ -65,4 +65,6 @@ int fringeSmoothX = psMetadataLookupS32(NULL, arguments, "FRINGE.XSMOOTH"); // Smoothing regions in x int fringeSmoothY = psMetadataLookupS32(NULL, arguments, "FRINGE.YSMOOTH"); // Smoothing regions in y + bool fringeSmooth = psMetadataLookupBool(NULL, arguments, "FRINGE.SMOOTH"); // Smooth the output image? + float fringeSmoothSigma = psMetadataLookupF32(NULL, arguments, "FRINGE.SMOOTH.SIGMA"); // Smooth the output image? // set the mask and mark bit values based on the named masks @@ -150,4 +152,7 @@ pmChip *outChip; ///< Chip of interest while ((outChip = pmFPAviewNextChip(view, outFPA, 1))) { + if (!outChip->process || !outChip->file_exists) { + continue; + } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { goto ERROR; @@ -159,4 +164,7 @@ while ((outCell = pmFPAviewNextCell(view, outFPA, 1))) { + if (!outCell->process || !outCell->file_exists) { + continue; + } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { goto ERROR; @@ -416,6 +424,15 @@ // Put the new readout into the cell after the existing readouts. if (type == PPMERGE_TYPE_FRINGE && outRO) { - pmFringeRegions *regions = pmFringeRegionsAlloc(fringeNum, fringeSize, fringeSize, - fringeSmoothX, fringeSmoothY); + if (fringeSmooth) { + if (outRO->mask) { + psImage *smoothed = psImageSmoothMask (NULL, outRO->image, outRO->mask, maskVal, fringeSmoothSigma, 3, 0.2); + psFree (outRO->image); + outRO->image = smoothed; + } else { + psImageSmooth (outRO->image, fringeSmoothSigma, 3); + } + } + + pmFringeRegions *regions = pmFringeRegionsAlloc(fringeNum, fringeSize, fringeSize, fringeSmoothX, fringeSmoothY); pmFringeStats *fringe = pmFringeStatsMeasure(regions, outRO, maskVal); psFree(regions); @@ -429,5 +446,9 @@ fringes->data[0] = fringe; - pmFringesFormat(outCell, NULL, fringes); + // XXX replaced this : pmFringesFormat(outCell, NULL, fringes); + if (!psMetadataAdd(outCell->analysis, PS_LIST_TAIL, "FRINGE.MEASUREMENTS", PS_DATA_ARRAY, "Fringes", fringes)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add fringe to analysis metadata\n"); + goto ERROR; + } psFree(fringes); // Drop reference } Index: branches/eam_branches/20090715/ppMerge/src/ppMergeScaleZero.c =================================================================== --- branches/eam_branches/20090715/ppMerge/src/ppMergeScaleZero.c (revision 25020) +++ branches/eam_branches/20090715/ppMerge/src/ppMergeScaleZero.c (revision 25022) @@ -67,7 +67,7 @@ pmChip *chip; ///< Chip of interest while ((chip = pmFPAviewNextChip(view, fpa, 1))) { - if (!chip->file_exists) { - continue; - } + if (!chip->process || !chip->file_exists) { + continue; + } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { goto ERROR; @@ -76,7 +76,7 @@ pmCell *cell; ///< Cell of interest while ((cell = pmFPAviewNextCell(view, fpa, 1))) { - if (!cell->file_exists) { - continue; - } + if (!cell->process || !cell->file_exists) { + continue; + } if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { goto ERROR; @@ -89,6 +89,6 @@ // skip cells with video data if (cell->readouts->n > 1) { - // psError(PS_ERR_BAD_PARAMETER_VALUE, true, "File %d chip %d cell %d contains more than one readout (%ld)", i, view->chip, view->cell, cell->readouts->n); - // goto ERROR; + // psError(PS_ERR_BAD_PARAMETER_VALUE, true, "File %d chip %d cell %d contains more than one readout (%ld)", i, view->chip, view->cell, cell->readouts->n); + // goto ERROR; psWarning("File %d chip %d cell %d contains more than one readout (%ld), skipping", i, view->chip, view->cell, cell->readouts->n); continue; Index: branches/eam_branches/20090715/ppMops/src/ppMops.c =================================================================== --- branches/eam_branches/20090715/ppMops/src/ppMops.c (revision 25020) +++ branches/eam_branches/20090715/ppMops/src/ppMops.c (revision 25022) @@ -139,6 +139,6 @@ psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", alt); psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", az); - psMetadataAddF64(outHeader, PS_LIST_TAIL, "DIFFIMID", 0, "Difference image identifier", imageid); - psMetadataAddF64(outHeader, PS_LIST_TAIL, "FPA_ID", 0, "Exposure identifier", fpaid); + psMetadataAddS32(outHeader, PS_LIST_TAIL, "DIFFIMID", 0, "Difference image identifier", imageid); + psMetadataAddS32(outHeader, PS_LIST_TAIL, "FPA_ID", 0, "Exposure identifier", fpaid); psMetadataAddStr(outHeader, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE); psMetadataAddF32(outHeader, PS_LIST_TAIL, "STARPSF", 0, "Stellar PSF (arcsec)", psf); Index: branches/eam_branches/20090715/ppSim/src/ppSimArguments.c =================================================================== --- branches/eam_branches/20090715/ppSim/src/ppSimArguments.c (revision 25020) +++ branches/eam_branches/20090715/ppSim/src/ppSimArguments.c (revision 25022) @@ -36,4 +36,5 @@ psMetadataAddF32(arguments, PS_LIST_TAIL, "-biaslevel", 0, "Bias level (e)", NAN); psMetadataAddF32(arguments, PS_LIST_TAIL, "-biasrange", 0, "Bias range (e)", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-dettemp", 0, "Detector Temperature (C)", NAN); psMetadataAddF32(arguments, PS_LIST_TAIL, "-darkrate", 0, "Dark rate (e/s)", NAN); psMetadataAddF32(arguments, PS_LIST_TAIL, "-flatsigma", 0, "Flat sigma", NAN); @@ -189,4 +190,5 @@ ppSimArgToRecipeF32(&status, options, "BIAS.RANGE", arguments, "-biasrange"); ppSimArgToRecipeF32(&status, options, "DARK.RATE", arguments, "-darkrate"); + ppSimArgToRecipeF32(&status, options, "DET.TEMP", arguments, "-dettemp"); ppSimArgToRecipeF32(&status, options, "FLAT.SIGMA", arguments, "-flatsigma"); ppSimArgToRecipeF32(&status, options, "FLAT.RATE", arguments, "-flatrate"); Index: branches/eam_branches/20090715/ppSim/src/ppSimMakeDark.c =================================================================== --- branches/eam_branches/20090715/ppSim/src/ppSimMakeDark.c (revision 25020) +++ branches/eam_branches/20090715/ppSim/src/ppSimMakeDark.c (revision 25022) @@ -20,4 +20,11 @@ } + float detTemp = psMetadataLookupF32(&mdok, recipe, "DET.TEMP"); // Detector Temperature + float darkTemp = psMetadataLookupF32(&mdok, recipe, "DARK.TEMP"); // Dark temperature trend + float darkTemp2 = psMetadataLookupF32(&mdok, recipe, "DARK.TEMP2"); // Dark temperature trend 2nd order term + if (!isfinite(detTemp)) detTemp = 0.0; + if (!isfinite(darkTemp)) darkTemp = 0.0; + if (!isfinite(darkTemp2)) darkTemp2 = 0.0; + float expTime = psMetadataLookupF32(&mdok, recipe, "EXPTIME"); // Exposure time if (isnan(expTime)) { @@ -26,11 +33,11 @@ } - psTrace("ppSim", 6, "darkRate: %f, expTime: %f\n", darkRate, expTime); + // Dark current + float darkCurrent = (darkRate + darkTemp*detTemp + darkTemp2*PS_SQR(detTemp)) * expTime; // Dark current accumulated + + psTrace("ppSim", 6, "dark current: %f, darkRate: %f, darkTempTrend: %f, darkTempSecondOrder: %f, detTemp: %f, expTime: %f\n", darkCurrent, darkRate, darkTemp, darkTemp2, detTemp, expTime); for (int y = 0; y < signal->numRows; y++) { for (int x = 0; x < signal->numCols; x++) { - - // Dark current - float darkCurrent = darkRate * expTime; // Dark current accumulated signal->data.F32[y][x] += darkCurrent; variance->data.F32[y][x] += darkCurrent; Index: branches/eam_branches/20090715/ppSim/src/ppSimUtils.c =================================================================== --- branches/eam_branches/20090715/ppSim/src/ppSimUtils.c (revision 25020) +++ branches/eam_branches/20090715/ppSim/src/ppSimUtils.c (revision 25022) @@ -127,4 +127,5 @@ float expTime = psMetadataLookupF32(NULL, recipe, "EXPTIME"); // Exposure time + float detTemp = psMetadataLookupF32(NULL, recipe, "DET.TEMP"); // Detector Temperature const char *filter = psMetadataLookupStr(NULL, recipe, "FILTER"); // Filter name @@ -136,11 +137,12 @@ char *obs_mode = psMetadataLookupStr(NULL, recipe, "OBS_MODE"); - psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBSTYPE", PS_META_REPLACE, "Observation type", typeStr); - psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBJECT", PS_META_REPLACE, "Observation name", typeStr); + psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBSTYPE", PS_META_REPLACE, "Observation type", typeStr); + psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBJECT", PS_META_REPLACE, "Observation name", typeStr); psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBS.MODE", PS_META_REPLACE, "Observation mode", obs_mode); psMetadataAddF32(fpa->concepts, PS_LIST_TAIL, "FPA.EXPOSURE", PS_META_REPLACE, "Exposure time (sec)", expTime); + psMetadataAddF32(fpa->concepts, PS_LIST_TAIL, "FPA.TEMP", PS_META_REPLACE, "Detector Temperature (C)", detTemp); psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.FILTERID", PS_META_REPLACE, "Filter name", filter); - psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.FILTER", PS_META_REPLACE, "Filter name", filter); - psMetadataAddF32(fpa->concepts, PS_LIST_TAIL, "FPA.AIRMASS", PS_META_REPLACE, "airmass (dummy value)", 1.0); + psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.FILTER", PS_META_REPLACE, "Filter name", filter); + psMetadataAddF32(fpa->concepts, PS_LIST_TAIL, "FPA.AIRMASS", PS_META_REPLACE, "airmass (dummy value)", 1.0); return true; Index: branches/eam_branches/20090715/ppStack/src/ppStackMatch.c =================================================================== --- branches/eam_branches/20090715/ppStack/src/ppStackMatch.c (revision 25020) +++ branches/eam_branches/20090715/ppStack/src/ppStackMatch.c (revision 25022) @@ -512,17 +512,20 @@ psBinaryOp(readout->variance, readout->variance, "*", psScalarAlloc(PS_SQR(norm), PS_TYPE_F32)); } - + // Ensure the background value is zero psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for background psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator if (!psImageBackground(bg, NULL, readout->image, readout->mask, maskVal | maskBad, rng)) { - psWarning("Can't measure background for image."); - psErrorClear(); + psWarning("Can't measure background for image."); + psErrorClear(); } else { - psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)", - psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV)); - (void)psBinaryOp(readout->image, readout->image, "-", - psScalarAlloc(psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32)); - } + if (!psMetadataLookupBool(NULL, config->arguments, "PPSTACK.SKIP.BG.SUB")) { + psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)", + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV)); + (void)psBinaryOp(readout->image, readout->image, "-", + psScalarAlloc(psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32)); + } + } + // Measure the variance level for the weighting Index: branches/eam_branches/20090715/ppSub/src/Makefile.am =================================================================== --- branches/eam_branches/20090715/ppSub/src/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/Makefile.am (revision 25022) @@ -45,5 +45,6 @@ ppSubReadoutStats.c \ ppSubReadoutSubtract.c \ - ppSubSetMasks.c + ppSubSetMasks.c \ + ppSubThreshold.c ppSubKernel_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSUB_CFLAGS) Index: branches/eam_branches/20090715/ppSub/src/ppSub.h =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSub.h (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/ppSub.h (revision 25022) @@ -76,4 +76,8 @@ /// looking up the parameters in the recipe and supplying them to the function pmSubtractionMatch() bool ppSubMatchPSFs(ppSubData *data ///< Processing data + ); + +/// Threshold low pixels in image +bool ppSubLowThreshold(ppSubData *data ///< Processing data ); Index: branches/eam_branches/20090715/ppSub/src/ppSubArguments.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubArguments.c (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/ppSubArguments.c (revision 25022) @@ -84,4 +84,5 @@ psMetadataAddS32(arguments, PS_LIST_TAIL, "-threads", 0, "Number of threads", 0); psMetadataAddStr(arguments, PS_LIST_TAIL, "-dumpconfig", 0, "file to dump configuration to", NULL); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-convolve", 0, "Image to convolve [1 or 2]", 0); psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Perform photometry?", NULL); psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Generate inverse subtractions?", NULL); Index: branches/eam_branches/20090715/ppSub/src/ppSubLoop.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubLoop.c (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/ppSubLoop.c (revision 25022) @@ -62,4 +62,9 @@ if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) { psError(PPSUB_ERR_IO, false, "Unable to close input files."); + return false; + } + + if (!ppSubLowThreshold(data)) { + psError(PS_ERR_UNKNOWN, false, "Unable to threshold images."); return false; } Index: branches/eam_branches/20090715/ppSub/src/ppSubMatchPSFs.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubMatchPSFs.c (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/ppSubMatchPSFs.c (revision 25022) @@ -18,5 +18,4 @@ #include #include -#include #include "ppSub.h" @@ -153,5 +152,24 @@ bool dual = psMetadataLookupBool(&mdok, recipe, "DUAL"); // Dual convolution? - pmSubtractionMode subMode = dual ? PM_SUBTRACTION_MODE_DUAL : PM_SUBTRACTION_MODE_UNSURE; // Subtracn mode + pmSubtractionMode subMode; // Subtraction mode + if (dual) { + subMode = PM_SUBTRACTION_MODE_DUAL; + } else { + int convolve = psMetadataLookupS32(NULL, config->arguments, "-convolve"); // Image number to convolve + switch (convolve) { + case 0: + subMode = PM_SUBTRACTION_MODE_UNSURE; + break; + case 1: + subMode = PM_SUBTRACTION_MODE_1; + break; + case 2: + subMode = PM_SUBTRACTION_MODE_2; + break; + default: + psErrorStackPrint(stderr, "Invalid value for -convolve"); + return false; + } + } int threads = psMetadataLookupS32(NULL, config->arguments, "-threads"); // Number of threads Index: branches/eam_branches/20090715/ppSub/src/ppSubReadout.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubReadout.c (revision 25020) +++ (revision ) @@ -1,67 +1,0 @@ -/** @file ppSubReadout.c - * - * @brief - * - * @ingroup ppSub - * - * @author IfA - * @version $Revision: 1.113 $ $Name: not supported by cvs2svn $ - * @date $Date: 2009-02-18 00:31:20 $ - * Copyright 2009 Institute for Astronomy, University of Hawaii - */ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include - -#include "ppSub.h" - -bool ppSubReadout(const char *name, bool reverse, ppSubData *data, const pmFPAview *view) -{ - psAssert(data, "Require processing data"); - pmConfig *config = data->config; // Configuration - psAssert(config, "Require configuration"); - - psAssert(name, "Require name"); - psAssert(view, "Require view"); - - if (!ppSubDefineOutput(name, config, data, view)) { - psError(PS_ERR_UNKNOWN, false, "Unable to define output."); - return false; - } - - if (!data->quality && !ppSubMakePSF(name, config, data, view)) { - psError(PS_ERR_UNKNOWN, false, "Unable to generate PSF."); - return false; - } - - if (!ppSubReadoutSubtract(name, reverse, config, view)) { - psError(PS_ERR_UNKNOWN, false, "Unable to subtract images."); - return false; - } - - // Higher order background subtraction using psphot - if (!ppSubBackground(name, config, view)) { - psError(PS_ERR_UNKNOWN, false, "Unable to subtract background."); - return false; - } - - if (!data->quality && data->!ppSubReadoutPhotometry(name, config, data, view)) { - psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry."); - return false; - } - - if (!ppSubReadoutUpdate(name, config, data, view)) { - psError(PS_ERR_UNKNOWN, false, "Unable to update."); - return false; - } - - - - - return true; -} Index: branches/eam_branches/20090715/ppSub/src/ppSubReadoutPhotometry.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubReadoutPhotometry.c (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/ppSubReadoutPhotometry.c (revision 25022) @@ -95,6 +95,11 @@ } + // If no sources were found, there's no error, but we want to trigger 'bad quality' + psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources + if (!sources) { + ppSubDataQuality(data, PSPHOT_ERR_DATA, PPSUB_FILES_PHOT_SUB | PPSUB_FILES_PHOT_INV); + } + if (data->stats) { - psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources bool mdok; int numSources = psMetadataLookupS32(&mdok, data->stats, "NUM_SOURCES"); // Number of sources Index: branches/eam_branches/20090715/ppSub/src/ppSubReadoutSubtract.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubReadoutSubtract.c (revision 25020) +++ branches/eam_branches/20090715/ppSub/src/ppSubReadoutSubtract.c (revision 25022) @@ -44,20 +44,4 @@ subtrahend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); } - -// XXX this should be replaced by SAVE options to the inConv and refConf pmFPAfiles -#ifdef TESTING - { - pmReadoutMaskApply(minuend, maskVal); - psFits *fits = psFitsOpen("minuend.fits", "w"); - psFitsWriteImage(fits, NULL, minuend->image, 0, NULL); - psFitsClose(fits); - } - { - pmReadoutMaskApply(subtrahend, maskVal); - psFits *fits = psFitsOpen("subtrahend.fits", "w"); - psFitsWriteImage(fits, NULL, subtrahend->image, 0, NULL); - psFitsClose(fits); - } -#endif // Do the actual subtraction Index: branches/eam_branches/20090715/ppSub/src/ppSubThreshold.c =================================================================== --- branches/eam_branches/20090715/ppSub/src/ppSubThreshold.c (revision 25022) +++ branches/eam_branches/20090715/ppSub/src/ppSubThreshold.c (revision 25022) @@ -0,0 +1,114 @@ +/** @file ppSubThreshold.c + * + * @brief Mask pixels below threshold + * + * @ingroup ppSub + * + * @author IfA + * @version $Revision: 1.3 $ + * @date $Date: 2009-02-18 00:31:20 $ + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppSub.h" + +// Apply low threshold to image +static bool lowThreshold( + pmReadout *ro, // Readout to threshold + float thresh, // Threshold to apply + psImageMaskType maskIgnore, // Ignore pixels with this mask + psImageMaskType maskThresh, // Give pixels this mask if below threshold + const char *description // Description of image + ) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PM_ASSERT_READOUT_IMAGE(ro, false); + PM_ASSERT_READOUT_MASK(ro, false); + PS_ASSERT_FLOAT_LARGER_THAN(thresh, 0.0, false); + + psImage *image = ro->image; // Image + psImage *mask = ro->mask; // Mask + int numCols = ro->image->numCols, numRows = ro->image->numRows; // Size of image + + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(stats, NULL, image, mask, maskIgnore, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine threshold."); + psFree(rng); + psFree(stats); + return false; + } + psFree(rng); + + float threshold = stats->robustMedian - thresh * stats->robustStdev; // Threshold below which to clip + psFree(stats); + psLogMsg("ppSub", PS_LOG_INFO, "Masking pixels below %f in %s", threshold, description); + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskIgnore) { + continue; + } + if (image->data.F32[y][x] < threshold) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskThresh; + } + } + } + + return true; +} + + +bool ppSubLowThreshold(ppSubData *data) +{ + psAssert(data, "Require processing data"); + pmConfig *config = data->config; // Configuration + psAssert(config, "Require configuration"); + + // Look up recipe values + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim + psAssert(recipe, "We checked this earlier, so it should be here."); + + psImageMaskType maskVal = pmConfigMaskGet("MASK.VALUE", config); // Bits to mask in inputs + psImageMaskType maskThresh = pmConfigMaskGet("LOW", config); // Bits to mask for low pixels + + float thresh = psMetadataLookupF32(NULL, recipe, "LOW.THRESHOLD"); // Threshold for masking + if (!isfinite(thresh)) { + return true; + } + + pmFPAview *view = ppSubViewReadout(); // View to readout + + // Input images + pmReadout *in = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input image + if (!in) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find readout."); + return false; + } + if (!lowThreshold(in, thresh, maskVal, maskThresh, "input convolved image")) { + psError(PS_ERR_UNKNOWN, false, "Unable to threshold input image."); + return false; + } + + pmReadout *ref = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference image + if (!ref) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find readout."); + return false; + } + if (!lowThreshold(ref, thresh, maskVal, maskThresh, "reference convolved image")) { + psError(PS_ERR_UNKNOWN, false, "Unable to threshold input image."); + return false; + } + + psFree(view); + + return true; +} Index: branches/eam_branches/20090715/psLib/src/fits/psFits.c =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFits.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/fits/psFits.c (revision 25022) @@ -214,6 +214,7 @@ errorMsg = errorBuf; # endif - psError (PS_ERR_IO, true, "Failed to delete a previously-existing file: %s", errorMsg); - fprintf (stderr, "errno: %d, %s, %s : %lx, %lx\n", thisErrno, errorMsg, errorBuf, (long int) errorMsg, (long int) errorBuf); + psError(PS_ERR_IO, true, "Failed to delete a previously-existing file (%s), error %d: %s", + name, thisErrno, errorMsg); + //fprintf(stderr, "errno: %d, %s, %s : %lx, %lx\n", thisErrno, errorMsg, errorBuf, (long int) errorMsg, (long int) errorBuf); return NULL; } Index: branches/eam_branches/20090715/psLib/src/fits/psFitsHeader.c =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFitsHeader.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/fits/psFitsHeader.c (revision 25022) @@ -147,5 +147,4 @@ return keyword; } - bool psFitsCheckCompressedImagePHU(const psFits *fits, psMetadata *header) @@ -497,11 +496,14 @@ static bool fitsWriteHeader(psFits *fits, // The FITS file handle const psMetadata *output, // Metadata that is to be output into the FITS file - bool keyStarts // Write out the key starts? + bool keyStarts, // Write out the key starts? + bool phuImage // Are we writing a PHU image? ) { int status = 0; // Status of cfitsio calls + int extnum = psFitsGetExtNum(fits); // Number of extension bool simple = true; // If SIMPLE is T, then the file should conform to the FITS standard psFitsCompressionType compress = psFitsCompressionGetType(fits); // Compression type - if (psFitsGetExtNum(fits) == 0) { + if (extnum == 0) { + // We allow the user to write SIMPLE, but it must be boolean psMetadataItem *simpleItem = psMetadataLookup(output, "SIMPLE"); // SIMPLE in the header @@ -526,5 +528,4 @@ bool compressing = ((!fits->options || fits->options->conventions.compression) && compress != PS_FITS_COMPRESS_NONE) ? true : false; // Are we compressing? - if (compressing) { psMetadataItem *simpleItem = psMetadataLookup(output, "SIMPLE"); // SIMPLE in the header @@ -538,7 +539,14 @@ } } - // int value = simple; // Temporary holder for boolean - // fits_update_key(fits->fd, TLOGICAL, "ZSIMPLE", &value, - // "Uncompressed file's conformance to FITS standard", &status); + if (simple && phuImage && extnum == 1) { + // ZSIMPLE is required for decompression with funpack, etc. For funpack to work, ZSIMPLE needs to + // go early in the FITS header (otherwise we get "Extension doesn't start with SIMPLE or XTENSION + // keyword."). We put it after ZIMAGE by reading ZIMAGE (which sets the insertion pointer) and + // then inserting ZSIMPLE. + char comment[FLEN_CARD]; // Comment for ZIMAGE; unused + int value; // Value for ZIMAGE; unused + fits_read_key(fits->fd, TLOGICAL, "ZIMAGE", &value, comment, &status); + fits_insert_key_log(fits->fd, "ZSIMPLE", simple, "Uncompressed file's conforms to FITS", &status); + } } @@ -699,5 +707,17 @@ PS_ASSERT_METADATA_NON_NULL(output, false); - return fitsWriteHeader(fits, output, true); + return fitsWriteHeader(fits, output, true, false); +} + +bool psFitsWriteHeaderImage(psFits *fits, + const psMetadata *output, + bool phuImage + ) +{ + PS_ASSERT_FITS_NON_NULL(fits, false); + PS_ASSERT_FITS_WRITABLE(fits, false); + PS_ASSERT_METADATA_NON_NULL(output, false); + + return fitsWriteHeader(fits, output, true, phuImage); } @@ -733,5 +753,5 @@ } - if (output && !fitsWriteHeader(fits, output, false)) { + if (output && !fitsWriteHeader(fits, output, false, false)) { psError(PS_ERR_IO, false, "Unable to write FITS header.\n"); return false; Index: branches/eam_branches/20090715/psLib/src/fits/psFitsHeader.h =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFitsHeader.h (revision 25020) +++ branches/eam_branches/20090715/psLib/src/fits/psFitsHeader.h (revision 25022) @@ -66,4 +66,14 @@ ); +/** Write a header for an image + * Principal difference with psFitsWriteHeader is this allows writing ZSIMPLE for uncompressing a single image + */ +bool psFitsWriteHeaderImage( + psFits *fits, ///< FITS file + const psMetadata *output, ///< Header to output + bool phuImage ///< Is this image supposed to be in the PHU? + ); + + /** Writes a "blank" --- a header only, with no image or table. * Index: branches/eam_branches/20090715/psLib/src/fits/psFitsImage.c =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFitsImage.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/fits/psFitsImage.c (revision 25022) @@ -439,4 +439,8 @@ p_psFitsReadInfo *info = p_psFitsReadInfoAlloc(fits, region, z); + if (!info) { + psError(PS_ERR_IO, false, "Unable to read FITS information"); + return NULL; + } // Size of image @@ -612,4 +616,6 @@ } + bool createPHU = false; // Are we creating a PHU? + // Create the image HDU int hdus = psFitsGetSize(fits); // Number of HDUs in file @@ -617,4 +623,5 @@ // We're creating the first image fits_create_img(fits->fd, bitPix, naxis, naxes, &status); + createPHU = true; } else { if (!after) { @@ -623,4 +630,5 @@ // Set status to signal fits_insert_img to insert a new primary HDU status = PREPEND_PRIMARY; + createPHU = true; } else { // Move back one to perform an insert after the previous HDU @@ -637,5 +645,5 @@ // write the header, if any. - if (header && !psFitsWriteHeader(fits, header)) { + if (header && !psFitsWriteHeaderImage(fits, header, createPHU)) { psError(PS_ERR_IO, false, "Unable to write FITS header.\n"); success = false; Index: branches/eam_branches/20090715/psLib/src/fits/psFitsScale.c =================================================================== --- branches/eam_branches/20090715/psLib/src/fits/psFitsScale.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/fits/psFitsScale.c (revision 25022) @@ -46,4 +46,6 @@ psAssert(image, "impossible"); psAssert(options, "impossible"); + + psTrace("psLib.fits", 3, "Scaling image to preserve dynamic range"); double range = pow(2.0, options->bitpix) - 1.0; // Range of values for target BITPIX, reduced by the BLANK @@ -109,4 +111,6 @@ psAssert(options, "impossible"); + psTrace("psLib.fits", 3, "Scaling image by statistics"); + // Measure the mean and stdev // psImageBackground automatically excludes pixels that are non-finite, so we don't need to bother about a @@ -131,6 +135,10 @@ } + psTrace("psLib.fits", 5, "Mean: %lf Stdev: %lf", mean, stdev); + long range = 1 << options->stdevBits; // Range of values to carry standard deviation *bscale = stdev / (double) range; + + psTrace("psLib.fits", 5, "Number of bits: %ld BSCALE: %lf", range, *bscale); double imageVal; // Value on image @@ -157,4 +165,6 @@ *bzero = imageVal - *bscale * diskVal; + + psTrace("psLib.fits", 5, "Image %lf corresponds to disk %ld --> BZERO: %lf", imageVal, diskVal, *bzero); return true; Index: branches/eam_branches/20090715/psLib/src/imageops/psImageCovariance.c =================================================================== --- branches/eam_branches/20090715/psLib/src/imageops/psImageCovariance.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/imageops/psImageCovariance.c (revision 25022) @@ -282,4 +282,55 @@ } +psKernel *psImageCovarianceAverageWeighted(const psArray *array, const psVector *weights) +{ + PS_ASSERT_ARRAY_NON_NULL(array, NULL); + PS_ASSERT_ARRAY_NON_EMPTY(array, NULL); + if (!weights) { + return psImageCovarianceAverage(array); + } + PS_ASSERT_VECTOR_TYPE(weights, PS_TYPE_F32, NULL); + + int xMin = INT_MAX, xMax = INT_MIN, yMin = INT_MAX, yMax = INT_MIN; // Range for covariance + double sumWeights = 0.0; // Sum of weights + for (int i = 0; i < array->n; i++) { + psKernel *covar = array->data[i]; // Covariance matrix + if (!covar) { + continue; + } + xMin = PS_MIN(xMin, covar->xMin); + xMax = PS_MAX(xMax, covar->xMax); + yMin = PS_MIN(yMin, covar->yMin); + yMax = PS_MAX(yMax, covar->yMax); + sumWeights += weights->data.F32[i]; + } + if (sumWeights == 0) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, "No covariance matrices supplied for summation"); + return NULL; + } + + psKernel *sum = psKernelAlloc(xMin, xMax, yMin, yMax); // Summed covariance + for (int i = 0; i < array->n; i++) { + psKernel *covar = array->data[i]; // Covariance matrix + if (!covar) { + continue; + } + for (int y = covar->yMin; y <= covar->yMax; y++) { + for (int x = covar->xMin; x <= covar->xMax; x++) { + if (!isfinite(covar->kernel[y][x])) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Non-finite covariance matrix element at %d,%d for input %d", + x, y, i); + psFree(sum); + return NULL; + } + sum->kernel[y][x] += weights->data.F32[i] * covar->kernel[y][x]; + } + } + } + psBinaryOp(sum->image, sum->image, "/", psScalarAlloc((float)sumWeights, PS_TYPE_F32)); + + return sum; +} + psKernel *psImageCovarianceTruncate(const psKernel *covar, float frac) Index: branches/eam_branches/20090715/psLib/src/imageops/psImageCovariance.h =================================================================== --- branches/eam_branches/20090715/psLib/src/imageops/psImageCovariance.h (revision 25020) +++ branches/eam_branches/20090715/psLib/src/imageops/psImageCovariance.h (revision 25022) @@ -57,4 +57,10 @@ ); +/// Weighted average of multiple covariance pseudo-matrices +psKernel *psImageCovarianceAverageWeighted( + const psArray *array, ///< Array of covariance pseudo-matrices + const psVector *weights ///< Weights for each (F32) + ); + /// Truncate covariance pseudo-matrix /// Index: branches/eam_branches/20090715/psLib/src/math/psPolynomialMD.c =================================================================== --- branches/eam_branches/20090715/psLib/src/math/psPolynomialMD.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/math/psPolynomialMD.c (revision 25022) @@ -321,5 +321,4 @@ } -// XXX this function should take a (psVectorMaskType markVal) argument bool psPolynomialMDClipFit(psPolynomialMD *poly, const psVector *values, const psVector *errors, const psVector *mask, psVectorMaskType maskVal, const psArray *coordsArray, Index: branches/eam_branches/20090715/psLib/src/math/psStats.c =================================================================== --- branches/eam_branches/20090715/psLib/src/math/psStats.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/math/psStats.c (revision 25022) @@ -129,15 +129,15 @@ # define COUNT_WARNING(LIMIT, INTERVAL, ...) { \ - static int nCalls = 1; \ - if (nCalls < LIMIT) { \ - psWarning(__VA_ARGS__); \ - } \ - if (!(nCalls % INTERVAL)) { \ - psWarning(__VA_ARGS__); \ + static int nCalls = 1; \ + if (nCalls < LIMIT) { \ + psWarning(__VA_ARGS__); \ + } \ + if (!(nCalls % INTERVAL)) { \ + psWarning(__VA_ARGS__); \ psWarning("(warning raised %d times)", nCalls); \ - } \ - nCalls ++; \ + } \ + nCalls ++; \ } - + /*****************************************************************************/ @@ -217,6 +217,6 @@ for (long i = 0; i < numData; i++) { // Check if the data is with the specified range - if (!isfinite(data[i])) - continue; + if (!isfinite(data[i])) + continue; if (useRange && (data[i] < stats->min)) continue; @@ -242,5 +242,5 @@ if (!isnan(mean)) { - stats->results |= PS_STAT_SAMPLE_MEAN; + stats->results |= PS_STAT_SAMPLE_MEAN; } return true; @@ -278,6 +278,6 @@ for (long i = 0; i < num; i++) { // Check if the data is with the specified range - if (!isfinite(vector[i])) - continue; + if (!isfinite(vector[i])) + continue; if (useRange && (vector[i] < stats->min)) continue; @@ -329,6 +329,6 @@ // into the temporary vectors. for (long i = 0; i < inVector->n; i++) { - if (!isfinite(input[i])) - continue; + if (!isfinite(input[i])) + continue; if (useRange && (input[i] < stats->min)) continue; @@ -353,5 +353,5 @@ // Sort the temporary vector. if (!psVectorSort(outVector, outVector)) { // Sort in-place (since it's a copy, it's OK) - // an error in psVectorSort is a serious error + // an error in psVectorSort is a serious error psError(PS_ERR_UNEXPECTED_NULL, false, _("Failed to sort input data.")); stats->sampleUQ = NAN; @@ -407,5 +407,5 @@ // If the mean is NAN, then generate a warning and set the stdev to NAN. if (isnan(stats->sampleMean)) { - COUNT_WARNING(10, 100, "WARNING: vectorSampleStdev(): sample mean is NAN. Setting stats->sampleStdev = NAN."); + COUNT_WARNING(10, 100, "WARNING: vectorSampleStdev(): sample mean is NAN. Setting stats->sampleStdev = NAN."); stats->sampleStdev = NAN; return true; @@ -425,6 +425,6 @@ for (long i = 0; i < myVector->n; i++) { // Check if the data is with the specified range - if (!isfinite(data[i])) - continue; + if (!isfinite(data[i])) + continue; if (useRange && (data[i] < stats->min)) { continue; @@ -502,6 +502,6 @@ for (long i = 0; i < myVector->n; i++) { // Check if the data is with the specified range - if (!isfinite(data[i])) - continue; + if (!isfinite(data[i])) + continue; if (useRange && (data[i] < stats->min)) { continue; @@ -771,5 +771,5 @@ stats->robustLQ = min; stats->robustN50 = numValid; - // XXX this is sort of an invalid / out-of-bounds result: to set or not to set these bits: + // XXX this is sort of an invalid / out-of-bounds result: to set or not to set these bits: stats->results |= PS_STAT_ROBUST_MEDIAN; stats->results |= PS_STAT_ROBUST_STDEV; @@ -801,12 +801,12 @@ // XXXXX we need to consider this step if errors -> variance if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) { - // if psVectorHistogram returns false, we have a programming error + // if psVectorHistogram returns false, we have a programming error psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for robust statistics.\n"); - psFree(histogram); - psFree(cumulative); - psFree(statsMinMax); - psFree(mask); - - return false; + psFree(histogram); + psFree(cumulative); + psFree(statsMinMax); + psFree(mask); + + return false; } if (psTraceGetLevel("psLib.math") >= 8) { @@ -838,9 +838,9 @@ // ADD step 3: Interpolate to the exact 50% position in bin units - stats->robustMedian = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0); - // float robustBin = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0); - // fprintf (stderr, "robustBin : %f vs %f\n", robustBin, stats->robustMedian); - - // convert bin to bin value: this is the robust histogram median. + stats->robustMedian = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0); + // float robustBin = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0); + // fprintf (stderr, "robustBin : %f vs %f\n", robustBin, stats->robustMedian); + + // convert bin to bin value: this is the robust histogram median. if (isnan(stats->robustMedian)) { COUNT_WARNING(10, 100, "Failed to fit a quadratic and calculate the 50-percent position.\n"); @@ -1054,6 +1054,6 @@ if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) { psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n"); - return false; - } + return false; + } } @@ -1107,5 +1107,5 @@ psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers) if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) { - // if psVectorHistogram returns false, we have a programming error + // if psVectorHistogram returns false, we have a programming error psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for fitted statistics.\n"); psFree(histogram); @@ -1172,6 +1172,6 @@ PS_VECTOR_PRINT_F32(y); } - - // psMinimizeLMChi2 can return false for bad data as well as for serious failures + + // psMinimizeLMChi2 can return false for bad data as well as for serious failures if (!psMinimizeLMChi2(minimizer, NULL, params, NULL, x, y, NULL, minimizeLMChi2Gauss1D)) { psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n"); @@ -1235,6 +1235,6 @@ if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) { psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n"); - return false; - } + return false; + } } @@ -1355,5 +1355,5 @@ // psVectorInit (fitMask, 0); - // XXX not sure if these should result in errors or not... + // XXX not sure if these should result in errors or not... if (!psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x)) { psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n"); @@ -1432,6 +1432,6 @@ if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) { psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n"); - return false; - } + return false; + } } @@ -1729,14 +1729,10 @@ if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) { psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n"); - return false; - } + return false; + } } // If the mean is NAN, then generate a warning and set the stdev to NAN. - if (isnan(stats->robustMedian)) { - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; - return true; - } + if (isnan(stats->robustMedian)) goto escape; float guessStdev = stats->robustStdev; // pass the guess sigma @@ -1770,6 +1766,14 @@ COUNT_WARNING(10, 100, "Failed to calculate the min/max of the input vector.\n"); psFree(statsMinMax); - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; + goto escape; + } + + // If all data points have the same value, then we set the appropriate members of stats and return. + if (fabs(max - min) <= FLT_EPSILON) { + COUNT_WARNING(10, 100, "All data points have the same value: %f.\n", min); + stats->fittedMean = min; + stats->fittedStdev = 0.0; + stats->results |= PS_STAT_FITTED_MEAN_V4; + stats->results |= PS_STAT_FITTED_STDEV_V4; return true; } @@ -1789,7 +1793,5 @@ psFree(histogram); psFree(statsMinMax); - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; - return true; + goto escape; } if (psTraceGetLevel("psLib.math") >= 8) { @@ -1823,7 +1825,5 @@ COUNT_WARNING(10, 100, "Failed to calculate the min/max of the input vector.\n"); psFree(statsMinMax); - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; - return true; + goto escape; } @@ -1886,6 +1886,6 @@ // fit 2nd order polynomial to ln(y) = -(x-xo)^2/2sigma^2 - // XXX this fit may fail with an error for an ill-conditioned matrix (bad data) - // we probably should be able to handle the data errors gracefully + // XXX this fit may fail with an error for an ill-conditioned matrix (bad data) + // we probably should be able to handle the data errors gracefully psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); bool status = psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x); @@ -1898,7 +1898,5 @@ psFree(histogram); psFree(statsMinMax); - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; - return true; + goto escape; } @@ -1919,7 +1917,5 @@ COUNT_WARNING(10, 100, "fit did not converge\n"); - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; - return true; + goto escape; } @@ -1991,7 +1987,5 @@ psFree(histogram); psFree(statsMinMax); - stats->fittedStdev = NAN; - stats->fittedStdev = NAN; - return true; + goto escape; } @@ -2038,4 +2032,12 @@ psTrace(TRACE, 6, "The fitted stdev is %f.\n", stats->fittedStdev); + stats->results |= PS_STAT_FITTED_MEAN_V4; + stats->results |= PS_STAT_FITTED_STDEV_V4; + + return true; + +escape: + stats->fittedMean = NAN; + stats->fittedStdev = NAN; stats->results |= PS_STAT_FITTED_MEAN_V4; stats->results |= PS_STAT_FITTED_STDEV_V4; @@ -2896,7 +2898,7 @@ *****************************************************************************/ static psF32 fitQuadraticSearchForYThenReturnBin(const psVector *xVec, - psVector *yVec, - psS32 binNum, - psF32 yVal + psVector *yVec, + psS32 binNum, + psF32 yVal ) { @@ -2980,27 +2982,35 @@ } - // I believe that mathematically the fitted bin position must be between binNum - 1 and binNum + 1 - assert (binValue >= binNum - 1); - assert (binValue <= binNum + 1); - - int fitBin = binValue; - float dX = xVec->data.F32[fitBin+1] - xVec->data.F32[fitBin]; - float dY = binValue - fitBin; - tmpFloat = xVec->data.F32[fitBin] + dY * dX; + // I believe that mathematically the fitted bin position must be between binNum - 1 and binNum + 1 + assert (binValue >= binNum - 1); + assert (binValue <= binNum + 1); + + int fitBin = binValue; + float dX = xVec->data.F32[fitBin+1] - xVec->data.F32[fitBin]; + float dY = binValue - fitBin; + tmpFloat = xVec->data.F32[fitBin] + dY * dX; } else { // These are special cases where the bin is at the beginning or end of the vector. if (binNum == 0) { // We have two points only at the beginning of the vectors x and y. - // X = (dX/dY)(Y - Yo) + Xo - float dX = xVec->data.F32[1] - xVec->data.F32[0]; - float dY = yVec->data.F32[1] - yVec->data.F32[0]; - tmpFloat = (yVal - yVec->data.F32[0]) * (dX / dY) + xVec->data.F32[0]; + // X = (dX/dY)(Y - Yo) + Xo + float dX = xVec->data.F32[1] - xVec->data.F32[0]; + float dY = yVec->data.F32[1] - yVec->data.F32[0]; + if (dY == 0.0) { + tmpFloat = xVec->data.F32[0]; + } else { + tmpFloat = (yVal - yVec->data.F32[0]) * (dX / dY) + xVec->data.F32[0]; + } } else if (binNum == (xVec->n - 1)) { // We have two points only at the end of the vectors x and y. - // X = (dX/dY)(Y - Yo) + Xo - float dX = xVec->data.F32[binNum] - xVec->data.F32[binNum-1]; - float dY = yVec->data.F32[binNum] - yVec->data.F32[binNum-1]; - tmpFloat = (yVal - yVec->data.F32[binNum-1]) * (dX / dY) + xVec->data.F32[binNum-1]; - } + // X = (dX/dY)(Y - Yo) + Xo + float dX = xVec->data.F32[binNum] - xVec->data.F32[binNum-1]; + float dY = yVec->data.F32[binNum] - yVec->data.F32[binNum-1]; + if (dY == 0.0) { + tmpFloat = xVec->data.F32[binNum-1]; + } else { + tmpFloat = (yVal - yVec->data.F32[binNum-1]) * (dX / dY) + xVec->data.F32[binNum-1]; + } + } } Index: branches/eam_branches/20090715/psLib/src/mathtypes/psVector.c =================================================================== --- branches/eam_branches/20090715/psLib/src/mathtypes/psVector.c (revision 25020) +++ branches/eam_branches/20090715/psLib/src/mathtypes/psVector.c (revision 25022) @@ -122,5 +122,4 @@ } - if (vector->nalloc == nalloc) { // No need to realloc to same size @@ -131,5 +130,5 @@ elementSize = PSELEMTYPE_SIZEOF(elemType); - long nstart = vector->n; + long nallocOld = vector->nalloc; if (nalloc < vector->n) { vector->n = nalloc; @@ -139,8 +138,8 @@ P_PSVECTOR_SET_NALLOC(vector,nalloc); - // fill newly allocated range with zeros: - if (nstart < nalloc) { - long nNew = nalloc - nstart; - memset (&vector->data.U8[nstart*elementSize], 0, nNew*elementSize); + // fill newly allocated range with zeros: + if (nallocOld < nalloc) { + long nNew = nalloc - nallocOld; + memset (&vector->data.U8[nallocOld*elementSize], 0, nNew*elementSize); } Index: branches/eam_branches/20090715/psModules/src/astrom/pmAstrometryVisual.c =================================================================== --- branches/eam_branches/20090715/psModules/src/astrom/pmAstrometryVisual.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/astrom/pmAstrometryVisual.c (revision 25022) @@ -17,7 +17,7 @@ #include "pmFPAExtent.h" -# if (HAVE_KAPA) -# include -# include "pmKapaPlots.h" +#if (HAVE_KAPA) +#include +#include "pmKapaPlots.h" #include "pmVisual.h" Index: branches/eam_branches/20090715/psModules/src/camera/Makefile.am =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/psModules/src/camera/Makefile.am (revision 25022) @@ -23,4 +23,5 @@ pmFPAfileIO.c \ pmFPAfileFitsIO.c \ + pmFPAfileFringeIO.c \ pmFPAFlags.c \ pmFPALevel.c \ @@ -51,4 +52,5 @@ pmFPAfileIO.h \ pmFPAfileFitsIO.h \ + pmFPAfileFringeIO.h \ pmFPAFlags.h \ pmFPALevel.h \ Index: branches/eam_branches/20090715/psModules/src/camera/pmFPABin.c =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/pmFPABin.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/camera/pmFPABin.c (revision 25022) @@ -60,4 +60,7 @@ continue; } + if (!isfinite(inImage->data.F32[y][x])) { + continue; + } sum += inImage->data.F32[y][x]; numPix++; Index: branches/eam_branches/20090715/psModules/src/camera/pmFPAWrite.c =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/pmFPAWrite.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/camera/pmFPAWrite.c (revision 25022) @@ -741,5 +741,5 @@ } if (!psFitsCompressionApply(fits, compress)) { - psError(PS_ERR_UNKNOWN, false, "Unable to set FITS compression to NONE"); + psError(PS_ERR_UNKNOWN, false, "Unable to restore FITS compression"); psFree(compress); return false; Index: branches/eam_branches/20090715/psModules/src/camera/pmFPAfileFringeIO.c =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/pmFPAfileFringeIO.c (revision 25022) +++ branches/eam_branches/20090715/psModules/src/camera/pmFPAfileFringeIO.c (revision 25022) @@ -0,0 +1,238 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pmConfig.h" +//#include "pmConfigMask.h" +//#include "pmDetrendDB.h" + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFPALevel.h" +//#include "pmFPARead.h" +//#include "pmFPAWrite.h" +//#include "pmFPAMaskWeight.h" +#include "pmFPAview.h" +#include "pmFPAfile.h" +#include "pmFPAfileFringeIO.h" +#include "pmFPAfileFitsIO.h" +//#include "pmFPACopy.h" +//#include "pmFPAConstruct.h" +//#include "pmConceptsWrite.h" +#include "pmFringeStats.h" + +# define FRINGE_TABLE "FRINGE.MEASUREMENTS" + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Reading Fringe data +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// given an already-opened fits file, read the table corresponding to the specified view +bool pmFPAviewReadFringes(const pmFPAview *view, pmFPAfile *file) +{ + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_PTR_NON_NULL(file, false); + + pmFPA *fpa = file->fpa; // FPA of interest + psFits *fits = file->fits; // FITS file + + if (view->chip == -1) { + return pmFPAReadFringes(fpa, fits) > 0; + } + + if (view->cell == -1) { + pmChip *chip = pmFPAviewThisChip(view, fpa); // Chip of interest + return pmChipReadFringes(chip, fits) > 0; + } + + pmCell *cell = pmFPAviewThisCell(view, fpa); // Cell of interest + return pmCellReadFringes(cell, fits) > 0; +} + +int pmFPAReadFringes(pmFPA *fpa, psFits *fits) +{ + PS_ASSERT_PTR_NON_NULL(fpa, 0); + PS_ASSERT_FITS_NON_NULL(fits, 0); + + int numRead = 0; // Number of reads + psArray *chips = fpa->chips; // Array of chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + numRead += pmChipReadFringes(chip, fits); + } + + return numRead; +} + +int pmChipReadFringes(pmChip *chip, psFits *fits) +{ + PS_ASSERT_PTR_NON_NULL(chip, 0); + PS_ASSERT_FITS_NON_NULL(fits, 0); + + int numRead = 0; // Number of reads + psArray *cells = chip->cells; // Array of cells + for (int i = 0; i < cells->n; i++) { + pmCell *cell = cells->data[i]; // Cell of interest + numRead += pmCellReadFringes(cell, fits); + } + + return numRead; +} + +int pmCellReadFringes(pmCell *cell, psFits *fits) +{ + PS_ASSERT_PTR_NON_NULL(cell, 0); + PS_ASSERT_FITS_NON_NULL(fits, 0); + + const char *chipName = psMetadataLookupStr(NULL, cell->parent->concepts, "CHIP.NAME"); // Name of chip + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); // Name of cell + + // Fringe Table Extension name + psString extname = NULL; + psStringAppend(&extname, "FRINGE_%s_%s", chipName, cellName); + + if (!psFitsMoveExtName(fits, extname)) { + psError(PS_ERR_IO, false, "Unable to move to extension %s\n", extname); + psFree(extname); + return 0; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); // The FITS header + if (!header) { + psError(PS_ERR_IO, false, "Unable to read header for extension %s\n", extname); + psFree(extname); + psFree(header); + return 0; + } + + psArray *table = psFitsReadTable(fits); // The table + if (!table) { + psError(PS_ERR_UNKNOWN, false, "Unable to add read the fringe table data from the file"); + psFree (header); + psFree (extname); + return 0; + } + + psArray *fringes = pmFringesParseTable(table, header); + psFree(table); + psFree(header); + + if (!fringes) { + psError(PS_ERR_UNKNOWN, false, "Unable to add parse the fringe table data"); + psFree(extname); + return 0; + } + + if (!psMetadataAdd(cell->analysis, PS_LIST_TAIL, FRINGE_TABLE, PS_DATA_ARRAY, "Fringes", fringes)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add fringe from extension %s to analysis metadata " + "for chip %s, cell %s\n", extname, chipName, cellName); + psFree(fringes); + psFree(extname); + return 0; + } + + psFree(fringes); // drop local reference + psFree(extname); + + return 1; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Writing fringe data +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// given an already-opened fits file, write the table corresponding to the specified view +bool pmFPAviewWriteFringes(const pmFPAview *view, pmFPAfile *file, pmConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(view, false); + PS_ASSERT_PTR_NON_NULL(file, false); + PS_ASSERT_PTR_NON_NULL(config, false); + + pmFPA *fpa = pmFPAfileSuitableFPA(file, view, config, false); // FPA of interest + psFits *fits = file->fits; // FITS file + + if (view->chip == -1) { + return pmFPAWriteFringes(fits, fpa) > 0; + } + + if (view->cell == -1) { + pmChip *chip = pmFPAviewThisChip(view, fpa); // Chip of interest + return pmChipWriteFringes(fits, chip) > 0; + } + + pmCell *cell = pmFPAviewThisCell(view, fpa); // Cell of interest + return pmCellWriteFringes(fits, cell) > 0; +} + +int pmFPAWriteFringes(psFits *fits, const pmFPA *fpa) +{ + PS_ASSERT_PTR_NON_NULL(fpa, 0); + PS_ASSERT_PTR_NON_NULL(fits, 0); + + int numWrite = 0; // Number of reads + psArray *chips = fpa->chips; // Array of chips + for (int i = 0; i < chips->n; i++) { + pmChip *chip = chips->data[i]; // Chip of interest + numWrite += pmChipWriteFringes(fits, chip); + } + + return numWrite; +} + +int pmChipWriteFringes(psFits *fits, const pmChip *chip) +{ + PS_ASSERT_PTR_NON_NULL(chip, 0); + PS_ASSERT_PTR_NON_NULL(fits, 0); + + int numWrite = 0; // Number of reads + psArray *cells = chip->cells; // Array of cells + for (int i = 0; i < cells->n; i++) { + pmCell *cell = cells->data[i]; // Cell of interest + numWrite += pmCellWriteFringes(fits, cell); + } + + return numWrite; +} + +int pmCellWriteFringes(psFits *fits, const pmCell *cell) +{ + PS_ASSERT_PTR_NON_NULL(cell, 0); + PS_ASSERT_PTR_NON_NULL(fits, 0); + + const char *chipName = psMetadataLookupStr(NULL, cell->parent->concepts, "CHIP.NAME"); // Name of chip + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); // Name of cell + + psArray *fringes = psMetadataLookupPtr(NULL, cell->analysis, FRINGE_TABLE); // The FITS table + if (!fringes) { + // We wrote everything we could find + return 0; + } + + psMetadata *header = psMetadataAlloc(); + psArray *table = pmFringesFormatTable(header, fringes); + if (!table) { + // We wrote everything we could find + psFree(header); + return 0; + } + + psString extname = NULL; // Extension name + psStringAppend(&extname, "FRINGE_%s_%s", chipName, cellName); + + if (!psFitsWriteTable(fits, header, table, extname)) { + psError(PS_ERR_UNKNOWN, false, "Unable to write table from chip %s, cell %s to extension %s\n", + chipName, cellName, extname); + psFree(extname); + psFree(header); + return 0; + } + + psFree(extname); + psFree(header); + return 1; +} + + Index: branches/eam_branches/20090715/psModules/src/camera/pmFPAfileFringeIO.h =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/pmFPAfileFringeIO.h (revision 25022) +++ branches/eam_branches/20090715/psModules/src/camera/pmFPAfileFringeIO.h (revision 25022) @@ -0,0 +1,30 @@ +/* @file pmFPAfileFringeIO.h + * @brief Read & Write Fringe tables + * + * @author EAM, IfA + * @author PAP, IfA + * + * @version $Revision: 1.16 $ + * @date $Date: 2009-02-06 02:31:24 $ + * Copyright 2004-2005 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_FPA_FILE_FRINGE_IO_H +#define PM_FPA_FILE_FRINGE_IO_H + +/// @addtogroup Camera Camera Layout +/// @{ + +/// Read an fringes into the current view +bool pmFPAviewReadFringes(const pmFPAview *view, pmFPAfile *file); +int pmFPAReadFringes(pmFPA *fpa, psFits *fits); +int pmChipReadFringes(pmChip *chip, psFits *fits); +int pmCellReadFringes(pmCell *cell, psFits *fits); +bool pmFPAviewWriteFringes(const pmFPAview *view, pmFPAfile *file, pmConfig *config); +int pmFPAWriteFringes(psFits *fits, const pmFPA *fpa); +int pmChipWriteFringes(psFits *fits, const pmChip *chip); +int pmCellWriteFringes(psFits *fits, const pmCell *cell); + +/// @} + +# endif Index: branches/eam_branches/20090715/psModules/src/camera/pmFPAfileIO.c =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/pmFPAfileIO.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/camera/pmFPAfileIO.c (revision 25022) @@ -23,4 +23,5 @@ #include "pmFPAWrite.h" #include "pmFPAfileFitsIO.h" +#include "pmFPAfileFringeIO.h" #include "pmSpan.h" #include "pmFootprint.h" @@ -192,5 +193,5 @@ status = pmFPAviewReadFitsImage(view, file, config); if (status) { - if (!pmFPAviewReadFitsTable(view, file, "FRINGE")) { + if (!pmFPAviewReadFringes(view, file)) { psError(PS_ERR_UNKNOWN, false, "Unable to read fringe data from %s.\n", file->filename); return false; @@ -451,5 +452,5 @@ status = pmFPAviewWriteFitsImage (view, file, config); if (status) { - if (!pmFPAviewWriteFitsTable(view, file, "FRINGE", config)) { + if (!pmFPAviewWriteFringes(view, file, config)) { psError(PS_ERR_UNKNOWN, false, "Unable to write fringe data from %s.\n", file->filename); return false; Index: branches/eam_branches/20090715/psModules/src/camera/pmReadoutStack.c =================================================================== --- branches/eam_branches/20090715/psModules/src/camera/pmReadoutStack.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/camera/pmReadoutStack.c (revision 25022) @@ -263,7 +263,13 @@ pmCell *cell = readout->parent; // The parent cell bool mdok = true; // Status of MD lookup - psRegion *trimsec = psMetadataLookupPtr(&mdok, cell->concepts, "CELL.TRIMSEC"); // Trim section + psRegion *trimsec = cell ? psMetadataLookupPtr(&mdok, cell->concepts, "CELL.TRIMSEC") : NULL; // Trim section if (!mdok || !trimsec || psRegionIsNaN(*trimsec)) { - psWarning("CELL.TRIMSEC is not set for readout %ld --- ignored.\n", i); + psWarning("CELL.TRIMSEC is not set for readout %ld --- attempting to use image size.\n", i); + xSize = PS_MAX(xSize, readout->image->numCols); + ySize = PS_MAX(ySize, readout->image->numRows); + xMin = PS_MIN(xMin, 0); + xMax = PS_MAX(xMax, readout->image->numCols - 1); + yMin = PS_MIN(yMin, 0); + yMax = PS_MAX(yMax, readout->image->numRows - 1); } else { xSize = PS_MAX(xSize, trimsec->x1 - trimsec->x0); Index: branches/eam_branches/20090715/psModules/src/concepts/pmConceptsRead.c =================================================================== --- branches/eam_branches/20090715/psModules/src/concepts/pmConceptsRead.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/concepts/pmConceptsRead.c (revision 25022) @@ -425,4 +425,5 @@ if (rows->n == 0) { psWarning("No rows returned from database query for concept %s --- ignored.", name); + psFree(rows); return NULL; } Index: branches/eam_branches/20090715/psModules/src/detrend/Makefile.am =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/psModules/src/detrend/Makefile.am (revision 25022) @@ -16,5 +16,6 @@ pmShifts.c \ pmDark.c \ - pmRemnance.c + pmRemnance.c \ + pmPattern.c # pmSkySubtract.c @@ -33,5 +34,6 @@ pmShifts.h \ pmDark.h \ - pmRemnance.h + pmRemnance.h \ + pmPattern.h # pmSkySubtract.h Index: branches/eam_branches/20090715/psModules/src/detrend/pmDark.c =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/pmDark.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/detrend/pmDark.c (revision 25022) @@ -1,5 +1,10 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + #include #include #include +#include #include "psPolynomialMD.h" @@ -12,9 +17,10 @@ #include "pmReadoutStack.h" #include "pmDetrendThreads.h" - +#include "pmErrorCodes.h" #include "pmDark.h" #define PM_DARK_FITS_EXTNAME "PS_DARK" // FITS extension name for ordinates table #define PM_DARK_FITS_NAME "NAME" // Column name for concept name in ordinates table +#define PM_DARK_FITS_RULE "RULE" // Column name for concept rule in ordinates table #define PM_DARK_FITS_ORDER "ORDER" // Column name for polynomial order in ordinates table #define PM_DARK_FITS_SCALE "SCALE" // Column name for scaling option in ordinates table @@ -22,40 +28,24 @@ #define PM_DARK_FITS_MAX "MAX" // Column name for maximum value in ordinates table - - -// Look up the value of an ordinate in a readout -static bool ordinateLookup(float *value, // Value of ordinate, to return - bool *inRange, // is value within min : max range? - const char *name, // Name of ordinate (concept name) - bool scale, // Scale the value? - float min, float max, // Minimum and maximum values for scaling - const pmReadout *ro // Readout of interest - ) -{ - assert(value); - assert(name); - assert(ro); - - *inRange = true; - - pmCell *cell = ro->parent; // Parent cell - if (!cell) { - return false; - } +static bool ordinateParseConcept(double *value, const pmReadout *readout, const char *name) { + + *value = NAN; + + pmCell *cell = readout->parent; // Parent cell + psAssert(cell, "readout is missing cell \n"); + psMetadataItem *item = psMetadataLookup(cell->concepts, name); if (!item) { pmChip *chip = cell->parent; // Parent chip - if (!chip) { - return false; - } + psAssert(chip, "cell is missing chip \n"); + item = psMetadataLookup(chip->concepts, name); if (!item) { pmFPA *fpa = chip->parent; // Parent FPA - if (!fpa) { - return false; - } + psAssert(fpa, "chip is missing fpa \n"); + item = psMetadataLookup(fpa->concepts, name); if (!item) { - psWarning("Unable to find concept %s in readout", name); + psError(PM_ERR_CONFIG, true, "Unable to find concept %s in readout", name); return false; } @@ -63,9 +53,85 @@ } - *value = psMetadataItemParseF32(item); // Value of interest + *value = psMetadataItemParseF64(item); // Value of interest if (!isfinite(*value)) { psWarning("Non-finite value (%f) of concept %s in readout", *value, name); - return false; - } + } + return true; +} + +static bool ordinateParseRule(double *value, const pmReadout *readout, const char *name, const char *rule) { + + psAssert(name, "ordinate name is not defined"); + psAssert(rule, "ordinate rule is not defined"); + + *value = NAN; + + psArray *words = psStringSplitArray(rule, " ", false); + + // we should have a rule of the form (concept) OP (concept) OP (concept) ... + // for now, the only allowed OP is * (eventually, we can steal code from opihi for a better + // RPN parser). + + if (words->n % 2 == 0) { + psError(PM_ERR_CONFIG, true, "syntax error in DARK.ORDINATE %s rule %s\n", name, rule); + psFree(words); + return false; + } + + for (int i = 1; i < words->n; i+=2) { + if (strcmp((char *)words->data[i], "*")) { + psError(PM_ERR_CONFIG, true, "syntax error in DARK.ORDINATE %s rule %s\n", name, rule); + psFree(words); + return false; + } + } + + if (!ordinateParseConcept(value, readout, words->data[0])) { + psError(PM_ERR_CONFIG, false, "syntax error in DARK.ORDINATE %s rule %s\n", name, rule); + psFree(words); + return false; + } + + double value2 = 0.0; + for (int i = 2; i < words->n; i+=2) { + if (!ordinateParseConcept(&value2, readout, words->data[i])) { + psError(PM_ERR_CONFIG, false, "syntax error in DARK.ORDINATE %s rule %s\n", name, rule); + psFree(words); + return false; + } + *value *= value2; + } + psFree(words); + return true; +} + +// Look up the value of an ordinate in a readout +static bool ordinateLookup(double *value, // Value of ordinate, to return + bool *inRange, // is value within min : max range? + const char *name, // Name of ordinate (concept or abstract name) + const char *rule, // Rule for generating the value (if NULL use name as concept) + bool scale, // Scale the value? + float min, float max, // Minimum and maximum values for scaling + const pmReadout *readout // Readout of interest + ) +{ + assert(value); + assert(name); + assert(readout); + + *inRange = true; + + if (rule) { + if (!ordinateParseRule(value, readout, name, rule)) { + psError(PM_ERR_CONFIG, false, "trouble parsing rule %s for DARK.ORDINATE %s", rule, name); + return false; + } + } else { + if (!ordinateParseConcept(value, readout, name)) { + psError(PM_ERR_CONFIG, false, "trouble parsing rule %s for DARK.ORDINATE %s", rule, name); + return false; + } + } + if (scale) { if (*value < min || *value > max) { @@ -82,4 +148,5 @@ { psFree(ord->name); + psFree(ord->rule); return; } @@ -91,4 +158,5 @@ ord->name = psStringCopy(name); + ord->rule = NULL; ord->order = order; ord->scale = false; @@ -118,7 +186,11 @@ pmReadout *readout = inputs->data[i]; // Readout of interest - float normValue; // Normalisation value - if (!ordinateLookup(&normValue, &inRange, normConcept, false, NAN, NAN, readout)) { - psWarning("Unable to find value of %s for readout %d", normConcept, i); + double normValue; // Normalisation value + if (!ordinateLookup(&normValue, &inRange, normConcept, NULL, false, NAN, NAN, readout)) { + psError(PM_ERR_CONFIG, false, "problem finding concept %s for DARK.NORM", normConcept); + return false; + } + if (!isfinite(normValue)) { + psWarning("Unable to find acceptable value of %s for readout %d", normConcept, i); roMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0xff; norm->data.F32[i] = NAN; @@ -140,5 +212,5 @@ pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate information if (ord->order <= 0) { - psError(PS_ERR_UNKNOWN, true, "Bad order for concept %s (%d) --- ignored", ord->name, ord->order); + psError(PS_ERR_UNKNOWN, true, "Bad order for DARK.ORDINATE %s (%d) --- ignored", ord->name, ord->order); psFree(values); psFree(roMask); @@ -157,6 +229,11 @@ pmReadout *readout = inputs->data[j]; // Readout of interest - float value = NAN; // Value of ordinate - if (!ordinateLookup(&value, &inRange, ord->name, ord->scale, ord->min, ord->max, readout)) { + double value = NAN; // Value of ordinate + if (!ordinateLookup(&value, &inRange, ord->name, ord->rule, ord->scale, ord->min, ord->max, readout)) { + psError(PM_ERR_CONFIG, false, "problem finding rule for DARK.ORDINATE %s", ord->name); + return false; + } + if (!isfinite(value)) { + psWarning("Unable to find acceptable value of DARK.ORDINATE %s for readout %d", ord->name, i); roMask->data.PS_TYPE_VECTOR_MASK_DATA[j] = 0xff; val->data.F32[i] = NAN; @@ -165,4 +242,5 @@ } if (!inRange) { + psWarning("Value of DARK.ORDINATE %s for readout %d is out of range", ord->name, i); roMask->data.PS_TYPE_VECTOR_MASK_DATA[j] = 0xff; val->data.F32[i] = NAN; @@ -309,4 +387,6 @@ return false; } + + pmDarkVisualInit(values); pmReadout *outReadout = output->readouts->data[0]; @@ -352,4 +432,8 @@ psVectorInit(poly->coeff, NAN); } + + pmDarkVisualPixelFit(pixels, mask); + pmDarkVisualPixelModel(poly, values); + for (int k = 0; k < poly->coeff->n; k++) { pmReadout *ro = output->readouts->data[k]; // Readout of interest @@ -447,16 +531,21 @@ for (int i = 0; i < numOrdinates; i++) { pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate of interest - float value = NAN; // Value for ordinate - if (!ordinateLookup(&value, &inRange, ord->name, ord->scale, ord->min, ord->max, readout)) { - psError(PS_ERR_UNKNOWN, true, "Unable to find value for %s", ord->name); + double value = NAN; // Value for ordinate + if (!ordinateLookup(&value, &inRange, ord->name, ord->rule, ord->scale, ord->min, ord->max, readout)) { + psError(PS_ERR_UNKNOWN, true, "Unable to find value for DARK.ORDINATE %s", ord->name); psFree(values); return false; } + if (!isfinite(value)) { + psError(PS_ERR_UNKNOWN, true, "Value for DARK.ORDINATE %s is NAN", ord->name); + psFree(values); + return false; + } values->data.F32[i] = value; } - float norm = NAN; // Normalisation value + double norm = NAN; // Normalisation value bool doNorm = false; // Do normalisation? if (normConcept && strlen(normConcept) > 0) { - if (!ordinateLookup(&norm, &inRange, normConcept, false, NAN, NAN, readout)) { + if (!ordinateLookup(&norm, &inRange, normConcept, NULL, false, NAN, NAN, readout)) { psError(PS_ERR_UNKNOWN, true, "Unable to find value for %s", normConcept); psFree(values); @@ -470,5 +559,5 @@ pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate information if (ord->order <= 0) { - psError(PS_ERR_UNKNOWN, true, "Bad order for concept %s (%d) --- ignored", ord->name, ord->order); + psError(PS_ERR_UNKNOWN, true, "Bad order for DARK.ORDINATE %s (%d) --- ignored", ord->name, ord->order); psFree(values); psFree(orders); @@ -701,5 +790,13 @@ pmDarkOrdinate *ord = ordinates->data[i]; // Ordinate of interest psMetadata *row = psMetadataAlloc(); // FITS table row - psMetadataAddStr(row, PS_LIST_TAIL, PM_DARK_FITS_NAME, 0, "Concept name", ord->name); + psMetadataAddStr(row, PS_LIST_TAIL, PM_DARK_FITS_NAME, 0, "DARK.ORDINATE name", ord->name); + + // XXX write a dummy value if ord->rule == NULL? (eg, NONE) + if (ord->rule) { + psMetadataAddStr(row, PS_LIST_TAIL, PM_DARK_FITS_RULE, 0, "DARK.ORDINATE rule", ord->rule); + } else { + psMetadataAddStr(row, PS_LIST_TAIL, PM_DARK_FITS_RULE, 0, "DARK.ORDINATE rule", "NONE"); + } + psMetadataAddS32(row, PS_LIST_TAIL, PM_DARK_FITS_ORDER, 0, "Polynomial order", ord->order); psMetadataAddBool(row, PS_LIST_TAIL, PM_DARK_FITS_SCALE, 0, "Scale values?", ord->scale); @@ -878,4 +975,12 @@ ord->max = psMetadataLookupF32(NULL, row, PM_DARK_FITS_MAX); + // load the ordinate rule; it is not an error if this field is missing or NULL + // a NULL rule means 'use the name as the single concept' + const char *rule = psMetadataLookupStr(&mdok, row, PM_DARK_FITS_RULE); + if (!rule || !strcasecmp(rule, "NONE")) { + ord->rule = NULL; + } else { + ord->rule = psStringCopy(rule); + } ordinates->data[i] = ord; } @@ -892,2 +997,152 @@ } +#if (HAVE_KAPA) +#include +#include "pmKapaPlots.h" +#include "pmVisual.h" + +static int nKapa = 0; +static int *kapa = NULL; +static bool plotFlag = true; +static psArray *xVectors = NULL; + +// this init function only gets the ordinates for the first readout... +bool pmDarkVisualInit(psArray *values) { + + if (!pmVisualIsVisual()) return true; + + // skip if we have already opened the windows (or if none are requested...) + if (nKapa) return true; + + // values has Ninput vectors with Norder elements; we need Norder vectors with Ninput elements... + int nOrders = 0; + for (int i = 0; i < values->n; i++) { + psVector *vect = values->data[i]; + if (!nOrders) { + nOrders = vect->n; + } else { + psAssert (nOrders == vect->n, "mismatch in order vector lengths"); + } + } + xVectors = psArrayAlloc(nOrders); + for (int i = 0; i < nOrders; i++) { + xVectors->data[i] = psVectorAlloc(values->n, PS_TYPE_F32); + } + + for (int i = 0; i < values->n; i++) { + psVector *vect = values->data[i]; + for (int j = 0; j < vect->n; j++) { + psVector *xVec = xVectors->data[j]; + xVec->data.F32[i] = vect->data.F32[j]; + } + } + + nKapa = nOrders; + + kapa = psAlloc(nKapa*sizeof(int)); + + for (int i = 0; i < nKapa; i++) { + kapa[i] = -1; + pmVisualInitWindow(&kapa[i], "ppmerge"); + } + return true; +} + +bool pmDarkVisualPixelFit(psVector *pixels, psVector *mask) { + + Graphdata graphdata; + + if (!pmVisualIsVisual()) return true; + + KapaInitGraph(&graphdata); + + psAssert(nKapa == xVectors->n, "inconsistent number of orders %d vs %ld\n", nKapa, xVectors->n); + + psVector *xSub = psVectorAlloc(pixels->n, PS_TYPE_F32); + psVector *ySub = psVectorAlloc(pixels->n, PS_TYPE_F32); + + for (int i = 0; i < xVectors->n; i++) { + psVector *x = xVectors->data[i]; + + // generate vectors of the unmasked values + int nSub = 0; + for (int j = 0; j < pixels->n; j++) { + if (mask && mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) continue; + xSub->data.F32[nSub] = x->data.F32[j]; + ySub->data.F32[nSub] = pixels->data.F32[j]; + nSub ++; + } + xSub->n = ySub->n = nSub; + + // plot the unmasked values + pmVisualScaleGraphdata (&graphdata, xSub, ySub, false); + KapaSetGraphData(kapa[i], &graphdata); + KapaSetLimits(kapa[i], &graphdata); + KapaClearPlots (kapa[i]); + + KapaSetFont (kapa[i], "courier", 14); + KapaBox (kapa[i], &graphdata); + KapaSendLabel (kapa[i], "ordinate", KAPA_LABEL_XM); + KapaSendLabel (kapa[i], "pixel values", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName("black"); + graphdata.style = 2; + graphdata.ptype = 2; + KapaPrepPlot (kapa[i], xSub->n, &graphdata); + KapaPlotVector(kapa[i], xSub->n, xSub->data.F32, "x"); + KapaPlotVector(kapa[i], xSub->n, ySub->data.F32, "y"); + } + pmVisualAskUser (&plotFlag); + return true; +} + +bool pmDarkVisualPixelModel(psPolynomialMD *poly, psArray *values) { + + Graphdata graphdata; + + if (!pmVisualIsVisual()) return true; + + KapaInitGraph(&graphdata); + + psAssert(nKapa == xVectors->n, "inconsistent number of orders %d vs %ld\n", nKapa, xVectors->n); + + psVector *yFit = psVectorAlloc(values->n, PS_TYPE_F32); + + for (int i = 0; i < values->n; i++) { + psVector *coord = values->data[i]; + yFit->data.F32[i] = psPolynomialMDEval (poly, coord); + } + + for (int i = 0; i < xVectors->n; i++) { + psVector *xFit = xVectors->data[i]; + + KapaGetGraphData(kapa[i], &graphdata); + graphdata.color = KapaColorByName("red"); + graphdata.style = 2; + graphdata.ptype = 7; + KapaPrepPlot (kapa[i], xFit->n, &graphdata); + KapaPlotVector(kapa[i], xFit->n, xFit->data.F32, "x"); + KapaPlotVector(kapa[i], xFit->n, yFit->data.F32, "y"); + } + pmVisualAskUser (&plotFlag); + return true; +} + +bool pmDarkVisualCleanup() { + + for (int i = 0; i < nKapa; i++) { + KapaClose(kapa[i]); + } + psFree (kapa); + psFree (xVectors); + return true; +} + +# else + +bool pmDarkVisualInit(psArray *values) { return true; } +bool pmDarkVisualPixelFit(psVector *pixels, psVector *mask) { return true; } +bool pmDarkVisualPixelModel(psPolynomialMD *poly, psArray *values) { return true; } +bool pmDarkVisualCleanup() { return true; } + +# endif Index: branches/eam_branches/20090715/psModules/src/detrend/pmDark.h =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/pmDark.h (revision 25020) +++ branches/eam_branches/20090715/psModules/src/detrend/pmDark.h (revision 25022) @@ -13,5 +13,6 @@ // An ordinate for fitting darks typedef struct { - psString name; // Name of concept to fit + psString name; // Name of ordinate + psString rule; // Rule for generating ordinate (math on concepts) int order; // Polynomial order to fit bool scale; // Rescale values? @@ -116,4 +117,8 @@ ); +bool pmDarkVisualInit(psArray *values); +bool pmDarkVisualPixelFit(psVector *pixels, psVector *mask); +bool pmDarkVisualCleanup(); +bool pmDarkVisualPixelModel(psPolynomialMD *poly, psArray *values); #endif Index: branches/eam_branches/20090715/psModules/src/detrend/pmFringeStats.c =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/pmFringeStats.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/detrend/pmFringeStats.c (revision 25022) @@ -464,4 +464,8 @@ for (long i = 0; i < fringes->n; i++) { pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + if (!fringe) { + psWarning ("skipping empty fringe stats for entry %ld -- video cell?\n", i); + continue; + } pmFringeRegions *regions = fringe->regions; // The fringe regions if (numPoints == 0) { @@ -496,4 +500,8 @@ for (long i = 0; i < fringes->n; i++) { pmFringeStats *fringe = fringes->data[i]; // The fringe of interest + if (!fringe) { + psWarning ("skipping empty fringe stats for entry %ld -- video cell?\n", i); + continue; + } pmFringeRegions *regions = fringe->regions; // The fringe regions // Copy the data over @@ -520,7 +528,7 @@ ////////////////////////////////////////////////////////////////////////////////////////////////////////////// -bool pmFringesFormat(pmCell *cell, psMetadata *header, const psArray *fringes) -{ - PS_ASSERT_PTR_NON_NULL(cell, false); +psArray *pmFringesFormatTable(psMetadata *header, const psArray *fringes) +{ + PS_ASSERT_PTR_NON_NULL(header, false); PS_ASSERT_ARRAY_NON_NULL(fringes, false); @@ -531,5 +539,5 @@ if (stats->regions != regions) { psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Regions for fringe statistics are not identical.\n"); - return false; + return NULL; } } @@ -557,20 +565,8 @@ // Vectors: x, y, mask, f, df - psMetadata *scalars = psMemIncrRefCounter(header); // Metadata to hold the scalars; will be the header - if (!scalars) { - scalars = psMetadataAlloc(); - } - psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGDX", PS_META_REPLACE, "Median box half-width", - regions->dX); - psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGDY", PS_META_REPLACE, "Median box half-height", - regions->dY); - psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGNX", PS_META_REPLACE, "Large-scale smoothing in x", - regions->nX); - psMetadataAddS32(scalars, PS_LIST_TAIL, "PSFRNGNY", PS_META_REPLACE, "Large-scale smoothing in y", - regions->nY); - psMetadataAdd(cell->analysis, PS_LIST_TAIL, "FRINGE.HEADER", PS_DATA_METADATA, - "Header for fringe data", scalars); - psFree(scalars); - + psMetadataAddS32(header, PS_LIST_TAIL, "PSFRNGDX", PS_META_REPLACE, "Median box half-width", regions->dX); + psMetadataAddS32(header, PS_LIST_TAIL, "PSFRNGDY", PS_META_REPLACE, "Median box half-height", regions->dY); + psMetadataAddS32(header, PS_LIST_TAIL, "PSFRNGNX", PS_META_REPLACE, "Large-scale smoothing in x", regions->nX); + psMetadataAddS32(header, PS_LIST_TAIL, "PSFRNGNY", PS_META_REPLACE, "Large-scale smoothing in y", regions->nY); psArray *table = psArrayAlloc(numRows); // The table @@ -605,27 +601,13 @@ } - psMetadataAdd(cell->analysis, PS_LIST_TAIL, "FRINGE", PS_DATA_ARRAY, "Fringe data", table); - - return true; -} - - -psArray *pmFringesParse(pmCell *cell) -{ - PS_ASSERT_PTR_NON_NULL(cell, NULL); + return table; +} + +psArray *pmFringesParseTable(psArray *table, psMetadata *header) +{ + PS_ASSERT_PTR_NON_NULL(table, NULL); + PS_ASSERT_PTR_NON_NULL(header, NULL); bool mdok; // Status of MD lookup - psMetadata *header = psMetadataLookupMetadata(&mdok, cell->analysis, "FRINGE.HEADER"); // Header - if (!mdok || !header) { - psError(PS_ERR_UNKNOWN, false, "Unable to find header for fringe data.\n"); - return NULL; - } - - psArray *table = psMetadataLookupPtr(NULL, cell->analysis, "FRINGE"); // FITS table - if (!table) { - psError(PS_ERR_UNKNOWN, false, "Unable to find table for fringe data.\n"); - return NULL; - } - // Read the scalars from the header @@ -722,4 +704,55 @@ } +bool pmFringesFormat(pmCell *cell, psMetadata *inHeader, const psArray *fringes) +{ + PS_ASSERT_PTR_NON_NULL(cell, false); + PS_ASSERT_ARRAY_NON_NULL(fringes, false); + + psMetadata *header = psMemIncrRefCounter(inHeader); // Metadata to hold the scalars; will be the header + if (!inHeader) { + inHeader = psMetadataAlloc(); + } + + psArray *table = pmFringesFormatTable(header, fringes); + if (!table) { + psError (PS_ERR_UNKNOWN, false, "unable to generate table from fringes"); + psFree(header); + return NULL; + } + + psMetadataAdd(cell->analysis, PS_LIST_TAIL, "FRINGE.HEADER", PS_DATA_METADATA, "Header for fringe data", header); + psFree(header); + + psMetadataAdd(cell->analysis, PS_LIST_TAIL, "FRINGE.TABLE", PS_DATA_ARRAY, "Fringe data", table); + psFree(table); + + return true; +} + +psArray *pmFringesParse(pmCell *cell) +{ + PS_ASSERT_PTR_NON_NULL(cell, NULL); + + bool mdok; // Status of MD lookup + psMetadata *header = psMetadataLookupMetadata(&mdok, cell->analysis, "FRINGE.HEADER"); // Header + if (!mdok || !header) { + psError(PS_ERR_UNKNOWN, false, "Unable to find header for fringe data.\n"); + return NULL; + } + + psArray *table = psMetadataLookupPtr(NULL, cell->analysis, "FRINGE.TABLE"); // FITS table + if (!table) { + psError(PS_ERR_UNKNOWN, false, "Unable to find table for fringe data.\n"); + return NULL; + } + + psArray *fringes = pmFringesParseTable(table, header); + if (!fringes) { + psError(PS_ERR_UNKNOWN, false, "Unable to add parse the fringe table data"); + return NULL; + } + + return fringes; +} ////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -822,5 +855,7 @@ // Solve the least-squares equation if (!psMatrixGJSolve(A, B)) { - psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations. Returning NULL.\n"); + psLogMsg("psModules.detrend", PS_LOG_INFO, "Could not solve linear equations. Returning NULL.\n"); + psFree(A); + psFree(B); return false; } Index: branches/eam_branches/20090715/psModules/src/detrend/pmFringeStats.h =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/pmFringeStats.h (revision 25020) +++ branches/eam_branches/20090715/psModules/src/detrend/pmFringeStats.h (revision 25022) @@ -142,11 +142,27 @@ ////////////////////////////////////////////////////////////////////////////////////////////////////////////// -/// Write an array of fringes measurements to a FITS table. -/// -/// Writes an array of fringe measurements for a cell as a FITS table in the analysis metadata. The array of -/// fringe statistics must all use the same fringe regions (or there is no point in storing them all -/// together). The header is supplemented with scalar values dX, dY, nX and nY (as PSFRNGDX, PSFRNGDY, -/// PSFRNGNX, PSFRNGNY) from the fringe regions, while the fringe coordinates and mask are written as a FITS -/// table (as x, y, mask, f, df; f and df are vectors). +/// Convert an array of fringes measurements to a psArray suitable for writing as a FITS table. +/// +/// Converts an array of fringe measurements for a cell into the corresponding rows of a FITS +/// table (array of psMetadata). The array of fringe statistics must all use the same fringe +/// regions (or there is no point in storing them all together). The header is supplemented +/// with scalar values dX, dY, nX and nY (as PSFRNGDX, PSFRNGDY, PSFRNGNX, PSFRNGNY) from the +/// fringe regions, while the fringe coordinates and mask are written as a FITS table rows (as +/// x, y, mask, f, df; f and df are vectors). Use psFitsTableWrite to save the resulting rows +/// to disk. +psArray *pmFringesFormatTable(psMetadata *header, const psArray *fringes); + + +/// Parses an array of fringes measurements from a FITS table. +/// +/// The fringes for the cell are read from the FITS table (array of psMetadata rows). The +/// table provides the region and the (possibly multiple) fringe statistics for that region. +/// The supplied header defines the scalar values dX, dY, nX and nY (as PSFRNGDX, PSFRNGDY, +/// PSFRNGNX, PSFRNGNY) +psArray *pmFringesParseTable(psArray *table, psMetadata *header); + +/// Deprecated Function: converts the fringes to a FITS table (array of psMetadata) and saves +/// them on the cell->analysis; scalar values are dX, dY, nX and nY are written to the header +/// (as PSFRNGDX, PSFRNGDY, PSFRNGNX, PSFRNGNY) bool pmFringesFormat(pmCell *cell, ///< Cell for which to write psMetadata *header, ///< Header, or NULL @@ -154,12 +170,8 @@ ); -/// Parses an array of fringes measurements from a FITS table. -/// -/// The fringes for the cell are read from the FITS table in the analysis metadata. The table provides the -/// region and the (possibly multiple) fringe statistics for that region. The current extension is used if -/// the extension name is not provided. +/// Deprecated Function: pulls a the header and FITS table (array of psMetadata) representing +/// the fringes from the cell->analysis and converts to fringe measurements psArray *pmFringesParse(pmCell *cell ///< Cell for which to read fringes ); - ////////////////////////////////////////////////////////////////////////////////////////////////////////////// Index: branches/eam_branches/20090715/psModules/src/detrend/pmPattern.c =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/pmPattern.c (revision 25022) +++ branches/eam_branches/20090715/psModules/src/detrend/pmPattern.c (revision 25022) @@ -0,0 +1,127 @@ +#include +#include + +#include "pmFPA.h" + +#include "pmPattern.h" + + +// Mask a row as bad +static void patternMaskRow(pmReadout *ro, // Readout to mask + int y, // Row to mask + psImageMaskType bad // Mask value to give + ) +{ + psImage *image = ro->image; // Image to mask + psAssert(image, "Require image"); + psAssert(y < image->numRows, "Row not in image"); + + int numCols = image->numCols; // Size of image + for (int x = 0; x < numCols; x++) { + image->data.F32[y][x] = NAN; + } + if (ro->mask) { + psImage *mask = ro->mask; // Mask image to mask + for (int x = 0; x < numCols; x++) { + mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= bad; + } + } + return; +} + +bool pmPatternRow(pmReadout *ro, int order, int iter, float rej, float thresh, + psStatsOptions clipMean, psStatsOptions clipStdev, + psImageMaskType maskVal, psImageMaskType maskBad) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PM_ASSERT_READOUT_IMAGE(ro, false); + PS_ASSERT_INT_NONNEGATIVE(order, false); + PS_ASSERT_INT_NONNEGATIVE(iter, false); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + PS_ASSERT_FLOAT_LARGER_THAN(thresh, 0.0, false); + + psImage *image = ro->image; // Image to correct + psImage *mask = ro->mask; // Mask for image + int numCols = image->numCols, numRows = image->numRows; // Size of image + + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator + if (!psImageBackground(stats, NULL, ro->image, ro->mask, maskVal, rng)) { + psWarning("Unable to calculate statistics on readout; masking entire readout."); + psErrorClear(); + psFree(stats); + psFree(rng); + psImageInit(image, NAN); + if (mask) { + psBinaryOp(mask, mask, "|", psScalarAlloc(maskBad, PS_TYPE_IMAGE_MASK)); + } + if (ro->variance) { + psImageInit(image, NAN); + } + return true; + } + float lower = stats->robustMedian - thresh * stats->robustStdev; // Lower bound for data + float upper = stats->robustMedian + thresh * stats->robustStdev; // Upper bound for data + psFree(stats); + psFree(rng); + + // Indices are distributed [-1:1) + psVector *indices = psVectorAlloc(numCols, PS_TYPE_F32); // Indices for fitting + float norm = 2.0 / (float)numCols; // Normalisation for indices + for (int x = 0; x < numCols; x++) { + indices->data.F32[x] = x * norm - 1.0; + } + + psStats *clip = psStatsAlloc(clipMean | clipStdev); // Clipping statistics + clip->clipIter = iter; + clip->clipSigma = rej; + psVector *clipMask = psVectorAlloc(numCols, PS_TYPE_VECTOR_MASK); // Mask for clipping + psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, order); // Polynomial to fit + psVector *data = psVectorAlloc(numCols, PS_TYPE_F32); // Data to fit + + for (int y = 0; y < numRows; y++) { + psVectorInit(clipMask, 0); + data = psImageRow(data, image, y); + int num = 0; // Number of good pixels + for (int x = 0; x < numCols; x++) { + if ((mask && mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal) || + data->data.F32[x] < lower || data->data.F32[x] > upper) { + clipMask->data.PS_TYPE_VECTOR_MASK_DATA[x] = 0xFF; + } else { + clipMask->data.PS_TYPE_VECTOR_MASK_DATA[x] = 0; + num++; + } + } + if (num < order + 1) { + // Not enough points to fit + patternMaskRow(ro, y, maskBad); + continue; + } + if (!psVectorClipFitPolynomial1D(poly, clip, clipMask, 0xFF, data, NULL, indices)) { + psWarning("Unable to fit polynomial to row %d", y); + psErrorClear(); + patternMaskRow(ro, y, maskBad); + continue; + } + psVector *solution = psPolynomial1DEvalVector(poly, indices); // Solution vector + if (!solution) { + psWarning("Unable to evaluate polynomial for row %d", y); + psErrorClear(); + patternMaskRow(ro, y, maskBad); + continue; + } + + for (int x = 0; x < numCols; x++) { + image->data.F32[y][x] -= solution->data.F32[x]; + } + psFree(solution); + } + + psFree(indices); + psFree(clip); + psFree(clipMask); + psFree(poly); + psFree(data); + + return true; +} Index: branches/eam_branches/20090715/psModules/src/detrend/pmPattern.h =================================================================== --- branches/eam_branches/20090715/psModules/src/detrend/pmPattern.h (revision 25022) +++ branches/eam_branches/20090715/psModules/src/detrend/pmPattern.h (revision 25022) @@ -0,0 +1,37 @@ +/* @file pmPattern.h + * @brief Fit and remove pattern noise + * + * @author Paul Price, IfA + * + * @version $Revision: 1.16 $ + * @date $Date: 2009-02-12 19:25:52 $ + * Copyright 2004-2009 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_PATTERN_H +#define PM_PATTERN_H + +#include +#include + +/// @addtogroup detrend Detrend Creation and Application +/// @{ + +/// Fit and remove pattern noise over rows +bool pmPatternRow( + pmReadout *ro, ///< Readout to correct + int order, ///< Polynomial order + int iter, ///< Number of clipping iterations + float rej, ///< Rejection threshold for clipping + float thresh, ///< Threshold for ignoring pixels + psStatsOptions clipMean, ///< Statistic to use for mean + psStatsOptions clipStdev, ///< Statistic to use for standard deviation + psImageMaskType maskVal, ///< Mask value to use + psImageMaskType maskBad ///< Mask value to give bad pixels + ); + +/// @} +#endif + + + Index: branches/eam_branches/20090715/psModules/src/extras/Makefile.am =================================================================== --- branches/eam_branches/20090715/psModules/src/extras/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/psModules/src/extras/Makefile.am (revision 25022) @@ -8,5 +8,6 @@ psIOBuffer.c \ pmKapaPlots.c \ - pmVisual.c + pmVisual.c \ + ippStages.c pkginclude_HEADERS = \ @@ -15,5 +16,6 @@ psIOBuffer.h \ pmKapaPlots.h \ - pmVisual.h + pmVisual.h \ + ippStages.h CLEANFILES = *~ Index: branches/eam_branches/20090715/psModules/src/extras/ippStages.c =================================================================== --- branches/eam_branches/20090715/psModules/src/extras/ippStages.c (revision 25022) +++ branches/eam_branches/20090715/psModules/src/extras/ippStages.c (revision 25022) @@ -0,0 +1,31 @@ +/** Functions that describe the various stages of the IPP + * @author Bill Sweeney, IfA + **/ + +#include +#include "ippStages.h" +#include + +ippStage ippStringToStage(const psString stageString) +{ + PS_ASSERT_PTR_NON_NULL(stageString, IPP_STAGE_NONE); + + if (!strcmp(stageString, "raw")) { + return IPP_STAGE_RAW; + } else if (!strcmp(stageString, "chip")) { + return IPP_STAGE_CHIP; + } else if (!strcmp(stageString, "camera")) { + return IPP_STAGE_CAMERA; + } else if (!strcmp(stageString, "warp")) { + return IPP_STAGE_WARP; + } else if (!strcmp(stageString, "fake")) { + return IPP_STAGE_FAKE; + } else if (!strcmp(stageString, "diff")) { + return IPP_STAGE_DIFF; + } else if (!strcmp(stageString, "stack")) { + return IPP_STAGE_STACK; + } else { + psError(PS_ERR_PROGRAMMING, true, "%s is not a valid IPP stage", stageString); + return IPP_STAGE_NONE; + } +} Index: branches/eam_branches/20090715/psModules/src/extras/ippStages.h =================================================================== --- branches/eam_branches/20090715/psModules/src/extras/ippStages.h (revision 25022) +++ branches/eam_branches/20090715/psModules/src/extras/ippStages.h (revision 25022) @@ -0,0 +1,27 @@ +/* @file psVisual.h + * @brief some macro defintions for the stages of the pipeline + * @author Bill Sweeney, IfA + * + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#ifndef IPP_STAGES_H +#define IPP_STAGES_H + +typedef enum { + IPP_STAGE_NONE = -1, + IPP_STAGE_RAW = 0, + IPP_STAGE_CHIP, + IPP_STAGE_CAMERA, + IPP_STAGE_FAKE, + IPP_STAGE_WARP, + IPP_STAGE_DIFF, + IPP_STAGE_STACK, +} ippStage; + +/** return the ippStage represented by a string + * @return the corresponding value or IPP_STAGE_NONE if invalid + */ +ippStage ippStringToStage(const psString stageString); + +#endif Index: branches/eam_branches/20090715/psModules/src/imcombine/pmReadoutCombine.c =================================================================== --- branches/eam_branches/20090715/psModules/src/imcombine/pmReadoutCombine.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/imcombine/pmReadoutCombine.c (revision 25022) @@ -281,4 +281,7 @@ #endif + // set up windows for visualization (if selected) + pmReadoutCombineVisualInit(); + // Dereference output products psF32 **outputImage = output->image->data.F32; // Output image @@ -397,4 +400,8 @@ outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine); + if (!isfinite(outputImage[yOut][xOut])) { + pmReadoutCombineVisualPixels(pixels, mask, outputImage[yOut][xOut]); + } + if (isnan(outputImage[yOut][xOut])) { outputImage[yOut][xOut] = NAN; @@ -434,2 +441,106 @@ } +#if (HAVE_KAPA) +#include +#include "pmKapaPlots.h" +#include "pmVisual.h" + +static int kapa = -1; +static bool plotFlag = true; + +// this init function only gets the ordinates for the first readout... +bool pmReadoutCombineVisualInit(void) { + + if (!pmVisualIsVisual()) return true; + + // skip if we have already opened the windows (or if none are requested...) + if (kapa != -1) return true; + + pmVisualInitWindow(&kapa, "ppmerge"); + return true; +} + +bool pmReadoutCombineVisualPixels(psVector *pixels, psVector *mask, float mean) { + + Graphdata graphdata; + float xline[2], yline[2]; + + if (!pmVisualIsVisual()) return true; + + if (!plotFlag) return true; + + KapaInitGraph(&graphdata); + + psVector *xAll = psVectorAlloc(pixels->n, PS_TYPE_F32); + psVector *xSub = psVectorAlloc(pixels->n, PS_TYPE_F32); + psVector *ySub = psVectorAlloc(pixels->n, PS_TYPE_F32); + + // generate vectors of the unmasked values + int nSub = 0; + for (int j = 0; j < pixels->n; j++) { + xAll->data.F32[j] = j; + if (mask && mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) continue; + xSub->data.F32[nSub] = j; + ySub->data.F32[nSub] = pixels->data.F32[j]; + nSub ++; + } + xSub->n = ySub->n = nSub; + xAll->n = pixels->n; + + xline[0] = 0; + xline[1] = pixels->n; + yline[0] = mean; + yline[1] = mean; + + // plot the unmasked values + pmVisualScaleGraphdata (&graphdata, xAll, pixels, false); + KapaSetGraphData(kapa, &graphdata); + KapaSetLimits(kapa, &graphdata); + KapaClearPlots (kapa); + + KapaSetFont (kapa, "courier", 14); + KapaBox (kapa, &graphdata); + KapaSendLabel (kapa, "ordinate", KAPA_LABEL_XM); + KapaSendLabel (kapa, "pixel values", KAPA_LABEL_YM); + + graphdata.color = KapaColorByName("black"); + graphdata.style = 2; + graphdata.ptype = 2; + KapaPrepPlot (kapa, xSub->n, &graphdata); + KapaPlotVector(kapa, xSub->n, xSub->data.F32, "x"); + KapaPlotVector(kapa, xSub->n, ySub->data.F32, "y"); + + graphdata.color = KapaColorByName("red"); + graphdata.style = 2; + graphdata.ptype = 7; + KapaPrepPlot (kapa, xAll->n, &graphdata); + KapaPlotVector(kapa, xAll->n, xAll->data.F32, "x"); + KapaPlotVector(kapa, xAll->n, pixels->data.F32, "y"); + + graphdata.color = KapaColorByName("blue"); + graphdata.style = 0; + graphdata.ptype = 7; + KapaPrepPlot (kapa, 2, &graphdata); + KapaPlotVector(kapa, 2, xline, "x"); + KapaPlotVector(kapa, 2, yline, "y"); + + pmVisualAskUser (&plotFlag); + return true; +} + +bool pmReadoutCombineVisualCleanup(void) { + + if (!pmVisualIsVisual()) return true; + if (kapa == -1) return true; + + KapaClose(kapa); + return true; +} + +# else + +bool pmReadoutCombineVisualInit(void) { return true; } +bool pmReadoutCombineVisualPixels(psVector *pixels, psVector *mask, float mean) { return true; } +bool pmReadoutCombineVisualCleanup(void) { return true; } + +# endif Index: branches/eam_branches/20090715/psModules/src/imcombine/pmReadoutCombine.h =================================================================== --- branches/eam_branches/20090715/psModules/src/imcombine/pmReadoutCombine.h (revision 25020) +++ branches/eam_branches/20090715/psModules/src/imcombine/pmReadoutCombine.h (revision 25022) @@ -47,4 +47,8 @@ ); +bool pmReadoutCombineVisualInit(void); +bool pmReadoutCombineVisualPixels(psVector *pixels, psVector *mask, float mean); +bool pmReadoutCombineVisualCleanup(void); + /// @} #endif Index: branches/eam_branches/20090715/psModules/src/imcombine/pmSubtractionEquation.c =================================================================== --- branches/eam_branches/20090715/psModules/src/imcombine/pmSubtractionEquation.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/imcombine/pmSubtractionEquation.c (revision 25022) @@ -793,17 +793,17 @@ #ifdef TESTING - // XXX double-check for NAN in data: - for (int iy = 0; iy < stamp->matrix1->numRows; iy++) { - for (int ix = 0; ix < stamp->matrix1->numCols; ix++) { - if (!isfinite(stamp->matrix1->data.F64[iy][ix])) { - fprintf (stderr, "WARNING: NAN in matrix1\n"); - } - } - } - for (int ix = 0; ix < stamp->vector1->n; ix++) { - if (!isfinite(stamp->vector1->data.F64[ix])) { - fprintf (stderr, "WARNING: NAN in vector1\n"); - } - } + // XXX double-check for NAN in data: + for (int iy = 0; iy < stamp->matrix1->numRows; iy++) { + for (int ix = 0; ix < stamp->matrix1->numCols; ix++) { + if (!isfinite(stamp->matrix1->data.F64[iy][ix])) { + fprintf (stderr, "WARNING: NAN in matrix1\n"); + } + } + } + for (int ix = 0; ix < stamp->vector1->n; ix++) { + if (!isfinite(stamp->vector1->data.F64[ix])) { + fprintf (stderr, "WARNING: NAN in vector1\n"); + } + } #endif @@ -818,9 +818,9 @@ #ifdef TESTING - for (int ix = 0; ix < sumVector->n; ix++) { - if (!isfinite(sumVector->data.F64[ix])) { - fprintf (stderr, "WARNING: NAN in vector1\n"); - } - } + for (int ix = 0; ix < sumVector->n; ix++) { + if (!isfinite(sumVector->data.F64[ix])) { + fprintf (stderr, "WARNING: NAN in vector1\n"); + } + } #endif @@ -828,9 +828,9 @@ #ifdef TESTING - for (int ix = 0; ix < sumVector->n; ix++) { - if (!isfinite(sumVector->data.F64[ix])) { - fprintf (stderr, "WARNING: NAN in vector1\n"); - } - } + for (int ix = 0; ix < sumVector->n; ix++) { + if (!isfinite(sumVector->data.F64[ix])) { + fprintf (stderr, "WARNING: NAN in vector1\n"); + } + } { psImage *inverse = psMatrixInvert(NULL, sumMatrix, NULL); @@ -866,10 +866,10 @@ #ifdef TESTING - // XXX double-check for NAN in data: - for (int ix = 0; ix < kernels->solution1->n; ix++) { - if (!isfinite(kernels->solution1->data.F64[ix])) { - fprintf (stderr, "WARNING: NAN in vector1\n"); - } - } + // XXX double-check for NAN in data: + for (int ix = 0; ix < kernels->solution1->n; ix++) { + if (!isfinite(kernels->solution1->data.F64[ix])) { + fprintf (stderr, "WARNING: NAN in vector1\n"); + } + } #endif @@ -1127,4 +1127,23 @@ source = stamp->image1; convolutions = stamp->convolutions1; + + // Having convolved image1 and changed its normalisation, we need to renormalise the residual + // so that it is on the scale of image1. + psImage *image = pmSubtractionKernelImage(kernels, stamp->xNorm, stamp->yNorm, + false); // Kernel image + if (!image) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate image of kernel."); + return false; + } + double sumKernel = 0; // Sum of kernel, for normalising residual + int size = kernels->size; // Half-size of kernel + int fullSize = 2 * size + 1; // Full size of kernel + for (int y = 0; y < fullSize; y++) { + for (int x = 0; x < fullSize; x++) { + sumKernel += image->data.F32[y][x]; + } + } + psFree(image); + devNorm = 1.0 / sumKernel / numPixels; break; case PM_SUBTRACTION_MODE_2: Index: branches/eam_branches/20090715/psModules/src/objects/Makefile.am =================================================================== --- branches/eam_branches/20090715/psModules/src/objects/Makefile.am (revision 25020) +++ branches/eam_branches/20090715/psModules/src/objects/Makefile.am (revision 25022) @@ -10,4 +10,5 @@ pmFootprintArraysMerge.c \ pmFootprintAssignPeaks.c \ + pmFootprintCullPeaks.c \ pmFootprintFind.c \ pmFootprintFindAtPoint.c \ Index: branches/eam_branches/20090715/psModules/src/objects/pmFootprint.h =================================================================== --- branches/eam_branches/20090715/psModules/src/objects/pmFootprint.h (revision 25020) +++ branches/eam_branches/20090715/psModules/src/objects/pmFootprint.h (revision 25022) @@ -51,12 +51,16 @@ void pmSetFootprintArrayIDsForImage(psImage *idImage, const psArray *footprints, // the footprints to insert - const bool relativeIDs); // show IDs starting at 0, not pmFootprint->id + const bool relativeIDs // show IDs starting at 0, not pmFootprint->id + ); psErrorCode pmFootprintsAssignPeaks(psArray *footprints, const psArray *peaks); -psErrorCode pmFootprintArrayCullPeaks(const psImage *img, const psImage *weight, psArray *footprints, - const float nsigma, const float threshold_min); -psErrorCode pmFootprintCullPeaks(const psImage *img, const psImage *weight, pmFootprint *fp, - const float nsigma, const float threshold_min); +psErrorCode pmFootprintCullPeaks(const psImage *img, // the image wherein lives the footprint + const psImage *weight, // corresponding variance image + pmFootprint *fp, // Footprint containing mortal peaks + const float nsigma_delta, // how many sigma above local background a peak needs to be to survive + const float fPadding, // fractional padding added to stdev since bright peaks have unreasonably high significance + const float min_threshold // minimum permitted coll height + ); psArray *pmFootprintArrayToPeaks(const psArray *footprints); Index: branches/eam_branches/20090715/psModules/src/objects/pmFootprintCullPeaks.c =================================================================== --- branches/eam_branches/20090715/psModules/src/objects/pmFootprintCullPeaks.c (revision 25022) +++ branches/eam_branches/20090715/psModules/src/objects/pmFootprintCullPeaks.c (revision 25022) @@ -0,0 +1,265 @@ +/* @file pmFootprint.c + * low-level pmFootprint functions + * + * @author RHL, Princeton & IfA; EAM, IfA + * + * @version $Revision: 1.3 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-12-08 02:51:14 $ + * Copyright 2006 Institute for Astronomy, University of Hawaii + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include "pmSpan.h" +#include "pmFootprint.h" +#include "pmPeaks.h" + + /* + * Examine the peaks in a pmFootprint, and throw away the ones that are not sufficiently + * isolated. More precisely, for each peak find the highest coll that you'd have to traverse + * to reach a still higher peak --- and if that coll's more than nsigma DN below your + * starting point, discard the peak. + */ + +# define IN_PEAK 1 +psErrorCode pmFootprintCullPeaks(const psImage *img, // the image wherein lives the footprint + const psImage *weight, // corresponding variance image + pmFootprint *fp, // Footprint containing mortal peaks + const float nsigma_delta, // how many sigma above local background a peak needs to be to survive + const float fPadding, // fractional padding added to stdev since bright peaks have unreasonably high significance + const float min_threshold) { // minimum permitted coll height + assert (img != NULL); assert (img->type.type == PS_TYPE_F32); + assert (weight != NULL); assert (weight->type.type == PS_TYPE_F32); + assert (img->row0 == weight->row0 && img->col0 == weight->col0); + assert (fp != NULL); + + if (fp->peaks == NULL || fp->peaks->n == 0) { // nothing to do + return PS_ERR_NONE; + } + + psRegion subRegion; // desired subregion; 1 larger than bounding box (grr) + subRegion.x0 = fp->bbox.x0; subRegion.x1 = fp->bbox.x1 + 1; + subRegion.y0 = fp->bbox.y0; subRegion.y1 = fp->bbox.y1 + 1; + + psImage *subImg = psImageSubset((psImage *)img, subRegion); + psImage *subWt = psImageSubset((psImage *)weight, subRegion); + assert (subImg != NULL && subWt != NULL); + + psImage *idImg = psImageAlloc(subImg->numCols, subImg->numRows, PS_TYPE_S32); + + // We need a psArray of peaks brighter than the current peak. + // We reject peaks which either: + // 1) are below the local threshold + // 2) have a brighter peak within their threshold + + // allocate the full-sized array. if the final array is much smaller, we can realloc + // at that point. + psArray *brightPeaks = psArrayAllocEmpty(fp->peaks->n); + psArrayAdd (brightPeaks, 128, fp->peaks->data[0]); + + // The brightest peak is always safe; go through other peaks trying to cull them + for (int i = 1; i < fp->peaks->n; i++) { // n.b. fp->peaks->n can change within the loop + const pmPeak *peak = fp->peaks->data[i]; + int x = peak->x - subImg->col0; + int y = peak->y - subImg->row0; + // + // Find the level nsigma below the peak that must separate the peak + // from any of its friends + // + assert (x >= 0 && x < subImg->numCols && y >= 0 && y < subImg->numRows); + + // const float stdev = sqrt(subWt->data.F32[y][x]); + // float threshold = subImg->data.F32[y][x] - nsigma_delta*stdev; + + const float stdev = sqrt(subWt->data.F32[y][x]); + const float flux = subImg->data.F32[y][x]; + const float fStdev = fabs(stdev/flux); + const float stdev_pad = fabs(flux * hypot(fStdev, fPadding)); + // if flux is negative, careful with fStdev + + float threshold = flux - nsigma_delta*stdev_pad; + + if (isnan(threshold) || threshold < min_threshold) { + // min_threshold is assumed to be below the detection threshold, + // so all the peaks are pmFootprint, and this isn't the brightest + continue; + } + + // XXX EAM : if stdev >= 0, i'm not sure how this can ever be true? + if (threshold > subImg->data.F32[y][x]) { + threshold = subImg->data.F32[y][x] - 10*FLT_EPSILON; + } + + // XXX this is a bit expensive: psImageAlloc for every peak contained in this footprint + // perhaps this should alloc a single ID image above and pass it in to be set. + + // XXX optionally use the faster pmFootprintsFind if the subimage size is large (eg, M31) + + // XXX this is not quite there yet: + // pmFootprint *peakFootprint = NULL; + // int area = subImg->numCols * subImg->numRows; + // if (area > 30000) { + + pmFootprint *peakFootprint = pmFootprintsFindAtPoint(subImg, threshold, brightPeaks, peak->y, peak->x); + + // at this point brightPeaks only has the peaks brighter than the current + + // XXX need to supply the image here + // we set the IDs to either 1 (in peak) or 0 (not in peak) + pmSetFootprintID (idImg, peakFootprint, IN_PEAK); + psFree(peakFootprint); + + // Check if any of the previous (brighter) peaks are within the footprint of this peak + // If so, the current peak is bogus; drop it. + bool keep = true; + for (int j = 0; keep && (j < brightPeaks->n); j++) { + const pmPeak *peak2 = fp->peaks->data[j]; + int x2 = peak2->x - subImg->col0; + int y2 = peak2->y - subImg->row0; + if (idImg->data.S32[y2][x2] == IN_PEAK) + // There's a brighter peak within the footprint above threshold; so cull our initial peak + keep = false; + } + if (!keep) continue; + + psArrayAdd (brightPeaks, 128, fp->peaks->data[i]); + } + + psFree (fp->peaks); + fp->peaks = brightPeaks; + + psFree(idImg); + psFree(subImg); + psFree(subWt); + + return PS_ERR_NONE; +} + + + /* + * Examine the peaks in a pmFootprint, and throw away the ones that are not sufficiently + * isolated. More precisely, for each peak find the highest coll that you'd have to traverse + * to reach a still higher peak --- and if that coll's more than nsigma DN below your + * starting point, discard the peak. + */ +psErrorCode pmFootprintCullPeaks_OLD(const psImage *img, // the image wherein lives the footprint + const psImage *weight, // corresponding variance image + pmFootprint *fp, // Footprint containing mortal peaks + const float nsigma_delta, // how many sigma above local background a peak needs to be to survive + const float fPadding, // fractional padding added to stdev since bright peaks have unreasonably high significance + const float min_threshold) { // minimum permitted coll height + assert (img != NULL); assert (img->type.type == PS_TYPE_F32); + assert (weight != NULL); assert (weight->type.type == PS_TYPE_F32); + assert (img->row0 == weight->row0 && img->col0 == weight->col0); + assert (fp != NULL); + + if (fp->peaks == NULL || fp->peaks->n == 0) { // nothing to do + return PS_ERR_NONE; + } + + psRegion subRegion; // desired subregion; 1 larger than bounding box (grr) + subRegion.x0 = fp->bbox.x0; subRegion.x1 = fp->bbox.x1 + 1; + subRegion.y0 = fp->bbox.y0; subRegion.y1 = fp->bbox.y1 + 1; + const psImage *subImg = psImageSubset((psImage *)img, subRegion); + const psImage *subWt = psImageSubset((psImage *)weight, subRegion); + assert (subImg != NULL && subWt != NULL); + // + // We need a psArray of peaks brighter than the current peak. We'll fake this + // by reusing the fp->peaks but lying about n. + // + // We do this for efficiency (otherwise I'd need two peaks lists), and we are + // rather too chummy with psArray in consequence. But it works. + // + psArray *brightPeaks = psArrayAlloc(0); + psFree(brightPeaks->data); + brightPeaks->data = psMemIncrRefCounter(fp->peaks->data);// use the data from fp->peaks + // + // The brightest peak is always safe; go through other peaks trying to cull them + // + for (int i = 1; i < fp->peaks->n; i++) { // n.b. fp->peaks->n can change within the loop + const pmPeak *peak = fp->peaks->data[i]; + int x = peak->x - subImg->col0; + int y = peak->y - subImg->row0; + // + // Find the level nsigma below the peak that must separate the peak + // from any of its friends + // + assert (x >= 0 && x < subImg->numCols && y >= 0 && y < subImg->numRows); + + // stdev ~ sqrt(flux) : for bright regions (f ~ 1e6, sqrt(f) = 1e3 -- unrealistically small deviations) + // add additional padding in quadrature: + + const float stdev = sqrt(subWt->data.F32[y][x]); + const float flux = subImg->data.F32[y][x]; + const float fStdev = stdev/flux; + const float stdev_pad = flux * hypot(fStdev, fPadding); + float threshold = flux - nsigma_delta*stdev_pad; + + if (isnan(threshold) || threshold < min_threshold) { +#if 1 // min_threshold is assumed to be below the detection threshold, + // so all the peaks are pmFootprint, and this isn't the brightest + // XXX mark peak to be dropped + (void)psArrayRemoveIndex(fp->peaks, i); + i--; // we moved everything down one + continue; +#else +#error n.b. We will be running LOTS of checks at this threshold, so only find the footprint once + threshold = min_threshold; +#endif + } + + // XXX EAM : if stdev >= 0, i'm not sure how this can ever be true? + if (threshold > subImg->data.F32[y][x]) { + threshold = subImg->data.F32[y][x] - 10*FLT_EPSILON; + } + + // XXX this is a bit expensive: psImageAlloc for every peak contained in this footprint + // perhaps this should alloc a single ID image above and pass it in to be set. + + const int peak_id = 1; // the ID for the peak of interest + brightPeaks->n = i; // only stop at a peak brighter than we are + + // XXX optionally use the faster pmFootprintsFind if the subimage size is large (eg, M31) + + pmFootprint *peakFootprint = pmFootprintsFindAtPoint(subImg, threshold, brightPeaks, peak->y, peak->x); + brightPeaks->n = 0; // don't double free + psImage *idImg = pmSetFootprintID(NULL, peakFootprint, peak_id); + psFree(peakFootprint); + + // Check if any of the previous (brighter) peaks are within the footprint of this peak + // If so, the current peak is bogus; drop it. + int j; + for (j = 0; j < i; j++) { + const pmPeak *peak2 = fp->peaks->data[j]; + int x2 = peak2->x - subImg->col0; + int y2 = peak2->y - subImg->row0; + const int peak2_id = idImg->data.S32[y2][x2]; // the ID for some other peak + + if (peak2_id == peak_id) { // There's a brighter peak within the footprint above + ; // threshold; so cull our initial peak + (void)psArrayRemoveIndex(fp->peaks, i); + i--; // we moved everything down one + break; + } + } + if (j == i) { + j++; + } + + psFree(idImg); + } + + brightPeaks->n = 0; psFree(brightPeaks); + psFree((psImage *)subImg); + psFree((psImage *)subWt); + + return PS_ERR_NONE; +} + Index: branches/eam_branches/20090715/psModules/src/objects/pmSource.c =================================================================== --- branches/eam_branches/20090715/psModules/src/objects/pmSource.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/objects/pmSource.c (revision 25022) @@ -242,5 +242,4 @@ extend |= (mySource->maskView == NULL); - // extend = true; if (extend) { // re-create the subimage @@ -250,5 +249,5 @@ mySource->pixels = psImageSubset(readout->image, newRegion); - mySource->variance = psImageSubset(readout->variance, newRegion); + mySource->variance = psImageSubset(readout->variance, newRegion); mySource->maskView = psImageSubset(readout->mask, newRegion); mySource->region = newRegion; Index: branches/eam_branches/20090715/psModules/src/objects/pmSourceContour.c =================================================================== --- branches/eam_branches/20090715/psModules/src/objects/pmSourceContour.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/objects/pmSourceContour.c (revision 25022) @@ -273,5 +273,5 @@ x0 = findContourPos (image, threshold, xR, yR, RIGHT, x1); if (x0 == x1) { - // fprintf (stderr, "top: %d (%d - %d)\n", yR, xR, x1); + // fprintf (stderr, "top: %d (%d - %d)\n", yR, xR, x1); goto pt1; } @@ -282,5 +282,5 @@ x1 = findContourNeg (image, threshold, xR, yR, RIGHT); } - // fprintf (stderr, "pos: %d (%d - %d)\n", yR, x0, x1); + // fprintf (stderr, "pos: %d (%d - %d)\n", yR, x0, x1); xVec->data.F32[Npt + 0] = image->col0 + x0; @@ -288,4 +288,5 @@ yVec->data.F32[Npt + 0] = image->row0 + yR; yVec->data.F32[Npt + 1] = image->row0 + yR; + Npt += 2; @@ -307,5 +308,5 @@ x0 = findContourPos (image, threshold, xR, yR, RIGHT, x1); if (x0 == x1) { - // fprintf (stderr, "top: %d (%d - %d)\n", yR, xR, x1); + // fprintf (stderr, "top: %d (%d - %d)\n", yR, xR, x1); goto pt2; } @@ -322,4 +323,5 @@ yVec->data.F32[Npt + 0] = image->row0 + yR; yVec->data.F32[Npt + 1] = image->row0 + yR; + Npt += 2; @@ -334,5 +336,4 @@ yVec->n = Npt; - // fprintf (stderr, "done\n"); psArray *tmpArray = psArrayAlloc(2); Index: branches/eam_branches/20090715/psModules/src/objects/pmSourceIO_MatchedRefs.c =================================================================== --- branches/eam_branches/20090715/psModules/src/objects/pmSourceIO_MatchedRefs.c (revision 25020) +++ branches/eam_branches/20090715/psModules/src/objects/pmSourceIO_MatchedRefs.c (revision 25022) @@ -95,4 +95,15 @@ psTrace ("psastro", 4, "Trying %ld refstars\n", refstars->n); +# if (0) + // XXX test + FILE *outfile = fopen ("refstars.dat", "w"); + assert (outfile); + for (int nn = 0; nn < refstars->n; nn++) { + pmAstromObj *ref = refstars->data[nn]; + fprintf (outfile, "%lf %lf\n", ref->sky->r*PS_DEG_RAD, ref->sky->d*PS_DEG_RAD); + } + fclose (outfile); +# endif + psArray *matches = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.MATCH"); if (matches == NULL) continue; Index: branches/eam_branches/20090715/psModules/src/psmodules.h =================================================================== --- branches/eam_branches/20090715/psModules/src/psmodules.h (revision 25020) +++ branches/eam_branches/20090715/psModules/src/psmodules.h (revision 25022) @@ -47,4 +47,5 @@ #include #include +#include #include #include @@ -76,4 +77,5 @@ #include #include +#include // the following headers are from psModule:astrom Index: branches/eam_branches/20090715/psModules/test/camera/tap_pmFPAMaskW.c =================================================================== --- branches/eam_branches/20090715/psModules/test/camera/tap_pmFPAMaskW.c (revision 25020) +++ branches/eam_branches/20090715/psModules/test/camera/tap_pmFPAMaskW.c (revision 25022) @@ -356,5 +356,5 @@ // ---------------------------------------------------------------------- // pmReadoutSetVariance() tests: NULL inputs - // bool pmReadoutSetVariance(pmReadout *readout, bool poisson) + // bool pmReadoutSetVariance(pmReadout *readout, const psImage *noiseMap, bool poisson) if (1) { psMemId id = psMemGetId(); @@ -367,6 +367,6 @@ // Set readout == NULL, ensure pmReadoutSetVariance() returnes FALSE, with no seg faults, memory leaks - rc = pmReadoutSetVariance(NULL, false); - ok(!rc, "pmReadoutSetVariance(NULL, false) returned FALSE with null pmReadout input"); + rc = pmReadoutSetVariance(NULL, NULL, false); + ok(!rc, "pmReadoutSetVariance(NULL, NULL, false) returned FALSE with null pmReadout input"); @@ -375,5 +375,7 @@ rc|= psMetadataAddF32(readout->parent->concepts, PS_LIST_HEAD, "CELL.READNOISE", PS_META_REPLACE, NULL, CELL_READNOISE); ok(rc, "Set GAIN and READNOISE in cell->concepts successfully"); - +/* + * Getting the section below to run requires generating a noiseMap + * // Call pmReadoutSetVariance() and then verify that the mask data was set correctly rc = pmReadoutSetVariance(readout, false); @@ -419,4 +421,5 @@ ok(!errorFlag, "pmReadoutSetWeight() set the weight values correctly (Poisson)"); +*/ psFree(fpa); psFree(camera); Index: branches/eam_branches/20090715/psModules/test/concepts/tap_pmConcepts.c =================================================================== --- branches/eam_branches/20090715/psModules/test/concepts/tap_pmConcepts.c (revision 25020) +++ branches/eam_branches/20090715/psModules/test/concepts/tap_pmConcepts.c (revision 25022) @@ -77,4 +77,22 @@ } +psMetadataItem *dummyConceptCopier( + const psMetadataItem *source, + const psMetadataItem *target, + const psMetadata *cameraFormat, + const pmFPA *fpa, + const pmChip *chip, + const pmCell *cell) +{ + if (target == NULL || + source == PM_CONCEPT_SOURCE_NONE || + cameraFormat == NULL || + fpa == NULL || + chip == NULL || + cell == NULL) { + printf("dummyConceptCopier() args are NULL\n"); + } + return(NULL); +} int main(int argc, char* argv[]) @@ -92,5 +110,5 @@ psMetadataItem *blank = psMetadataItemAlloc("myItem1", PS_DATA_BOOL, "I am a boolean", true); pmConceptSpec *tmp = pmConceptSpecAlloc(blank, dummyConceptParser, - dummyConceptFormatter, true); + dummyConceptFormatter, dummyConceptCopier, true); ok(tmp != NULL, "pmConceptSpecAlloc() returned non-NULL"); skip_start(tmp == NULL, 4, "Skipping tests because pmConceptSpecAlloc() returned NULL"); @@ -98,4 +116,5 @@ ok(tmp->parse == dummyConceptParser, "pmConceptSpecAlloc() set the ->parse member correctly"); ok(tmp->format == dummyConceptFormatter, "pmConceptSpecAlloc() set the ->format member correctly"); + ok(tmp->copy == dummyConceptCopier, "pmConceptSpecAlloc() set the ->copy member correctly"); ok(tmp->required == true, "pmConceptSpecAlloc() set the ->required member correctly"); skip_end(); @@ -109,5 +128,5 @@ { psMemId id = psMemGetId(); - pmConceptSpec *tmp = pmConceptSpecAlloc(NULL, NULL, NULL, false); + pmConceptSpec *tmp = pmConceptSpecAlloc(NULL, NULL, NULL, NULL, false); ok(tmp != NULL, "pmConceptSpecAlloc() returned non-NULL with NULL inputs"); psFree(tmp); Index: branches/eam_branches/20090715/psModules/test/objects/tap_pmGrowthCurve.c =================================================================== --- branches/eam_branches/20090715/psModules/test/objects/tap_pmGrowthCurve.c (revision 25020) +++ branches/eam_branches/20090715/psModules/test/objects/tap_pmGrowthCurve.c (revision 25022) @@ -129,5 +129,5 @@ source->modelPSF->dparams->data.F32[PM_PAR_I0] = 1; - source->mode = PM_SOURCE_MODE_SUBTRACTED; + source->mode = PM_SOURCE_MODE_PSFSTAR; source->modelPSF->radiusFit = 15.0; @@ -232,5 +232,5 @@ source->modelPSF->dparams->data.F32[PM_PAR_I0] = 1; - source->mode = PM_SOURCE_MODE_SUBTRACTED; + source->mode = PM_SOURCE_MODE_PSFSTAR; source->modelPSF->radiusFit = 15.0; @@ -334,5 +334,5 @@ source->modelPSF->dparams->data.F32[PM_PAR_I0] = 1; - source->mode = PM_SOURCE_MODE_SUBTRACTED; + source->mode = PM_SOURCE_MODE_PSFSTAR; source->modelPSF->radiusFit = 15.0; Index: branches/eam_branches/20090715/psModules/test/objects/tap_pmSourceIO_PS1_DEV_1.c =================================================================== --- branches/eam_branches/20090715/psModules/test/objects/tap_pmSourceIO_PS1_DEV_1.c (revision 25020) +++ branches/eam_branches/20090715/psModules/test/objects/tap_pmSourceIO_PS1_DEV_1.c (revision 25022) @@ -48,5 +48,5 @@ psMetadata *tableHeader = psMetadataAlloc(); psString extname = psStringCopy("ext"); - bool rc = pmSourcesWrite_PS1_DEV_1(NULL, sources, imageHeader, tableHeader, extname, NULL); + bool rc = pmSourcesWrite_PS1_DEV_1(NULL, sources, imageHeader, tableHeader, extname); ok(rc == false, "pmSourcesWrite_PS1_DEV_1() returned FALSE with NULL psFits input parameter"); psFree(fitsFile); @@ -72,5 +72,5 @@ psMetadata *tableHeader = psMetadataAlloc(); psString extname = psStringCopy("ext"); - bool rc = pmSourcesWrite_PS1_DEV_1(fitsFile, NULL, imageHeader, tableHeader, extname, NULL); + bool rc = pmSourcesWrite_PS1_DEV_1(fitsFile, NULL, imageHeader, tableHeader, extname); ok(rc == false, "pmSourcesWrite_PS1_DEV_1() returned FALSE with NULL pmSource input parameter"); psFree(fitsFile); @@ -96,5 +96,5 @@ psMetadata *tableHeader = psMetadataAlloc(); psString extname = psStringCopy("ext"); - bool rc = pmSourcesWrite_PS1_DEV_1(fitsFile, sources, imageHeader, tableHeader, NULL, NULL); + bool rc = pmSourcesWrite_PS1_DEV_1(fitsFile, sources, imageHeader, tableHeader, NULL); ok(rc == false, "pmSourcesWrite_PS1_DEV_1() returned FALSE with NULL extname input parameter"); psFree(fitsFile); @@ -216,5 +216,5 @@ psMetadata *tableHeader = psMetadataAlloc(); psString extname = psStringCopy("ext"); - bool rc = pmSourcesWrite_PS1_DEV_1(fitsFile, sources, imageHeader, tableHeader, extname, NULL); + bool rc = pmSourcesWrite_PS1_DEV_1(fitsFile, sources, imageHeader, tableHeader, extname); ok(rc == true, "pmSourcesWrite_PS1_DEV_1() returned TRUE with acceptable input parameters"); psFree(fitsFile); Index: branches/eam_branches/20090715/psastro/src/psastroChooseRefstars.c =================================================================== --- branches/eam_branches/20090715/psastro/src/psastroChooseRefstars.c (revision 25020) +++ branches/eam_branches/20090715/psastro/src/psastroChooseRefstars.c (revision 25022) @@ -49,4 +49,12 @@ pmFPAview *view = pmFPAviewAlloc (0); pmFPA *fpa = input->fpa; + + // XXX kind of a hack -- think this through a bit more clearly: + if (psMetadataLookupPtr (&status, fpa->analysis, "MATCHED_REFS")) { + // we loaded a set of matched references from an earlier astrometry + // analysis. however, we are re-doing the astrometry here, so remove + // that prior set of matched references + psMetadataRemoveKey (fpa->analysis, "MATCHED_REFS"); + } // sort by mag @@ -114,4 +122,15 @@ psTrace ("psastro", 4, "Added %ld refstars\n", refstars->n); +# if (0) + // XXX test + FILE *outfile = fopen ("refstars.dat", "w"); + assert (outfile); + for (int nn = 0; nn < refstars->n; nn++) { + pmAstromObj *ref = refstars->data[nn]; + fprintf (outfile, "%lf %lf\n", ref->sky->r*PS_DEG_RAD, ref->sky->d*PS_DEG_RAD); + } + fclose (outfile); +# endif + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.REFSTARS", PS_DATA_ARRAY, "astrometry matches", refstars); psFree (refstars); Index: branches/eam_branches/20090715/psastro/src/psastroLoadRefstars.c =================================================================== --- branches/eam_branches/20090715/psastro/src/psastroLoadRefstars.c (revision 25020) +++ branches/eam_branches/20090715/psastro/src/psastroLoadRefstars.c (revision 25022) @@ -128,4 +128,13 @@ if (!strcmp (outformat, "PS1_DEV_0")) { refstars = psastroReadGetstar_PS1_DEV_0 (fits); + + // XXX test + // FILE *outfile = fopen ("refstars.dat", "w"); + // assert (outfile); + // for (int nn = 0; nn < refstars->n; nn++) { + // pmAstromObj *ref = refstars->data[nn]; + // fprintf (outfile, "%lf %lf\n", ref->sky->r*PS_DEG_RAD, ref->sky->d*PS_DEG_RAD); + // } + // fclose (outfile); } if (refstars == NULL) { @@ -218,6 +227,6 @@ psMetadata *row = psFitsReadTableRow(fits, i); // Table row - ref->sky->r = RAD_DEG*psMetadataLookupF32 (&status, row, "RA"); - ref->sky->d = RAD_DEG*psMetadataLookupF32 (&status, row, "DEC"); + ref->sky->r = RAD_DEG*psMetadataLookupF64 (&status, row, "RA"); + ref->sky->d = RAD_DEG*psMetadataLookupF64 (&status, row, "DEC"); ref->Mag = psMetadataLookupF32 (&status, row, "MAG"); float MagC1 = psMetadataLookupF32 (&status, row, "MAG_C1"); Index: branches/eam_branches/20090715/psconfig/pschecklibs =================================================================== --- branches/eam_branches/20090715/psconfig/pschecklibs (revision 25020) +++ branches/eam_branches/20090715/psconfig/pschecklibs (revision 25022) @@ -100,4 +100,11 @@ # search for, and drop, existing libdir entry in libpath? unshift @libpath, $libdir; + +# XXX a temporary hack for libreadline (this failed... needed to modify libreadline/support/shobj-conf +# $shlib_libs = $ENV{'SHLIB_LIBS'}; +# unless ($shlib_libs =~ m|-lncurses|) { +# $shlib_libs .= " -lncurses"; +# } +# $ENV{'SHLIB_LIBS'} = $shlib_libs; # add the path defined by PATH Index: branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.libs =================================================================== --- branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.libs (revision 25020) +++ branches/eam_branches/20090715/psconfig/tagsets/ipp-2.9.libs (revision 25022) @@ -29,5 +29,5 @@ lib libpthread NONE NONE NONE NONE N NONE NONE NONE lib libncurses curses,termcap NONE ncurses-5.6.tar.gz ncurses-5.6 N NONE NONE NONE -lib libreadline NONE NONE readline-5.2-p12.tar.gz readline-5.2-p12 Y NONE NONE NONE +lib libreadline NONE NONE readline-5.2-p13.tar.gz readline-5.2-p13 Y NONE NONE NONE lib libz NONE NONE zlib-1.2.3.tar.gz zlib-1.2.3 N --shared NONE NONE lib libpng NONE NONE libpng-1.2.15.tar.gz libpng-1.2.15 N NONE NONE NONE Index: branches/eam_branches/20090715/psphot/doc/notes.20090523.txt =================================================================== --- branches/eam_branches/20090715/psphot/doc/notes.20090523.txt (revision 25020) +++ branches/eam_branches/20090715/psphot/doc/notes.20090523.txt (revision 25022) @@ -1,19 +1,58 @@ -20090523 : extended source analysis -- further development +20090725 : extended source radial profiles - issues to address: + I need to fix the Petrosian and similar code. There are several issues: - * quality of moments used for initial psf selection - - what is the behavior of moments with Gaussian windowing as function of S/N & size? - - can we choose a sigma value that will give reliable moments for an expected range of seeing? + 1) correct implementation of the actual Petrosian radius / Petrosian flux measurement: - * moments for galaxy model guesses - - trade-off between Gaussian windowing, tophat windowing, and S/N windowing in footprint area + R_p : Given F(R) : F(0) / F(R_p) = 0.9 ?? (something like this...) - initial dev work: + Actually implemented by calculating the average surface brightness in annullii: - * add Gaussian windowing, S/N windowing, (tophat windowing) and options to select + R_p : Given F(R) -> define f(R) = \sum_{R-dR}^{R+dR}F(R) / 4\pi R dR + R_p : f(dR) / F(R_p) = P_ratio - * check visualization of moments + 2) need to construct a clean, normalized radial profile to make this measurement + + * generate radial profiles for each of N annular wedges + * find R_50 : f(0) / f(R_50) = 0.5 + $ fit the ellipse of R_50 to Rmaj sin(T) + Rmin cos(T) + + + cos(a+b) = cos(a) cos(b) - sin(a) sin(b) + sin(a+b) = cos(a) sin(b) + cos(b) sin(a) + + cos(theta+To) = cos(theta) cos(To) - sin(theta) sin(To) + sin(theta+To) = cos(theta) sin(To) + sin(theta) cos(To) + + R_x = R_maj cos(theta + theta_o) = R_maj cos(theta) cos(theta_o) - Rmaj sin(theta) sin(theta_o) + R_y = R_min sin(theta + theta_o) = R_min cos(theta) sin(theta_o) + Rmin sin(theta) cos(theta_o) + + R_X = R_x cos(phi) - R_y sin(phi) + R_Y = R_x sin(phi) + R_y cos(phi) + + R_X = R_maj cos(phi) cos(theta + To) - R_min sin(phi) sin(theta + To) + R_Y = R_maj sin(phi) cos(theta + To) + R_min cos(phi) sin(theta + To) + + R_X = (R_maj cos(phi) cos (To) - R_min sin(phi) sin(To)) cos(theta) - (Rmaj cos(phi) sin(To) + Rmin sin(phi) cos(To)) sin(theta) + R_Y = (R_maj sin(phi) cos (To) + R_min cos(phi) sin(To)) cos(theta) - (Rmaj cos(phi) sin(To) + Rmin sin(phi) cos(To)) sin(theta) + + + R_X = Ax cos(theta) + Bx sin(theta) + R_Y = Ay cos(theta) + By sin(theta) + + Ax = \sum (R_X * cos(theta)) + Bx = \sum (R_X * sin(theta)) + Ay = \sum (R_Y * cos(theta)) + By = \sum (R_Y * sin(theta)) + + R_maj = sqrt(Ax^2 + Ay^2) + R_min = sqrt(Bx^2 + By^2) + + phi = atan2(Bx + By, Ax + Ay) + + *** get rest of math from my notebook... + + 20090525 : some clarity of issues: @@ -40,2 +79,20 @@ * QGAUSS with see + +20090523 : extended source analysis -- further development + + issues to address: + + * quality of moments used for initial psf selection + - what is the behavior of moments with Gaussian windowing as function of S/N & size? + - can we choose a sigma value that will give reliable moments for an expected range of seeing? + + * moments for galaxy model guesses + - trade-off between Gaussian windowing, tophat windowing, and S/N windowing in footprint area + + initial dev work: + + * add Gaussian windowing, S/N windowing, (tophat windowing) and options to select + + * check visualization of moments + Index: branches/eam_branches/20090715/psphot/src/psphot.h =================================================================== --- branches/eam_branches/20090715/psphot/src/psphot.h (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphot.h (revision 25022) @@ -157,5 +157,5 @@ bool psphotSubWithTest (pmSource *source, bool useState, psImageMaskType maskVal); bool psphotSetState (pmSource *source, bool curState, psImageMaskType maskVal); -bool psphotDeblendSatstars (psArray *sources, psMetadata *recipe); +bool psphotDeblendSatstars (pmReadout *readout, psArray *sources, psMetadata *recipe); bool psphotSourceSize (pmConfig *config, pmReadout *readout, psArray *sources, psMetadata *recipe, long first); Index: branches/eam_branches/20090715/psphot/src/psphotApResid.c =================================================================== --- branches/eam_branches/20090715/psphot/src/psphotApResid.c (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphotApResid.c (revision 25022) @@ -168,4 +168,7 @@ if (source->mode & PM_SOURCE_MODE_POOR) SKIPSTAR ("POOR STAR"); + if (source->mode & PM_SOURCE_MODE_EXT_LIMIT) SKIPSTAR ("EXTENDED"); + if (source->mode & PM_SOURCE_MODE_CR_LIMIT) SKIPSTAR ("COSMIC RAY"); + if (!isfinite(source->apMag) || !isfinite(source->psfMag)) { continue; @@ -317,4 +320,5 @@ */ +// XXX this still sucks... need a better way to estimate the error floor... bool psphotMagErrorScale (float *errorScale, float *errorFloor, psVector *dMag, psVector *dap, psVector *mask, int nGroup) { @@ -385,4 +389,5 @@ for (int i = 0; i < dSo->n; i++) { *errorFloor = dSo->data.F32[i]; + if (fabs(*errorFloor) <= FLT_EPSILON) continue; if (isfinite(*errorFloor)) break; } Index: branches/eam_branches/20090715/psphot/src/psphotCullPeaks.c =================================================================== --- branches/eam_branches/20090715/psphot/src/psphotCullPeaks.c (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphotCullPeaks.c (revision 25022) @@ -18,25 +18,14 @@ nsigma_min = 0; } + float fPadding = psMetadataLookupF32(&status, recipe, "FOOTPRINT_CULL_NSIGMA_PAD"); + if (!status) { + fPadding = 0; + } const float skyStdev = psMetadataLookupF32(NULL, recipe, "SKY_STDEV"); - - return pmFootprintArrayCullPeaks(image, weight, footprints, - nsigma_delta, nsigma_min*skyStdev); -} - - -/* - * Cull an entire psArray of pmFootprints - * XXX drop this intermediate level function? - */ -psErrorCode -pmFootprintArrayCullPeaks(const psImage *img, // the image wherein lives the footprint - const psImage *weight, // corresponding variance image - psArray *footprints, // array of pmFootprints - const float nsigma_delta, // how many sigma above local background a peak - // needs to be to survive - const float min_threshold) { // minimum permitted coll height + const float min_threshold = nsigma_min*skyStdev; + for (int i = 0; i < footprints->n; i++) { pmFootprint *fp = footprints->data[i]; - if (pmFootprintCullPeaks(img, weight, fp, nsigma_delta, min_threshold) != PS_ERR_NONE) { + if (pmFootprintCullPeaks(image, weight, fp, nsigma_delta, fPadding, min_threshold) != PS_ERR_NONE) { return psError(PS_ERR_UNKNOWN, false, "Culling pmFootprint %d", fp->id); } @@ -45,220 +34,2 @@ return PS_ERR_NONE; } - - /* - * Examine the peaks in a pmFootprint, and throw away the ones that are not sufficiently - * isolated. More precisely, for each peak find the highest coll that you'd have to traverse - * to reach a still higher peak --- and if that coll's more than nsigma DN below your - * starting point, discard the peak. - */ -psErrorCode pmFootprintCullPeaks_OLD(const psImage *img, // the image wherein lives the footprint - const psImage *weight, // corresponding variance image - pmFootprint *fp, // Footprint containing mortal peaks - const float nsigma_delta, // how many sigma above local background a peak - // needs to be to survive - const float min_threshold) { // minimum permitted coll height - assert (img != NULL); assert (img->type.type == PS_TYPE_F32); - assert (weight != NULL); assert (weight->type.type == PS_TYPE_F32); - assert (img->row0 == weight->row0 && img->col0 == weight->col0); - assert (fp != NULL); - - if (fp->peaks == NULL || fp->peaks->n == 0) { // nothing to do - return PS_ERR_NONE; - } - - psRegion subRegion; // desired subregion; 1 larger than bounding box (grr) - subRegion.x0 = fp->bbox.x0; subRegion.x1 = fp->bbox.x1 + 1; - subRegion.y0 = fp->bbox.y0; subRegion.y1 = fp->bbox.y1 + 1; - const psImage *subImg = psImageSubset((psImage *)img, subRegion); - const psImage *subWt = psImageSubset((psImage *)weight, subRegion); - assert (subImg != NULL && subWt != NULL); - // - // We need a psArray of peaks brighter than the current peak. We'll fake this - // by reusing the fp->peaks but lying about n. - // - // We do this for efficiency (otherwise I'd need two peaks lists), and we are - // rather too chummy with psArray in consequence. But it works. - // - psArray *brightPeaks = psArrayAlloc(0); - psFree(brightPeaks->data); - brightPeaks->data = psMemIncrRefCounter(fp->peaks->data);// use the data from fp->peaks - // - // The brightest peak is always safe; go through other peaks trying to cull them - // - for (int i = 1; i < fp->peaks->n; i++) { // n.b. fp->peaks->n can change within the loop - const pmPeak *peak = fp->peaks->data[i]; - int x = peak->x - subImg->col0; - int y = peak->y - subImg->row0; - // - // Find the level nsigma below the peak that must separate the peak - // from any of its friends - // - assert (x >= 0 && x < subImg->numCols && y >= 0 && y < subImg->numRows); - const float stdev = sqrt(subWt->data.F32[y][x]); - float threshold = subImg->data.F32[y][x] - nsigma_delta*stdev; - if (isnan(threshold) || threshold < min_threshold) { -#if 1 // min_threshold is assumed to be below the detection threshold, - // so all the peaks are pmFootprint, and this isn't the brightest - // XXX mark peak to be dropped - (void)psArrayRemoveIndex(fp->peaks, i); - i--; // we moved everything down one - continue; -#else -#error n.b. We will be running LOTS of checks at this threshold, so only find the footprint once - threshold = min_threshold; -#endif - } - - // XXX EAM : if stdev >= 0, i'm not sure how this can ever be true? - if (threshold > subImg->data.F32[y][x]) { - threshold = subImg->data.F32[y][x] - 10*FLT_EPSILON; - } - - // XXX this is a bit expensive: psImageAlloc for every peak contained in this footprint - // perhaps this should alloc a single ID image above and pass it in to be set. - - const int peak_id = 1; // the ID for the peak of interest - brightPeaks->n = i; // only stop at a peak brighter than we are - - // XXX optionally use the faster pmFootprintsFind if the subimage size is large (eg, M31) - - pmFootprint *peakFootprint = pmFootprintsFindAtPoint(subImg, threshold, brightPeaks, peak->y, peak->x); - brightPeaks->n = 0; // don't double free - psImage *idImg = pmSetFootprintID(NULL, peakFootprint, peak_id); - psFree(peakFootprint); - - // Check if any of the previous (brighter) peaks are within the footprint of this peak - // If so, the current peak is bogus; drop it. - int j; - for (j = 0; j < i; j++) { - const pmPeak *peak2 = fp->peaks->data[j]; - int x2 = peak2->x - subImg->col0; - int y2 = peak2->y - subImg->row0; - const int peak2_id = idImg->data.S32[y2][x2]; // the ID for some other peak - - if (peak2_id == peak_id) { // There's a brighter peak within the footprint above - ; // threshold; so cull our initial peak - (void)psArrayRemoveIndex(fp->peaks, i); - i--; // we moved everything down one - break; - } - } - if (j == i) { - j++; - } - - psFree(idImg); - } - - brightPeaks->n = 0; psFree(brightPeaks); - psFree((psImage *)subImg); - psFree((psImage *)subWt); - - return PS_ERR_NONE; -} - - /* - * Examine the peaks in a pmFootprint, and throw away the ones that are not sufficiently - * isolated. More precisely, for each peak find the highest coll that you'd have to traverse - * to reach a still higher peak --- and if that coll's more than nsigma DN below your - * starting point, discard the peak. - */ - -# define IN_PEAK 1 -psErrorCode pmFootprintCullPeaks(const psImage *img, // the image wherein lives the footprint - const psImage *weight, // corresponding variance image - pmFootprint *fp, // Footprint containing mortal peaks - const float nsigma_delta, // how many sigma above local background a peak - // needs to be to survive - const float min_threshold) { // minimum permitted coll height - assert (img != NULL); assert (img->type.type == PS_TYPE_F32); - assert (weight != NULL); assert (weight->type.type == PS_TYPE_F32); - assert (img->row0 == weight->row0 && img->col0 == weight->col0); - assert (fp != NULL); - - if (fp->peaks == NULL || fp->peaks->n == 0) { // nothing to do - return PS_ERR_NONE; - } - - psRegion subRegion; // desired subregion; 1 larger than bounding box (grr) - subRegion.x0 = fp->bbox.x0; subRegion.x1 = fp->bbox.x1 + 1; - subRegion.y0 = fp->bbox.y0; subRegion.y1 = fp->bbox.y1 + 1; - - psImage *subImg = psImageSubset((psImage *)img, subRegion); - psImage *subWt = psImageSubset((psImage *)weight, subRegion); - assert (subImg != NULL && subWt != NULL); - - psImage *idImg = psImageAlloc(subImg->numCols, subImg->numRows, PS_TYPE_S32); - - // We need a psArray of peaks brighter than the current peak. - // We reject peaks which either: - // 1) are below the local threshold - // 2) have a brighter peak within their threshold - - // allocate the full-sized array. if the final array is much smaller, we can realloc - // at that point. - psArray *brightPeaks = psArrayAllocEmpty(fp->peaks->n); - psArrayAdd (brightPeaks, 128, fp->peaks->data[0]); - - // The brightest peak is always safe; go through other peaks trying to cull them - for (int i = 1; i < fp->peaks->n; i++) { // n.b. fp->peaks->n can change within the loop - const pmPeak *peak = fp->peaks->data[i]; - int x = peak->x - subImg->col0; - int y = peak->y - subImg->row0; - // - // Find the level nsigma below the peak that must separate the peak - // from any of its friends - // - assert (x >= 0 && x < subImg->numCols && y >= 0 && y < subImg->numRows); - const float stdev = sqrt(subWt->data.F32[y][x]); - float threshold = subImg->data.F32[y][x] - nsigma_delta*stdev; - if (isnan(threshold) || threshold < min_threshold) { - // min_threshold is assumed to be below the detection threshold, - // so all the peaks are pmFootprint, and this isn't the brightest - continue; - } - - // XXX EAM : if stdev >= 0, i'm not sure how this can ever be true? - if (threshold > subImg->data.F32[y][x]) { - threshold = subImg->data.F32[y][x] - 10*FLT_EPSILON; - } - - // XXX this is a bit expensive: psImageAlloc for every peak contained in this footprint - // perhaps this should alloc a single ID image above and pass it in to be set. - - // XXX optionally use the faster pmFootprintsFind if the subimage size is large (eg, M31) - - // at this point brightPeaks only has the peaks brighter than the current - pmFootprint *peakFootprint = pmFootprintsFindAtPoint(subImg, threshold, brightPeaks, peak->y, peak->x); - - // XXX need to supply the image here - // we set the IDs to either 1 (in peak) or 0 (not in peak) - pmSetFootprintID (idImg, peakFootprint, IN_PEAK); - psFree(peakFootprint); - - // Check if any of the previous (brighter) peaks are within the footprint of this peak - // If so, the current peak is bogus; drop it. - bool keep = true; - for (int j = 0; keep && (j < brightPeaks->n); j++) { - const pmPeak *peak2 = fp->peaks->data[j]; - int x2 = peak2->x - subImg->col0; - int y2 = peak2->y - subImg->row0; - if (idImg->data.S32[y2][x2] == IN_PEAK) - // There's a brighter peak within the footprint above threshold; so cull our initial peak - keep = false; - } - if (!keep) continue; - - psArrayAdd (brightPeaks, 128, fp->peaks->data[i]); - } - - psFree (fp->peaks); - fp->peaks = brightPeaks; - - psFree(idImg); - psFree(subImg); - psFree(subWt); - - return PS_ERR_NONE; -} - Index: branches/eam_branches/20090715/psphot/src/psphotDeblendSatstars.c =================================================================== --- branches/eam_branches/20090715/psphot/src/psphotDeblendSatstars.c (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphotDeblendSatstars.c (revision 25022) @@ -1,5 +1,5 @@ # include "psphotInternal.h" -bool psphotDeblendSatstars (psArray *sources, psMetadata *recipe) { +bool psphotDeblendSatstars (pmReadout *readout, psArray *sources, psMetadata *recipe) { int N; @@ -9,6 +9,10 @@ int Nblend = 0; - float SAT_TEST_LEVEL = 50000; float SAT_MIN_RADIUS = 5.0; + + bool status; + pmCell *cell = readout->parent; + float SATURATION = 0.75*psMetadataLookupF32 (&status, cell->concepts, "CELL.SATURATION"); + float SAT_TEST_LEVEL = 0.5*SATURATION; // we need sources spatially-sorted to find overlaps @@ -32,5 +36,5 @@ // XXX filter? if (source->mode & PM_SOURCE_MODE_SATSTAR) continue; if (source->mode & PM_SOURCE_MODE_BLEND) continue; - if (source->peak->flux < SAT_TEST_LEVEL) continue; + if (source->peak->flux < SATURATION) continue; // save these for reference below @@ -45,6 +49,5 @@ psVector *xVec = contour->data[0]; psVector *yVec = contour->data[1]; - - // XXX should we filter based on the number of pixels in the contour? + if (xVec->n < 5) continue; // find the center of the contour (let's just use mid[x,y]) @@ -62,4 +65,9 @@ int yCenter = 0.5*(yMin + yMax); psFree (contour); + + psAssert (xCenter >= source->pixels->col0, "invalid shift in object center"); + psAssert (xCenter < source->pixels->col0 + source->pixels->numCols, "invalid shift in object center"); + psAssert (yCenter >= source->pixels->row0, "invalid shift in object center"); + psAssert (yCenter < source->pixels->row0 + source->pixels->numRows, "invalid shift in object center"); // reset the peak for this source to the value of the center pixel Index: branches/eam_branches/20090715/psphot/src/psphotGuessModels.c =================================================================== --- branches/eam_branches/20090715/psphot/src/psphotGuessModels.c (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphotGuessModels.c (revision 25022) @@ -6,19 +6,4 @@ // 2) loop over the sources once and associate them with their cell // 3) define the threaded function to work with sources for a given cell - -// A guess for when the moments aren't available -static pmModel *wildGuess(pmSource *source, // Source for which to guess - pmPSF *psf // The point-spread function - ) -{ - pmModel *model = pmModelAlloc(psf->type); - psF32 *PAR = model->params->data.F32; - PAR[PM_PAR_SKY] = 0; - // XXX get this from the image pixels - PAR[PM_PAR_I0] = source->peak->flux; - PAR[PM_PAR_XPOS] = source->peak->xf; - PAR[PM_PAR_YPOS] = source->peak->yf; - return model; -} // construct an initial PSF model for each object @@ -81,11 +66,4 @@ } psFree(job); - -# if (0) - if (!psphotGuessModel_Unthreaded (readout, cells->data[j], psf, maskVal, markVal)) { - psError(PS_ERR_UNKNOWN, false, "Unable to guess model."); - return false; - } -# endif } @@ -152,47 +130,46 @@ nSrc ++; - // XXX if a source is faint, it will not have moments measured. - // it must be modelled as a PSF. In this case, we need to use - // the peak centroid to get the coordinates and get the peak flux - // from the image? - pmModel *modelEXT; - if (!source->moments) { - modelEXT = wildGuess(source, psf); + // the guess central intensity comes from the peak: + float Io = source->peak->flux; + + // We have two options to get a guess for the object position: the position from the + // peak and the position from the moments. Use the peak position if (a) there are no + // moments and (b) the sources is not saturated + + bool useMoments = false; + useMoments = (source->mode & PM_SOURCE_MODE_SATSTAR); // we only want to try if SATSTAR is set, but.. + useMoments = (useMoments && source->moments); // can't if there are no moments + useMoments = (useMoments && source->moments->nPixels); // can't if the moments were not measured + useMoments = (useMoments && !(source->mode && PM_SOURCE_MODE_MOMENTS_FAILURE)); // can't if the moments failed... + + float Xo, Yo; + if (useMoments) { + Xo = source->moments->Mx; + Yo = source->moments->My; } else { - // use the source moments, etc to guess basic model parameters - modelEXT = pmSourceModelGuess (source, psf->type); // ALLOC X5 - if (!modelEXT) { - modelEXT = wildGuess(source, psf); - } - // these valuse are set in pmSourceModelGuess, should this rule be in there as well? - if (source->mode & PM_SOURCE_MODE_SATSTAR) { - modelEXT->params->data.F32[PM_PAR_XPOS] = source->moments->Mx; - modelEXT->params->data.F32[PM_PAR_YPOS] = source->moments->My; - } else { - modelEXT->params->data.F32[PM_PAR_XPOS] = source->peak->xf; - modelEXT->params->data.F32[PM_PAR_YPOS] = source->peak->yf; - } + Xo = source->peak->xf; + Yo = source->peak->yf; } - // set PSF parameters for this model (apply 2D shape model) - pmModel *modelPSF = pmModelFromPSF (modelEXT, psf); // ALLOC X5 + // set PSF parameters for this model (apply 2D shape model to coordinates Xo, Yo) + pmModel *modelPSF = pmModelFromPSFforXY(psf, Xo, Yo, Io); + if (modelPSF == NULL) { - psWarning ("Failed to determine PSF model at r,c = (%d,%d); trying centre of image", - source->peak->y, source->peak->x); + psWarning ("Failed to determine PSF model at (%f,%f); trying image center", Xo, Yo); - // Try the center of the image - modelEXT->params->data.F32[PM_PAR_XPOS] = 0.5*readout->image->numCols; - modelEXT->params->data.F32[PM_PAR_YPOS] = 0.5*readout->image->numRows; - modelPSF = pmModelFromPSF (modelEXT, psf); + float Xc = 0.5*readout->image->numCols; + float Yc = 0.5*readout->image->numRows; + pmModel *modelPSF = pmModelFromPSFforXY(psf, Xc, Yc, Io); if (modelPSF == NULL) { psError(PSPHOT_ERR_PSF, false, "Failed to determine PSF model at center of image"); - psFree(modelEXT); return false; } + + // Now set the object position at the expected location: + modelPSF->params->data.F32[PM_PAR_XPOS] = Xo; + modelPSF->params->data.F32[PM_PAR_YPOS] = Yo; source->mode |= PM_SOURCE_MODE_BADPSF; } - psFree (modelEXT); // FREE (x3) - // XXX need to define the guess flux? // set the fit radius based on the object flux limit and the model // this function affects the mask pixels @@ -209,85 +186,2 @@ return true; } - -# if (0) -// construct models only for sources in the specified region -bool psphotGuessModel_Unthreaded (pmReadout *readout, psArray *sources, pmPSF *psf, psImageMaskType maskVal, psImageMaskType markVal) { - - int nSrc = 0; - - for (int i = 0; i < sources->n; i++) { - pmSource *source = sources->data[i]; - - // XXXX this is just for a test: use this to mark sources for which the model is measured - // check later that all are used. - source->mode |= PM_SOURCE_MODE_EXT_LIMIT; - - // skip non-astronomical objects (very likely defects) - if (source->type == PM_SOURCE_TYPE_DEFECT) continue; - if (source->type == PM_SOURCE_TYPE_SATURATED) continue; - if (!source->peak) continue; - - nSrc ++; - - // XXX if a source is faint, it will not have moments measured. - // it must be modelled as a PSF. In this case, we need to use - // the peak centroid to get the coordinates and get the peak flux - // from the image? - pmModel *modelEXT; - if (!source->moments) { - modelEXT = wildGuess(source, psf); - } else { - // use the source moments, etc to guess basic model parameters - modelEXT = pmSourceModelGuess (source, psf->type); // ALLOC - if (!modelEXT) { - modelEXT = wildGuess(source, psf); - } - // these valuse are set in pmSourceModelGuess, should this rule be in there as well? - if (source->mode & PM_SOURCE_MODE_SATSTAR) { - modelEXT->params->data.F32[PM_PAR_XPOS] = source->moments->Mx; - modelEXT->params->data.F32[PM_PAR_YPOS] = source->moments->My; - } else { - modelEXT->params->data.F32[PM_PAR_XPOS] = source->peak->xf; - modelEXT->params->data.F32[PM_PAR_YPOS] = source->peak->yf; - } - } - - // set PSF parameters for this model (apply 2D shape model) - pmModel *modelPSF = pmModelFromPSF (modelEXT, psf); // ALLOC - if (modelPSF == NULL) { - psError(PSPHOT_ERR_PSF, false, - "Failed to determine PSF model at r,c = (%d,%d); trying centre of image", - source->peak->y, source->peak->x); - // - // Try the centre of the image - // - modelEXT->params->data.F32[PM_PAR_XPOS] = 0.5*readout->image->numCols; - modelEXT->params->data.F32[PM_PAR_YPOS] = 0.5*readout->image->numRows; - modelPSF = pmModelFromPSF (modelEXT, psf); - if (modelPSF == NULL) { - psError(PSPHOT_ERR_PSF, false, - "Failed to determine PSF model at centre of image"); - psFree(modelEXT); - return false; - } - - source->mode |= PM_SOURCE_MODE_BADPSF; - } - psFree (modelEXT); - - // XXX need to define the guess flux? - // set the fit radius based on the object flux limit and the model - // this function affects the mask pixels - psphotCheckRadiusPSF (readout, source, modelPSF, markVal); - - // set the source PSF model - source->modelPSF = modelPSF; - source->modelPSF->residuals = psf->residuals; - - pmSourceCacheModel (source, maskVal); - - } - - return true; -} -# endif Index: branches/eam_branches/20090715/psphot/src/psphotReadout.c =================================================================== --- branches/eam_branches/20090715/psphot/src/psphotReadout.c (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphotReadout.c (revision 25022) @@ -87,5 +87,5 @@ // find blended neighbors of very saturated stars // XXX merge this with Basic Deblend? - psphotDeblendSatstars (sources, recipe); + psphotDeblendSatstars (readout, sources, recipe); // mark blended peaks PS_SOURCE_BLEND @@ -236,2 +236,3 @@ return psphotReadoutCleanup(config, readout, recipe, detections, psf, sources); } + Index: branches/eam_branches/20090715/psphot/src/psphotReadoutMinimal.c =================================================================== --- branches/eam_branches/20090715/psphot/src/psphotReadoutMinimal.c (revision 25020) +++ branches/eam_branches/20090715/psphot/src/psphotReadoutMinimal.c (revision 25022) @@ -59,5 +59,5 @@ // find blended neighbors of very saturated stars // XXX merge this with Basic Deblend? - psphotDeblendSatstars (sources, recipe); + psphotDeblendSatstars (readout, sources, recipe); // mark blended peaks PS_SOURCE_BLEND Index: branches/eam_branches/20090715/pstamp/scripts/dquery_finish.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/dquery_finish.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/dquery_finish.pl (revision 25022) @@ -20,5 +20,5 @@ use PS::IPP::Config qw( :standard ); -my ( $req_id, $req_name, $req_file, $product, $dbname, $verbose, $save_temps ); +my ( $req_id, $req_name, $req_file, $product, $dbname, $dbserver, $verbose, $save_temps ); GetOptions( @@ -28,4 +28,5 @@ 'product=s' => \$product, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -56,4 +57,8 @@ my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT'); exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} if ($product eq "NULL") { @@ -89,4 +94,5 @@ my $command = "$pstamptool -listjob -req_id $req_id"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -149,4 +155,5 @@ $command .= " -fault $fault" if $fault; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/scripts/dqueryparse.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/dqueryparse.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/dqueryparse.pl (revision 25022) @@ -26,5 +26,5 @@ ); -my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $verbose, $save_temps); +my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps); # @@ -39,4 +39,5 @@ 'mode=s' => \$mode, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -69,4 +70,9 @@ warn("Can't find required tools."); exit ($PS_EXIT_CONFIG_ERROR); +} + +if (!$dbserver) { + my $ipprc = = PS::IPP::Config->new(); + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); } @@ -131,4 +137,5 @@ $command .= " -rownum 0"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -153,4 +160,5 @@ $command .= " -fault $result" if $result; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_finish.pl (revision 25022) @@ -24,5 +24,5 @@ use PS::IPP::PStamp::Job qw( :standard ); -my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output); +my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output); # the char to the right of the bar may be used as a single - alias for the longer name @@ -35,4 +35,5 @@ 'out_dir=s' => \$out_dir, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -50,4 +51,8 @@ my $logDest = "$out_dir/psfinish.$req_id.log"; $ipprc->redirect_output($logDest); +} + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); } @@ -89,5 +94,5 @@ if (!mkdir $out_dir) { print STDERR "cannot create output directory $out_dir"; - stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname); + stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR); } @@ -97,10 +102,10 @@ # request print STDERR "output directory $out_dir exists but is not a directory"; - stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname); + stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR); } if (! -e $req_file ) { print STDERR "request file $req_file is missing\n"; - stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); + stop_request($req_id, $PS_EXIT_CONFIG_ERROR); } @@ -112,5 +117,5 @@ # point print STDERR "failed to read request_file $req_file" ; - stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname); + stop_request($req_id, $PS_EXIT_CONFIG_ERROR); } @@ -138,4 +143,5 @@ my $command = "$pstamptool -listjob -req_id $req_id"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -167,4 +173,6 @@ } + my $exp_info; + my $last_exp_id = -1; foreach my $job (@jobs) { my $job_id = $job->{job_id}; @@ -175,10 +183,18 @@ my ($row, $req_info, $project) = get_request_info($rows, $rownum); - my $proj_hash = resolve_project($ipprc, $project, $dbname); + + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); my $image_db = $proj_hash->{dbname}; - - # get the metadata for the exposure (if any i.e. stack) - # returns an appropriate string if !$exp_id - my $exp_info = get_exposure_info($image_db, $exp_id); + if (!$image_db) { + carp("failed to find imagedb for project: $project"); + stop_request($req_id, $PS_EXIT_CONFIG_ERROR); + } + + if ($exp_id ne $last_exp_id) { + # get the metadata for the exposure (if any i.e. stack) + # returns an appropriate string if !$exp_id + $exp_info = get_exposure_info($image_db, $exp_id); + $last_exp_id = $exp_id; + } if (($job_type eq "stamp") || ($job_type eq "get_image")) { @@ -251,4 +267,5 @@ my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $request_fault"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -263,8 +280,8 @@ my $req_id = shift; my $fault = shift; - my $dbname = shift; my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $fault"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -285,5 +302,5 @@ # Create a results file module and provide a list of the names (we have the data in the columns) my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|"; - $rowinfo .= "$row->{ID}|$row->{CLASS_ID}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|"; + $rowinfo .= "$row->{ID}|$row->{TESS_ID}|$row->{COMPONENT}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|"; $rowinfo .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|"; $rowinfo .= "$row->{WIDTH}|$row->{HEIGHT}"; Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_job_run.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_job_run.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_job_run.pl (revision 25022) @@ -14,23 +14,29 @@ use Digest::MD5::File qw( file_md5_hex ); use PS::IPP::PStamp::RequestFile qw( :standard ); - -my $verbose; -my $dbname; -my $job_id; -my $redirect_output; -my $output_base; +use IPC::Cmd 0.36 qw( can_run run ); + +use PS::IPP::Metadata::Config; +#use PS::IPP::Metadata::Stats; +#use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw( :standard ); + +my ($job_id, $redirect_output, $outputBase, $rownum, $jobType); +my ($verbose, $dbname, $dbserver, $no_update); + +GetOptions( + 'job_id=s' => \$job_id, + 'job_type=s' => \$jobType, + 'rownum=s' => \$rownum, + 'output_base=s' => \$outputBase, + 'redirect-output' => \$redirect_output, + 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, + 'verbose' => \$verbose, + 'no-update' => \$no_update, +); + my $host = hostname(); - - -GetOptions( - 'job_id=s' => \$job_id, - 'output_base=s' => \$output_base, - 'redirect-output' => \$redirect_output, - 'dbname=s' => \$dbname, - 'verbose' => \$verbose, -); - - if ($verbose) { print "\n\n"; @@ -39,28 +45,20 @@ die "job_id is required" if !$job_id; -die "output_base is required" if !$output_base; - -use IPC::Cmd 0.36 qw( can_run run ); - -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use PS::IPP::Metadata::List qw( parse_md_list ); - -use PS::IPP::Config qw($PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - metadataLookupStr - metadataLookupBool - caturi - ); + +my_die( "job_type is required", $job_id, $PS_EXIT_PROG_ERROR) if !$jobType; +my_die("rownum is required", $job_id, $PS_EXIT_PROG_ERROR) if !$rownum; +my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase; + + my $ipprc = PS::IPP::Config->new(); # IPP Configuration - if ($redirect_output) { - my $logDest = "$output_base.log"; - $ipprc->redirect_output($logDest); + my $logDest = "$outputBase.log"; + $ipprc->redirect_output($logDest) + or my_die ("unable to redirect output to $logDest", $job_id, $PS_EXIT_UNKNOWN_ERROR); +} + + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); } @@ -72,85 +70,54 @@ if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); + my_die("Can't find required tools", $job_id, $PS_EXIT_CONFIG_ERROR); } my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -my $psjob; -#Look up the uri for the given job -{ - my $command = "$pstamptool -pendingjob -job_id $job_id"; +my $jobStatus; +if ($jobType eq "stamp") { + my $argslist = "$outputBase.args"; + open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR); + my $argString = ; + close ARGSLIST; + chomp $argString; + + # XXX: should we do any other sanity checking? + my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString; + + my $command = "$ppstamp $outputBase $argString"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - die("Unable to perform pstamptool -pendingjob: $error_code"); - } - - if (@$stdout_buf == 0) { - print STDERR "pending pstamp job id $job_id not found\n"; - exit 0; - } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - die("Unable to parse metdata config doc"); - - my $jobs = parse_md_list($metadata); - my $num = @$jobs; - die "unexpected number of jobs $num found for job: $job_id" if $num > 1; - - $psjob = $jobs->[0]; -} - -if (!$psjob) { - print STDERR "postage stamp job $job_id not found\n"; - exit 1; -} - -my $rownum = $psjob->{rownum}; -my $uri = $psjob->{uri}; -my $outputBase = $psjob->{outputBase}; -my $argString = $psjob->{args}; -my $jobType = $psjob->{jobType}; - -my $jobStatus; -if ($jobType eq "stamp") { - my $command = "$ppstamp -file $uri $outputBase $argString"; - $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); if ($success) { - # XXX shouldn't need to do this, review schema` my $dir = dirname($outputBase); my $reglist = "$dir/reglist$job_id"; - open F, ">$reglist" or die "can't open $reglist for output"; - - # figure out what output images we should expect - - # Note: we are assuming the contents of the filerules here. - my @extensions = ( "fits", "mk.fits", "wt.fits"); - - # we always require an image + open F, ">$reglist" or my_die( "can't open $reglist for output", $job_id, $PS_EXIT_UNKNOWN_ERROR); + + # Figure out what output images were produced + + # Note: we are assuming the contents of the PSTAMP filerules here. + my %extensions = ( $PSTAMP_SELECT_IMAGE => "fits", + $PSTAMP_SELECT_MASK => "mk.fits", + $PSTAMP_SELECT_WEIGHT => "wt.fits"); + + # we always create a stamp of the image my $output_mask = $PSTAMP_SELECT_IMAGE; - # we search the argString for -mask and -weight. + # we search the argString for -mask and -variance. # searching the arg string allows us to avoid adding a column in pstampJob - # XXX yeah but does this make us susceptible to user input fouling us up? - # For example, what if somebody sets the output base to file-mask-weight - # TODO: clean this up - $output_mask |= $PSTAMP_SELECT_MASK if ($argString =~ /-mask/); - $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-weight/); - - # XXX we're getting a bit intimate with the bit field definitions here. - # do better use a hash - - my $m = 1; - foreach my $extension (@extensions) { - my $do_this_one = $m & $output_mask; - $m = $m << 1; + $output_mask |= $PSTAMP_SELECT_MASK if ($argString =~ /-mask/); + $output_mask |= $PSTAMP_SELECT_WEIGHT if ($argString =~ /-variance/); + + foreach my $key (keys (%extensions)) { + my $do_this_one = $key & $output_mask; + next if (! $do_this_one); + + my $extension = $extensions{$key}; my $basename = basename($outputBase); @@ -167,9 +134,13 @@ } else { $jobStatus = $error_code >> 8; - print STDERR "ppstamp failed with error code: $jobStatus\n"; + my_die( "ppstamp failed with error code: $jobStatus", $job_id, $jobStatus); } } elsif ($jobType eq "get_image") { + my_die( "get_image jobs not working right now", $job_id, $PS_EXIT_CONFIG_ERROR); + + my $uri = ""; my $command = "$pstamp_get_image_job --job_id $job_id --uri $uri --out_dir $outputBase --rownum $rownum"; $command .= " --dbname $dbname" if $dbname; + $command .= " --dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -179,24 +150,29 @@ } else { $jobStatus = $error_code >> 8; - print STDERR "ppstamp failed with error code: $jobStatus\n"; + my_die( "pstamp_get_image_job failed with error code: $jobStatus", $job_id, $jobStatus); } } elsif ($jobType eq "detect_query") { - die("multiple detect_query jobs not supported yet"); + my_die("detect_query jobs not supported yet", $job_id,$PS_EXIT_CONFIG_ERROR); } else { - die("unknown jobType $jobType found"); -} - -# stop the job and set the result value + my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR); +} + +# mark the job stopped in the database { - my $command = "$pstamptool -updatejob -job_id $job_id -state stop -fault $jobStatus"; + my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - die("Unable to perform $command: $error_code"); - } -} - -exit $jobStatus; + $command .= " -dbserver $dbserver" if $dbserver; + if (!$no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + die("Unable to perform $command: $error_code"); + } + } else { + print STDERR "skipping command: $command\n" + } +} + +exit 0; # create a string to be passed as input to dsreg when registering this file in a fileset @@ -209,5 +185,5 @@ if (-e $path) { my @finfo = stat($path); - die "failed to stat $path" unless (@finfo); # XXX clean up + my_die("failed to stat $path", $job_id, $PS_EXIT_UNKNOWN_ERROR) unless (@finfo); my $bytes = $finfo[7]; my $md5sum = file_md5_hex($path); @@ -215,5 +191,26 @@ return "$filename|$bytes|$md5sum|$filetype|"; } else { - die "$filename not found at $path"; - } -} + my_die("$filename not found at $path", $job_id, $PS_EXIT_UNKNOWN_ERROR); + } +} + +sub my_die +{ + my $msg = shift; # Warning message on die + my $job_id = shift; # job identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + if (defined $job_id and not $no_update) { + my $command = "$pstamptool -updatejob"; + $command .= " -job_id $job_id"; + $command .= " -fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -dbserver $dbserver" if defined $dbserver; + system($command); + } + exit $exit_code; +} + Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_listjobs.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_listjobs.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_listjobs.pl (revision 25022) @@ -12,8 +12,10 @@ my $verbose; my $dbname; +my $dserver; GetOptions( 'verbose' => \$verbose, 'dbname=s' => \$dbname, + 'dbserver=s'=> \$dbserver, ); @@ -68,4 +70,5 @@ my $command = "$pstamptool -listjob -req_id $request_id"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_parser_run.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_parser_run.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_parser_run.pl (revision 25022) @@ -13,4 +13,5 @@ use Getopt::Long qw( GetOptions ); use File::Basename qw( basename dirname); +use POSIX qw( strftime ); my $req_id; @@ -20,4 +21,5 @@ my $verbose; my $dbname; +my $dbserver; GetOptions( @@ -28,4 +30,5 @@ 'verbose' => \$verbose, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, ); @@ -68,8 +71,17 @@ exit ($PS_EXIT_CONFIG_ERROR) unless defined $pstamp_workdir; # lookup failure outputs a message -# workdir is where we download request files to and place any error output from the parser +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} + +# workdir is where all of the files generated for this request are placed # NOTE: this location needs to be kept in sync with the web interface ( request.php ) -my $workdir = "$pstamp_workdir/$req_id"; - +my $datestr = strftime "%Y%m%d", gmtime; +my $datedir = "$pstamp_workdir/$datestr"; +if (! -e $datedir ) { + mkdir $datedir or die "failed to create working directory $datedir for request id $req_id"; +} + +my $workdir = "$datedir/$req_id"; if (! -e $workdir ) { mkdir $workdir or die "failed to create working directory $workdir for request id $req_id"; @@ -167,4 +179,5 @@ $command .= " -fault $PS_EXIT_DATA_ERROR"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -177,4 +190,5 @@ $parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $workdir --file $uri"; $parse_cmd .= " --dbname $dbname" if $dbname; +$parse_cmd .= " --dbserver $dbserver" if $dbserver; $parse_cmd .= " --verbose" if $verbose; @@ -218,4 +232,5 @@ $command .= " -fault $fault" if $fault; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_queue_requests.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_queue_requests.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_queue_requests.pl (revision 25022) @@ -17,4 +17,5 @@ my $verbose; my $dbname; +my $dbserver; my $limit; @@ -22,4 +23,5 @@ 'verbose' => \$verbose, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'limit=i' => \$limit, ); @@ -59,5 +61,10 @@ } +my $ipprc = PS::IPP::Config->new(); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +if (!$dbserver) { + $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); +} my @dataStores; @@ -66,4 +73,5 @@ my $command = "$pstamptool -datastore"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -177,4 +185,5 @@ my $command = "$pstamptool -addreq -uri $req_uri -ds_id $ds_id"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -192,4 +201,5 @@ my $command = "$pstamptool -ds_id $ds_id -moddatastore -last_fileset $lastFileset"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_results_file.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_results_file.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_results_file.pl (revision 25022) @@ -88,5 +88,6 @@ { name => 'IMG_TYPE', type => '16A', writetype => TSTRING }, { name => 'ID', type => '16A', writetype => TSTRING }, - { name => 'CLASS_ID', type => '16A', writetype => TSTRING }, + { name => 'TESS_ID', type => '64A', writetype => TSTRING }, + { name => 'COMPONENT', type => '64A', writetype => TSTRING }, # output parameters Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_revert_request.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_revert_request.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_revert_request.pl (revision 25022) @@ -31,9 +31,10 @@ -my ( $req_id, $dbname, $verbose, $save_temps ); +my ( $req_id, $dbname, $dbserver, $verbose, $save_temps ); GetOptions( 'req_id=s' => \$req_id, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -65,4 +66,5 @@ my $command = "$pstamptool -listreq -req_id $req_id"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -97,5 +99,6 @@ if ($product and $fileset) { my $command = "$dsreg --del $fileset --product $product --rm --force"; - $command .= " --dbname $dbname" if $dbname; +# don't add dbname let dsreg get it from the DS_DBNAME in site.config +# $command .= " --dbname $dbname" if $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -109,4 +112,5 @@ my $command = "$pstamptool -revertreq -req_id $req_id"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/scripts/pstamp_webrequest.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstamp_webrequest.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstamp_webrequest.pl (revision 25022) @@ -24,4 +24,5 @@ my $verbose = 0; my $dbname; +my $dbserver; my $project; my $job_type; @@ -30,4 +31,5 @@ 'job_type=s' => \$job_type, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'project=s' => \$project, 'verbose' => \$verbose, @@ -101,4 +103,5 @@ my $command = "$pstampparse --mode list_uri --file $request_file"; $command .= " --dbname $dbname" if $dbname; + $command .= " --dbserver $dbserver" if $dbserver; $command .= " --verbose" if $verbose; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -123,4 +126,5 @@ my $command = "$pstamptool -addreq -uri $request_file -ds_id 0"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/pstampparse.pl (revision 25022) @@ -13,7 +13,9 @@ use PS::IPP::PStamp::RequestFile qw( :standard ); use PS::IPP::PStamp::Job qw( :standard ); +use File::Temp qw(tempfile); my $verbose; my $dbname; +my $dbserver; my $req_id; my $request_file_name; @@ -21,4 +23,6 @@ my $out_dir; my $product; +my $save_temps; +my $no_update; GetOptions( @@ -29,5 +33,8 @@ 'mode=s' => \$mode, 'dbname=s' => \$dbname, + 'dbserver=s'=> \$dbserver, 'verbose' => \$verbose, + 'save-temps'=> \$save_temps, + 'no-update' => \$no_update, ); @@ -54,4 +61,5 @@ my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); my $pstampdump = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1); +my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1); my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1); @@ -60,4 +68,6 @@ exit ($PS_EXIT_CONFIG_ERROR); } + +$no_update = 1 if $mode eq "list_job"; my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -85,8 +95,9 @@ die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1"); -if ($req_id) { +if ($req_id and !$no_update) { my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name"; $command .= " -outProduct $product"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -114,183 +125,89 @@ } +my @rowList; my $num_jobs = 0; +my $imageList; +my $stage; +my $need_magic; foreach my $row (@$rows) { + # XXX: TODO: sanity check all parameters + + # XXX: TODO: We shouldn't really just die in this loop. + # If we encounter an error for a particular row + # add a job with the proper fault code. If there is a db or config error we should probably just + # trash the request. + my $rownum = $row->{ROWNUM}; my $job_type = $row->{JOB_TYPE}; + + # parameters that select the images of interest my $project = $row->{PROJECT}; my $req_type = $row->{REQ_TYPE}; - my $img_type = $row->{IMG_TYPE}; + $stage = $row->{IMG_TYPE}; my $id = $row->{ID}; - my $class_id = $row->{CLASS_ID}; + my $component = $row->{COMPONENT}; + my $filter = $row->{REQFILT}; my $mjd_min = $row->{MJD_MIN}; my $mjd_max = $row->{MJD_MAX}; - my $x = $row->{CENTER_X}; - my $y = $row->{CENTER_Y}; - my $w = $row->{WIDTH}; - my $h = $row->{HEIGHT}; - my $coord_mask = $row->{COORD_MASK}; + my $option_mask= $row->{OPTION_MASK}; - $class_id = "" if ($class_id eq "null" or $class_id eq "all"); + die "valid region of interest is required to make postage stamps" + if (($job_type eq "stamp") and ! validROI($row)); + + my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS); + $component = "" if (defined($component) and ($component eq "null" or $component eq "all")); - # XXX: TODO: sanity check all parameters - - # XXX: TODO: We shouldn't just die in this loop. - # If we encounter an error for a particular row - # add a job with the proper fault code. If there is a db or config error we should probably just - # trash the request. die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri"); - my $proj_hash = resolve_project($ipprc, $project, $dbname); + + # note: resolve_project avoids running pstamptool every time by remembering the + # last project resolved + my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver); die "project $project not found\n" unless $proj_hash; - my $image_db = $proj_hash->{dbname}; - my $camera = $proj_hash->{camera}; - my $need_magic = $proj_hash->{need_magic}; - - my $roi_string; - # XXX we're depending on other code to insure valid values for roi components - # this is checked in and ppstamp but I should check here so that we don't get that far, - # but not today - if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { - $roi_string = "-pixcenter $x $y"; + my $image_db = $proj_hash->{dbname}; + my $camera = $proj_hash->{camera}; + $need_magic = $proj_hash->{need_magic}; + + # collect rows with the same images of interest in a list so that they + # can be processed optimially + if (@rowList) { + my $firstRow = $rowList[0]; + # XXX: the collecting might work with !$skycenter but I need to think about it + if ($skycenter and same_images_of_interest($row, $firstRow)) { + push @rowList, $row; + # On to the next row + next; } else { - $roi_string = "-skycenter $x $y"; - } - if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { - $roi_string .= " -pixrange $w $h"; - } else { - $roi_string .= " -arcrange $w $h"; - } - } - - die "region of interest is required to make postage stamps" - if (($job_type eq "stamp") && !defined($roi_string)); - - # find the uris for this request + # queue the jobs for this set of rows + $num_jobs += queueJobs($mode, \@rowList, $imageList); + @rowList = (); + } + } if ($req_type eq "bycoord") { - die "region of interest is required for request type bycoord" if !defined($roi_string) ; - die "center must be specified in sky coordintes for bycoord" - if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS); - } - - # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification - my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id, - $x, $y, $mjd_min, $mjd_max, $filter, $verbose); - - if (!$images) { + die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter); + die "lookup bycoord is not yet implemented"; + } else { + + # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this + # request specification. An array reference is returned + my ($x, $y); + $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter, + $x, $y, $mjd_min, $mjd_max, $filter, $verbose); + } + + if (!$imageList or !@$imageList) { print STDERR "no matching images found for row $rownum\n"; next; } - if ($mode eq "list_uri") { - foreach my $image (@$images) { - print "$image->{image}\n"; - } - } elsif ($job_type eq "get_image") { - # XXX TODO: Get rid of this block and use the same code as the stamp jobs - # XXX This doesn't work to get the mask and weight images - my $listfile = "$out_dir/filelist"; - - - # map our img_type to the Data Store file types. - my %filelist_img_types = ( "raw" => "chip", - "chip" => "chipproc", - "warp" => "warp", - "stack"=>"stack", - "diff" => "diff"); - - my $filelist_img_type = $filelist_img_types{$img_type}; - - open LISTFILE, ">$listfile" - or die "failed to open file list: $listfile while parsing get_image request $req_id"; - - foreach my $image (@$images) { - my $class_id = $image->{class_id} ? $image->{class_id} : ""; - print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n"; - } - close LISTFILE; - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image" - . " -uri $listfile -outputBase $out_dir -rownum $rownum"; - $command .= " -dbname $dbname" if $dbname; - if ($mode eq "list_job") { - print "$command\n"; - } else { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - die "failed to queue job for request $req_id: rownum: $rownum"; - } - } - } else { - # sequence number for the the job for a request spec. Not to be confused with job_id - my $job_num = 0; - - # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images - # use -list instead of -file - foreach my $image (@$images) { - my $uri = $image->{image}; - my $exp_id = $image->{exp_id}; - - my $args = $roi_string ? $roi_string : ""; - $args .= " -class_id $class_id" if $class_id; - - $job_num++; - - my $output_base = "$out_dir/${rownum}_${job_num}"; - - # add astrometry file for raw and chip images if one is available - if (($img_type eq "chip") || ($img_type eq "raw")) { - $args .= " -astrom $image->{astrom}" if $image->{astrom}; - } - - if (($option_mask & $PSTAMP_SELECT_MASK) && $image->{mask} ) { - $args .= " -mask $image->{mask}"; - } - if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { - $args .= " -weight $image->{weight}"; - } - - # XXX sounds like magic will be handled outside of the postage stamp server - #if ($need_magic) { - # die "no magic mask available" if ! $image->{magic_mask}; - # $args .= " -magic_mask $image->{magic_mask}" - #} - - # XXX: TODO: here is where we need to check whether or not the source inputs still exist - # and if not, queue a regeneration job and set the job state appropriately. - - my $state = "run"; - - $num_jobs++; - my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type" - . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum" - . " -state $state"; - $command .= " -exp_id $exp_id" if $exp_id; - $command .= " -dbname $dbname" if $dbname; - - if ($mode eq "list_job") { - # this is a debugging mode, just print the pstamptool that would have run - # this is sort of like the mode -noupdate that some other tools support - print "$command\n"; - } else { - # mode eq "queue_job" - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR @$stderr_buf; - # XXX TODO: now what? Should we mark the error state for the request? - # should we keep going for other uris? If so how do we report that some - # of the work that the request wanted isn't going to get done - die "failed to queue job for request $req_id"; - } - } - } - } + $row->{need_magic} = $need_magic; + push @rowList, $row; +} + +if (@rowList) { + $num_jobs += queueJobs($mode, \@rowList, $imageList); } @@ -306,2 +223,293 @@ exit 0; } + + +sub queueJobsForRow +{ + my $row = shift; + my $stage = shift; + my $imageList = shift; + my $have_skycells = shift; + my $need_magic = shift; + + # loop over images + my $job_num = 0; + foreach my $image (@$imageList) { + my $component; + if ($have_skycells) { + $component = $image->{skycell_id}; + } else { + $component = $image->{class_id}; + } + + # skip this component if it is not in the list for this row + next if !$row->{components}->{$component}; + + my $rownum = $row->{ROWNUM}; + + my $roi_string; + + # note values were insured to be numbers in validROI() + my $x = $row->{CENTER_X}; + my $y = $row->{CENTER_Y}; + my $w = $row->{WIDTH}; + my $h = $row->{HEIGHT}; + my $coord_mask = $row->{COORD_MASK}; + if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) { + if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) { + $roi_string = "-pixcenter $x $y"; + } else { + $roi_string = "-skycenter $x $y"; + } + if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) { + $roi_string .= " -pixrange $w $h"; + } else { + $roi_string .= " -arcrange $w $h"; + } + } + + my $imagefile = $image->{image}; + if (($stage ne "stack") and ($need_magic and !$image->{magicked})) { + # XXX: should we add a faulted job so the client can know what happened? + print STDERR "skippping non-magicked image $imagefile\n" if $verbose; + next; + } + my $exp_id = $image->{exp_id}; + + my $args = $roi_string ? $roi_string : ""; + if ($stage eq "raw" or $stage eq "chip") { + $args .= " -class_id $component" if $component; + } + + $job_num++; + + # add astrometry file for raw and chip images if one is available + if (($stage eq "chip") || ($stage eq "raw")) { + $args .= " -astrom $image->{astrom}" if $image->{astrom}; + } + + $args .= " -file $imagefile"; + + if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) && $image->{mask} ) { + $args .= " -mask $image->{mask}"; + } + if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) { + $args .= " -variance $image->{weight}"; + } + + my $output_base = "$out_dir/${rownum}_${job_num}"; + my $argslist = "${output_base}.args"; + + # copy the argument list to a file + open ARGSLIST, ">$argslist" or die "failed to open $argslist"; + print ARGSLIST "$args\n"; + close ARGSLIST or die "failed to close $argslist"; + + # XXX: TODO: here is where we need to check whether or not the source inputs still exist + # and if not, queue an update job and set the job state appropriately. + + my $newState = "run"; + + $num_jobs++; + my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}" + . " -outputBase $output_base -rownum $rownum -state $newState"; + $command .= " -exp_id $exp_id" if $exp_id; + $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; + + if ($mode eq "list_job") { + # this is a debugging mode, just print the pstamptool that would have run + # this is sort of like the mode -noupdate that some other tools support + print "$command\n"; + } elsif (!$no_update) { + # mode eq "queue_job" + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + # XXX TODO: now what? Should we mark the error state for the request? + # should we keep going for other uris? If so how do we report that some + # of the work that the request wanted isn't going to get done + die "failed to queue job for request $req_id"; + } + } else { + print "skipping command: $command\n"; + } + } + return $num_jobs; +} + +sub queueJobs +{ + my $mode = shift; + my $rowList = shift; + my $imageList = shift; + + my $firstRow = $rowList[0]; + my $stage = $firstRow->{IMG_TYPE}; + my $job_type = $firstRow->{JOB_TYPE}; + my $need_magic = $firstRow->{need_magic}; + + my $num_jobs = 0; + + if ($mode eq "list_uri") { + foreach my $image (@$imageList) { + print "$image->{image}\n"; + } + } elsif ($job_type eq "get_image") { + die "get_image jobs not implemented yet"; + } else { + my $thisRun; + + my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps); + foreach my $row (@$rowList) { + print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n"; + $row->{components} = {}; + } + close $pointsList; + + my $have_skycells; + if (($stage eq "raw") or ($stage eq "chip")) { + $have_skycells = 0; + } else { + $have_skycells = 1; + } + + my $tess_dir_abs; + my $last_tess_id = ""; + while ($thisRun = getOneRun($stage, $imageList)) { + { + my $command = "$dvoImagesAtCoords $pointsListName"; + if ($have_skycells) { + my $tess_id = $thisRun->[0]->{tess_id}; + if ($tess_id ne $last_tess_id) { + $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id ); + $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs ); + $last_tess_id = $tess_id; + } + $command .= " -D CATDIR $tess_dir_abs"; + } else { + my $astrom = $thisRun->[0]->{astrom}; + die "no astrometry file found" if !$astrom; + my $astrom_resolved = $ipprc->file_resolve($astrom); + $command .= " -astrom $astrom_resolved"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR @$stderr_buf; + my $rc = $error_code >> 8; + die "dvoImagesAtCoords failed: $rc"; + } + # now we have a list of row numbers and components + # eventually we might want to multiple stamp requests for the same image + # into the same ppstamp job but not yet. For now we will queue a new + my @lines = split "\n", join "", @$stdout_buf; + foreach my $line (@lines) { + # parse the line, ignoring the ra and dec + my ($rownum, undef, undef, $component) = split " ", $line; + + # I guess since we need this function we should be useing a hash for rowList + my $row = findRow($rownum, $rowList); + $row->{components}->{$component} = 1; + } + } + + foreach my $row (@$rowList) { + $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic); + } + } + } + return $num_jobs; +} + +sub get_run_id +{ + my $stage = shift; + my $image = shift; + + if ($stage eq "raw") { + return $image->{exp_id}; + } elsif ($stage eq "chip") { + return $image->{chip_id}; + } elsif ($stage eq "warp") { + return $image->{warp_id}; + } elsif ($stage eq "stack") { + return $image->{stack_id}; + } elsif ($stage eq "diff") { + return $image->{diff_id}; + } else { + die "unenexpected stage: $stage found"; + } +} + +# extract components from the imageList that have the same run id and return the list +sub getOneRun { + my $stage = shift; + my $imageList = shift; + + # return if array is empty + return undef if ! @$imageList; + + my $last_run_id = 0; + my @runList; + while ($imageList->[0]) { + my $run_id = get_run_id($stage, $imageList->[0]); + + last if ($last_run_id and ($run_id ne $last_run_id)); + + my $image = shift @$imageList; + $image->{stage} = $stage; + push @runList, $image; + $last_run_id = $run_id; + } + return \@runList; +} + +sub same_images_of_interest { + my $r1 = shift; + my $r2 = shift; + + return 0 if ($r1->{PROJECT} ne $r2->{PROJECT}); + return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE}); + return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE}); + return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE}); + return 0 if ($r1->{ID} ne $r2->{ID}); + + if (defined($r1->{COMPONENT})) { + return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT}); + } elsif (defined($r2->{COMPONENT})) { + return 0; + } + + return 1; +} + +sub validNumber +{ + my $val = shift; + + return $val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/; +} + +sub validROI +{ + my $row = shift; + return 0 if !validNumber($row->{CENTER_X}); + return 0 if !validNumber($row->{CENTER_Y}); + return 0 if !validNumber($row->{WIDTH}); + return 0 if !validNumber($row->{HEIGHT}); + + return 1; +} + +sub findRow +{ + my $rownum = shift; + my $rowList = shift; + + foreach my $row (@$rowList) { + return $row if $row->{ROWNUM} eq $rownum; + } + return undef; +} Index: branches/eam_branches/20090715/pstamp/scripts/request_finish.pl =================================================================== --- branches/eam_branches/20090715/pstamp/scripts/request_finish.pl (revision 25020) +++ branches/eam_branches/20090715/pstamp/scripts/request_finish.pl (revision 25022) @@ -21,5 +21,5 @@ use PS::IPP::Config qw( :standard ); -my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output ); +my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output ); GetOptions( @@ -31,4 +31,5 @@ 'product=s' => \$product, 'dbname=s' => \$dbname, + 'dbserver=s' => \$dbserver, 'verbose' => \$verbose, 'save-temps' => \$save_temps, @@ -80,4 +81,5 @@ my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --out_dir $out_dir"; $command .= " --dbname $dbname" if $dbname; + $command .= " --dbserver $dbserver" if $dbserver; $command .= " --verbose" if $verbose; $command .= " --save-temps" if $save_temps; @@ -93,4 +95,5 @@ my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $PS_EXIT_DATA_ERROR"; $command .= " -dbname $dbname" if $dbname; + $command .= " -dbserver $dbserver" if $dbserver; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/eam_branches/20090715/pstamp/src/ppstampArguments.c =================================================================== --- branches/eam_branches/20090715/pstamp/src/ppstampArguments.c (revision 25020) +++ branches/eam_branches/20090715/pstamp/src/ppstampArguments.c (revision 25022) @@ -22,7 +22,7 @@ fprintf(stderr, "Optional arguments:\n"); fprintf(stderr, " [-class_id class_id] selects class_id (only used with -pixcenter)\n"); - fprintf(stderr, " [-astrom astrom.cmp] : provide an alternative astrometry calibration\n"); - fprintf(stderr, " [-mask mk_image] : mask image\n"); - fprintf(stderr, " [-weight wt_image] : weight image\n"); + fprintf(stderr, " [-astrom astrom.cmp] : provide an alternative astrometry calibration\n"); + fprintf(stderr, " [-mask mk_image] : mask image\n"); + fprintf(stderr, " [-variance var_image] : variance image\n"); fprintf(stderr, "\n"); @@ -81,5 +81,5 @@ pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", "-astromlist"); pmConfigFileSetsMD(config->arguments, &argc, argv, "MASK", "-mask", "-masklist"); - pmConfigFileSetsMD(config->arguments, &argc, argv, "WEIGHT", "-weight", "-weightlist"); + pmConfigFileSetsMD(config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist"); // the input file is a required argument; if not found, we will exit Index: branches/eam_branches/20090715/pstamp/src/ppstampParseCamera.c =================================================================== --- branches/eam_branches/20090715/pstamp/src/ppstampParseCamera.c (revision 25020) +++ branches/eam_branches/20090715/pstamp/src/ppstampParseCamera.c (revision 25022) @@ -18,5 +18,5 @@ } if (doWeight) { - pmFPAfile *outWeight = pmFPAfileDefineSkycell(config, output->fpa, "PPSTAMP.OUTPUT.WEIGHT"); + pmFPAfile *outWeight = pmFPAfileDefineSkycell(config, output->fpa, "PPSTAMP.OUTPUT.VARIANCE"); outWeight->save = true; } @@ -54,7 +54,7 @@ } bool doWeight = false; - pmFPAfile *weight = pmFPAfileBindFromArgs (&status, input, config, "PPSTAMP.INPUT.WEIGHT", "WEIGHT"); + pmFPAfile *weight = pmFPAfileBindFromArgs (&status, input, config, "PPSTAMP.INPUT.VARIANCE", "VARIANCE"); if (!status) { - psError(PS_ERR_IO, false, "Failed to build FPA from PPSTAMP.INPUT.WEIGHT"); + psError(PS_ERR_IO, false, "Failed to build FPA from PPSTAMP.INPUT.VARIANCE"); return false; } Index: branches/eam_branches/20090715/pstamp/test/pstamp_req_create =================================================================== --- branches/eam_branches/20090715/pstamp/test/pstamp_req_create (revision 25020) +++ branches/eam_branches/20090715/pstamp/test/pstamp_req_create (revision 25022) @@ -26,6 +26,6 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --input --output", - -exitval => 3) unless defined $input and defined $output; +pod2usage( -msg => "Required options: --input --output | --req_name", + -exitval => 3) unless defined $input and (defined $output or defined $req_name); # The header kewords @@ -55,5 +55,6 @@ { name => 'IMG_TYPE', type => '16A', writetype => TSTRING }, { name => 'ID', type => '16A', writetype => TSTRING }, - { name => 'CLASS_ID', type => '16A', writetype => TSTRING }, + { name => 'TESS_ID', type => '64A', writetype => TSTRING }, + { name => 'COMPONENT', type => '64A', writetype => TSTRING }, @@ -95,5 +96,11 @@ if ($req_name) { $header->[0]->{value} = $req_name; -} +} else { + $req_name = $header->[0]->{value}; +} + +die "no request name defined" unless defined $req_name; + +$output = $req_name . ".fits" if !$output; my $status = make_fits_table($output, EXTNAME, $numRows, \@colData, $columns, $header); Index: branches/eam_branches/20090715/tools/diff_inputs.pl =================================================================== --- branches/eam_branches/20090715/tools/diff_inputs.pl (revision 25020) +++ branches/eam_branches/20090715/tools/diff_inputs.pl (revision 25022) @@ -28,4 +28,5 @@ my ($diff_id, $skycell_id); # Diff and skycell of interest my ($input); # Input list +my ($muggle); # No magicked inputs? GetOptions( @@ -37,4 +38,5 @@ 'skycell_id=s' => \$skycell_id, # Skycell identifier 'input=s' => \$input, # Input list to generate + 'muggle' => \$muggle, # No magicked inputs? ) or die "Unable to parse arguments.\n"; die "Unknown option: @ARGV\n" if @ARGV; @@ -88,4 +90,11 @@ if (defined $warp) { copy_extensions( $warp, WARP_EXTENSIONS() ); + if (defined $muggle) { + my @warp = split /\//, $warp; + my $last = pop @warp; + push @warp, "SR_$last"; + $warp = join '/', @warp; + copy_extensions( $warp, WARP_EXTENSIONS() ); + } } elsif (defined $stack) { copy_extensions( $stack, STACK_EXTENSIONS() ); Index: branches/eam_branches/20090715/tools/ipp_apply_burntool.pl =================================================================== --- branches/eam_branches/20090715/tools/ipp_apply_burntool.pl (revision 25020) +++ branches/eam_branches/20090715/tools/ipp_apply_burntool.pl (revision 25022) @@ -20,5 +20,5 @@ use Pod::Usage qw( pod2usage ); -my ( $class_id, $dateobs_begin, $dateobs_end, $skip_burned, $dbname, $logfile, $verbose, $save_temps); +my ( $class_id, $dateobs_begin, $dateobs_end, $skip_burned, $rerun_from_first, $dbname, $logfile, $verbose, $save_temps); GetOptions( 'class_id=s' => \$class_id, # chip identifier @@ -28,4 +28,5 @@ 'logfile=s' => \$logfile, 'skip_burned' => \$skip_burned, # Print to stdout + 'rerun_from_first' => \$rerun_from_first, # Print to stdout 'verbose' => \$verbose, # Print to stdout 'save-temps' => \$save_temps, # Save temporary files? @@ -42,4 +43,6 @@ defined $dbname; +if ($skip_burned and $rerun_from_first) { &my_die("-rerun_from_first and -skip_burned are incompatible"); } + my $missing_tools; my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); @@ -58,5 +61,5 @@ $command .= " -dateobs_begin $dateobs_begin"; $command .= " -dateobs_end $dateobs_end"; -# $command .= " -limit 20"; +$command .= " -ordered_by_date"; $command .= " -dbname $dbname" if defined $dbname; @@ -101,4 +104,12 @@ if (! $skip_burned) { foreach my $file (@files) { + + # rerun_from_first treats the first image as already burned (if it is already burned) + # the artifact table from the first file is used for the rest of the sequence. + if ($rerun_from_first) { + $skip_burned = 1; + $rerun_from_first = 0; + next; + } my $exp_id = $file->{exp_id}; @@ -108,4 +119,5 @@ &my_die("failed to update imfile"); } + $file->{user_1} = 0.1; } } @@ -122,5 +134,5 @@ ($rawImfileReal) = $rawImfile =~ m|^neb:/(.*)|; } - # print "rawImfile: $rawImfile -> $rawImfileReal\n"; + print "rawImfile: $rawImfile -> $rawImfileReal\n"; # mangle name, create tmp file (always a UNIX file) @@ -149,5 +161,8 @@ } + print "$rawImfile : $skip_burned, $file->{user_1}\n"; + if (! ($skip_burned and ($file->{user_1} > 0.5))) { + print "running on: $rawImfile\n"; # uncompress the image (do we need to check if it is compressed?) my $status = vsystem ("$funpack -S $rawImfileReal > $tmpImfileReal", $REALRUN); Index: branches/eam_branches/20090715/tools/stack_inputs.pl =================================================================== --- branches/eam_branches/20090715/tools/stack_inputs.pl (revision 25020) +++ branches/eam_branches/20090715/tools/stack_inputs.pl (revision 25022) @@ -24,4 +24,5 @@ my ($stack_id); # Stack of interest my ($input); # Input list +my ($muggle); # No magicked inputs? GetOptions( @@ -32,4 +33,5 @@ 'stack_id=s' => \$stack_id, # Stack identifier 'input=s' => \$input, # Input list to generate + 'muggle' => \$muggle, # No magicked inputs? ) or die "Unable to parse arguments.\n"; die "Unknown option: @ARGV\n" if @ARGV; @@ -70,6 +72,13 @@ print $list "INPUT METADATA\n" if defined $input; foreach my $type ( keys %{EXTENSIONS()} ) { + my $path = $warp_path; + if (defined $muggle and $type ne "PSF") { + my @warp = split /\//, $warp_path; + my $last = pop @warp; + push @warp, "SR_$last"; + $path = join '/', @warp; + } my $ext = ${EXTENSIONS()}{$type}; # Extension - my $file = "$warp_path$ext"; # File + my $file = "$path$ext"; # File my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Resolved file name chomp $source;