Index: /trunk/ippdor/README =================================================================== --- /trunk/ippdor/README (revision 32874) +++ /trunk/ippdor/README (revision 32874) @@ -0,0 +1,10 @@ +Installation: + ./deployment/deploy.py -h + +Documentation: + cd ./doc && make + +Code Quality Metrics (pylint 0.25 required): + ./quality_metrics.sh + + Index: /trunk/ippdor/conf/pylintrc =================================================================== --- /trunk/ippdor/conf/pylintrc (revision 32874) +++ /trunk/ippdor/conf/pylintrc (revision 32874) @@ -0,0 +1,310 @@ +# lint Python modules using external checkers. +# +# This is the main checker controlling the other ones and the reports +# generation. It is itself both a raw checker and an astng checker in order +# to: +# * handle message activation / deactivation at the module level +# * handle some basic but necessary stats'data (number of classes, methods...) +# +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# Set the cache size for astng objects. +cache-size=500 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable only checker(s) with the given id(s). This option conflicts with the +# disable-checker option +#enable-checker= + +# Enable all checker(s) except those with the given id(s). This option +# conflicts with the enable-checker option +#disable-checker= + +# Enable all messages in the listed categories (IRCWEF). +#enable-msg-cat= + +# Disable all messages in the listed categories (IRCWEF). +disable-msg-cat=I + +# Enable the message(s) with the given id(s). +#enable-msg= + +# Disable the message(s) with the given id(s). +disable-msg=W0704 + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Include message's id in output +include-ids=no + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (R0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (R0004). +comment=no + +# Enable the report(s) with the given id(s). +#enable-report= + +# Disable the report(s) with the given id(s). +#disable-report= + + +# checks for : +# * doc strings +# * modules / classes / functions / methods / arguments / variables name +# * number of arguments, local variables, branches, returns and statements in +# functions, methods +# * required module attributes +# * dangerous default values as arguments +# * redefinition of function / method / class +# * uses of the global statement +# +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + + +# try to find bugs in the code using type inference +# +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. +generated-members=REQUEST,acl_users,aq_parent + + +# checks for +# * unused variables / imports +# * undefined variables +# * redefinition of variable from builtins or from an outer scope +# * use of variable before assignment +# +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +# checks for : +# * methods without self as first argument +# * overridden methods signature +# * access only to existent members via self +# * attributes not defined in the __init__ method +# * supported interfaces implementation +# * unreachable code +# +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + + +# checks for +# * external modules dependencies +# * relative / wildcard imports +# * cyclic imports +# * uses of deprecated modules +# +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report R0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report R0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report R0402 must +# not be disabled) +int-import-graph= + + +# checks for sign of poor/misdesign: +# * number of methods, attributes, local variables... +# * size, complexity of functions, methods +# +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Maximum number of locals for function / method body +max-locals=25 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=15 + +# Maximum number of statements in function / method body +max-statements=150 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=30 + + +# checks for: +# * warning notes in the code like FIXME, XXX +# * PEP 263: source code with non ascii character but no encoding declaration +# +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +# checks for similarities and duplicated code. This computation may be +# memory / CPU intensive, so you should disable it if you experiments some +# problems. +# +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=10 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +# checks for : +# * unauthorized constructions +# * strict indentation +# * line length +# * use of <> instead of != +# +[FORMAT] + +# Maximum number of characters on a single line. +#max-line-length=80 +max-line-length=120 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' Index: /trunk/ippdor/deployment/Deployment.py =================================================================== --- /trunk/ippdor/deployment/Deployment.py (revision 32874) +++ /trunk/ippdor/deployment/Deployment.py (revision 32874) @@ -0,0 +1,200 @@ +#!/usr/bin/env python + +"""Utility class for Condor IPP software deployent +""" + +import os +import subprocess +import tempfile +import shutil +import logging + +class Deployment: + """Utility class for Condor IPP software deployent + """ + logger = logging.getLogger('ippdor') + + CLASSES_FILENAMES = [ '__init__.py', + 'ChipManager.py', + 'CamManager.py', + 'FakeManager.py', + 'WarpManager.py', + 'StackManager.py', + 'Gpc1Manager.py', + 'helpers.py', + 'constants/__init__.py', + 'constants/Globals.py', + 'constants/Chip.py', + 'constants/Camera.py', + 'constants/Fake.py', + 'constants/Warp.py', + 'constants/Stack.py', + 'exceptions/__init__.py', + 'exceptions/IppException.py', + 'exceptions/UninstantiableIppException.py', + ] + EXECUTABLES_FILENAMES = [ 'generate_chip_jobs.py', + 'generate_cam_jobs.py', + 'generate_fake_jobs.py', + 'generate_warp_jobs.py', + 'generate_stack_jobs.py', + 'generate_exposure_dag.py', + 'ippdor_chip_to_warp.py', + # 'ippdor_chip_to_stack.py', + 'undo_dag_exposure.py', + 'send_email.py', + 'encapsulate_chip.sh', + 'encapsulate_camera.sh', + 'encapsulate_fake.sh', + 'encapsulate_warp.sh', + 'encapsulate_stack.sh', + ] + + def __init__(self, + svn_source_url, + target_directory, + target_binary_directory, + logger = None): + """ + Constructor: + + svn_source_url: SVN Source URL + target_directory: Target Python Directory + target_binary_directory: Target Executables Directory + logger: optional argument (if None: logging to logging.getLogger('ippdor')) + """ + self.svn_source_url = svn_source_url + self.target_directory = target_directory + self.target_binary_directory = target_binary_directory + self.temporary_directory = None # Set when necessary + if logger is not None: + Deployment.logger = logger + + def copy_to_targetdir(self, path_to_file, filename): + """ + Copy a file from the SVN temporary directory to the target + directory. + """ + Deployment.logger.info('...... Installing %s' % filename) + shutil.copyfile('%s/%s/%s' % (self.temporary_directory, + path_to_file, + filename), + '%s/%s' % (self.target_directory, filename)) + + def copy_to_bindir(self, path_to_file, filename): + """ + Copy a file from the SVN temporary directory to the target + binary directory. + """ + Deployment.logger.info('...... Installing %s' % filename) + shutil.copyfile('%s/%s/%s' % (self.temporary_directory, + path_to_file, + filename), + '%s/%s' % (self.target_binary_directory, filename)) + + def check(self): + """ + Check different elements + """ + Deployment.logger.info('Checking Condor-IPP Softwares Configuration') + # self.target_directory + Deployment.logger.info('... Checking write permission to %s' % self.target_directory) + extra = '' + if not os.path.exists(self.target_directory): + os.makedirs(self.target_directory) + extra = ' (directory was created)' + if not os.access(self.target_directory, os.W_OK): + raise IOError('Directory [%s] is not writable' % self.target_directory) + Deployment.logger.info('...... OK%s' % extra) + # self.target_binary_directory + Deployment.logger.info('... Checking write permission to %s' % self.target_binary_directory) + extra = '' + if not os.path.exists(self.target_binary_directory): + os.makedirs(self.target_binary_directory) + extra = ' (directory was created)' + if not os.access(self.target_binary_directory, os.W_OK): + raise IOError('Directory [%s] is not writable' % self.target_binary_directory) + Deployment.logger.info('...... OK%s' % extra) + # Check deployment software version + # Anything else? + Deployment.logger.info('... Check complete') + + def svn_export(self): + """ + Export the software under configuration into the SVN temporary directory + """ + Deployment.logger.info('Updating Condor-IPP Softwares SVN temporary directory') + Deployment.logger.info('... Creating temporary directory:') + self.temporary_directory = tempfile.mkdtemp() + Deployment.logger.info('...... OK [%s]' % self.temporary_directory) + Deployment.logger.info('... Exporting Condor source code to temporary directory') + process = subprocess.Popen( ('svn export --force %s %s' % (self.svn_source_url, + self.temporary_directory)).split(' '), + stdout = open('/dev/null', 'w'), + stderr = open('/dev/null', 'w') ) + status = process.wait() + if status != 0: + Deployment.logger.error('Error: svn export failed') + raise Exception('svn export failed') + Deployment.logger.info('...... OK') + Deployment.logger.info('... SVN Update complete') + + def generate(self): + """ + Generate any file depending on the installation target + """ + Deployment.logger.info('Generating Condor-IPP Softwares') + Deployment.logger.debug('--- Nothing to generate yet from %s' % self.svn_source_url) + Deployment.logger.info('... Generation complete') + + def deploy(self): + """ + Deploy files on the target + """ + Deployment.logger.info('Deploying Condor-IPP Softwares') + Deployment.logger.info('... Installing Python classes to [%s]' % self.target_directory) + for filename in Deployment.CLASSES_FILENAMES: + path_elements = filename.split('/') + if len(path_elements) > 1: + try: + os.makedirs('%s/%s' % (self.target_directory, + path_elements[0])) + except OSError, e: + Deployment.logger.warn(e) + Deployment.logger.info('Ignored') + for filename in Deployment.CLASSES_FILENAMES: + self.copy_to_targetdir('ipp', filename) + Deployment.logger.info('... Installing executables to [%s]' % self.target_binary_directory) + for filename in Deployment.EXECUTABLES_FILENAMES: + self.copy_to_bindir('executables', filename) + Deployment.logger.info('... Deployment complete') + + def clean(self, doit = False): + """ + Clean the installation, i.e. remove files and subdirectories + if doit is True (default is False). Otherwise, doesn't do + anything... + """ + Deployment.logger.info('Cleaning') + if not doit: + Deployment.logger.warn('!!! NO DELETION WILL BE PERFORMED SINCE doit IS False') + try: + Deployment.logger.info('... Deleting temporary directory [%s]' % self.temporary_directory) + if doit: + for root, dirs, files in os.walk(self.temporary_directory, topdown=False): + Deployment.logger.debug('...... Deleting [%s]' % root) + for name in files: + Deployment.logger.debug('...... Deleting [%s] (in files)' % name) + os.remove(os.path.join(root, name)) + for name in dirs: + Deployment.logger.debug('...... Deleting [%s] (in dirs)' % name) + os.rmdir(os.path.join(root, name)) + os.rmdir(self.temporary_directory) + except AttributeError: + Deployment.logger.info('... Nothing to be cleaned') + Deployment.logger.info('... Cleaning complete') + +if __name__ == '__main__': + print "Running doctest" + import doctest + doctest.testmod() Index: /trunk/ippdor/deployment/__init__.py =================================================================== --- /trunk/ippdor/deployment/__init__.py (revision 32874) +++ /trunk/ippdor/deployment/__init__.py (revision 32874) @@ -0,0 +1,3 @@ +""" +This is the deployment module: a module to deploy Condor-IPP software. +""" Index: /trunk/ippdor/deployment/deploy.py =================================================================== --- /trunk/ippdor/deployment/deploy.py (revision 32874) +++ /trunk/ippdor/deployment/deploy.py (revision 32874) @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +""" +Utility executable for IPP Condor deployment +""" + +# Default values +class DefaultValues: + """ + This class contains all default values for the IPP Condor deployment + """ + svnSourceUrl = 'file:///home/schastel/svnrepo/ippdor/trunk/src' + targetDirectory = '/home/schastel/local/share/python/ipp' + targetBinaryDirectory = '/home/schastel/local/bin' + + def __init__(self): + """Not to be used as a class + """ + raise Exception('Not to be used as a class') + +import sys +import logging +#optparse Deprecated in python 2.7. Will be replaced by argparse +from optparse import OptionParser +from Deployment import Deployment + +if __name__ == '__main__': + USAGE = """usage: %prog [options] arg1 arg2 + +Deploy the Condor-IPP softwares +""" + PARSER = OptionParser(usage = USAGE) + PARSER.set_defaults(verbose=True) + PARSER.add_option( '-v', action = 'store_true', + dest = 'verbose', + help = 'Be verbose', + default = False) + PARSER.add_option('-q', action='store_true', + dest = 'quiet', + help = 'Be quiet', + default = False) + PARSER.add_option( '-s', '--svnSourceUrl', + dest = "svnSourceUrl", + help = 'SVN Source URL [default: %default]', + default = DefaultValues.svnSourceUrl ) + PARSER.add_option( '-t', '--targetDirectory', + dest = "targetDirectory", + help = 'Target Python Directory for classes [default: %default]', + default = DefaultValues.targetDirectory ) + PARSER.add_option( '-b', '--targetBinaryDirectory', + dest = "targetBinaryDirectory", + help = 'Target Executables Directory [default: %default]', + default = DefaultValues.targetBinaryDirectory ) + (OPTIONS, ARGS) = PARSER.parse_args() # ... the values are stored in options + + LOGGER = logging.getLogger('ippdor') + FORMATTER = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', + '%Y-%m-%dT%H:%M:%S') + STDOUT = logging.StreamHandler(sys.stdout) + STDOUT.setFormatter(FORMATTER) + LOGGER.addHandler(STDOUT) + if OPTIONS.verbose: + LOGGER.setLevel(logging.DEBUG) + LOGGER.debug('Logging level set to DEBUG') + elif OPTIONS.quiet: + LOGGER.setLevel(logging.ERROR) + LOGGER.info('Logging level set to ERROR') + else: + LOGGER.setLevel(logging.INFO) + + try: + DEPLOYMENT = Deployment(OPTIONS.svnSourceUrl, + OPTIONS.targetDirectory, + OPTIONS.targetBinaryDirectory) + DEPLOYMENT.check() + DEPLOYMENT.svn_export() + DEPLOYMENT.generate() + DEPLOYMENT.deploy() + DEPLOYMENT.clean(doit = True) + except Exception, e: + LOGGER.info('Exception caught') + DEPLOYMENT.clean(doit = False) + LOGGER.error(e) + import traceback + traceback.print_exc() Index: /trunk/ippdor/doc/Chip2Stack/stack.tex =================================================================== --- /trunk/ippdor/doc/Chip2Stack/stack.tex (revision 32874) +++ /trunk/ippdor/doc/Chip2Stack/stack.tex (revision 32874) @@ -0,0 +1,97 @@ +\documentclass[10pt]{article} +\usepackage[hmargin=0.5in,vmargin=0.5in]{geometry} + +\begin{document} + +\textbf{\Large FOR EACH EXPOSURE} + +\begin{description} +\item[Prerequisites] The exposures to be processed are + \textbf{labelled} externally in the chipRun table. +\item[A. Chip stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE A010] (Script) Generate the Condor job file containing the + \verb+chip_imfile.pl+ commands to be run on the cluster + \hfill{}\mbox{Equivalent to chip.imfile.load} + \item[A010] (Condor) Queue that Condor job file (exactly 60 + commands/jobs: one for each ota) \hfill{}\mbox{Equivalent to + chip.imfile.run} + \item[POST A010] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A020] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A020] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A030] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A030] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A040] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A040] (Script) If there is failing otas, send e-mail to + operator and stop processing. Otherwise generate stage 050 jobs file. + \item[A050] (Script) Advance the exposure with + \verb+chiptool -advanceexp+ \hfill{}\mbox{Equivalent to + chip.advanceexp} + \end{description} +\item[B. Camera stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE B010] (Script) Generate the Condor job file containing the + \verb+camera_exp.pl+ command(s?) \hfill{}\mbox{Equivalent to + camera.exp.load} + \item[B010] (Condor) Queue that Condor job file + \hfill{}\mbox{Equivalent to camera.exp.run} + \item[PRE B020] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B020] (Condor) Queue that Condor job file + \item[PRE B030] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B030] (Condor) Queue that Condor job file + \item[PRE B040] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B040] (Condor) Queue that Condor job file + \item[POST B040] (Script) If the camera entry could be reverted, send + e-mail to operator and stop processing + \end{description} +\item[C. Fake stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE C010] (Script) Generate the Condor job file containing the + \verb+fake_imfile.pl+ command(s?) \hfill{}\mbox{Equivalent to + fake.imfile.load} + \item[C010] (Condor) Queue that Condor job file + \hfill{}\mbox{Equivalent to fake.imfile.run} + \item[C020] (Script (or Script+Condor)) Run the + \verb+faketool -advanceexp+ command(s?) (or: PRE C020: Generate + the Condor job file containing the \verb+faketool -advanceexp+ + command(s?); C020: Queue that Condor job file) + \hfill{}\mbox{Equivalent to fake.advanceexp} + \end{description} +\item[D. Warp stage] Detail prerequisites \hrulefill{}\textbf{BARF}\mbox{ } + \begin{description} + \item[TODO] \verb+warptool -tooverlap+ \hfill{}\mbox{Equivalent to warp.exp.load} + \item[TODO] \verb+warp_overlap.pl+ \hfill{}\mbox{Equivalent to warp.exp.run} + \item[TODO] \hfill{}\mbox{Equivalent to warp.revert.overlap} + \item[TODO] \verb+warptool -towarp+ \hfill{}\mbox{Equivalent to warp.skycell.load} + \item[TODO] \verb+warp_skycell.pl+ \hfill{}\mbox{Equivalent to warp.skycell.run} + \item[TODO] \verb++ \hfill{}\mbox{Equivalent to warp.revert.warped} + \item[TODO] \verb+ warptool -advancerun+ \hfill{}\mbox{Equivalent to warp.advancerun} + \item[Check if needed?] \verb++ \hfill{}\mbox{Equivalent to + warp.summary.load and/or warp.summary.run} + \end{description} +\item[E. Stack stage] Detail prerequisites? \mbox{ } + \begin{description} + \item[TODO] \verb+stacktool -tosum+ \hfill{}\mbox{Equivalent to stack.skycell.load} + \item[TODO] \verb+stack_skycell.pl + \hfill{}\mbox{Equivalent to stack.skycell.run} + \item[TODO] \verb++ \hfill{}\mbox{Equivalent to stack.revert} + \item[Check if needed?] \verb++ \hfill{}\mbox{Equivalent to + stack.summary.load and/or stack.summary.run} + \end{description} +\end{description} + +\end{document} Index: /trunk/ippdor/doc/Full/Makefile =================================================================== --- /trunk/ippdor/doc/Full/Makefile (revision 32874) +++ /trunk/ippdor/doc/Full/Makefile (revision 32874) @@ -0,0 +1,10 @@ +MAIN = ippdor + +all: + pdflatex $(MAIN).tex + makeindex $(MAIN) + pdflatex $(MAIN).tex + +clean: + /bin/rm -f $(MAIN).pdf + CleanTex.sh Index: /trunk/ippdor/doc/Full/administration.tex =================================================================== --- /trunk/ippdor/doc/Full/administration.tex (revision 32874) +++ /trunk/ippdor/doc/Full/administration.tex (revision 32874) @@ -0,0 +1,34 @@ + +\chapter{Administration} +\label{chap-adm} + +\section{The \texttt{ippdor} user} +\label{sec-adm-ippdor} + +\texttt{ippdor} is the dedicated user for any processing happening in +Condor. This user uses the \texttt{bash} shell. The ippdor has no +particular privilege on the system. + +The \texttt{bashrc} file contains some definitions and expansions: +\begin{itemize} +\item \texttt{PATH} adds \texttt{\$HOME/local/bin}; +\item \texttt{LD\_LIBRARY\_PATH} adds \texttt{\$HOME/local/lib}; +\item \texttt{PYTHONPATH} adds \texttt{\$HOME/local/share/python} and + \texttt{\$HOME/local/lib/python2.6/site-packages}; +\item The \texttt{PSCONFIG} value is fixed to \texttt{current}. See + section~\ref{sec-link-ippTODO} for details. The + \texttt{psconfig.bash} utility is called using that value. +\end{itemize} + +\texttt{cmake} was locally installed to allow Condor build and +install. It is now the only external software. It was built using the +\texttt{--prefix=\$HOME/local} option and can be found in the +\texttt{\$HOME/softwares} directory. + +\section{Deploying Condor on a new host} +\label{sec-adm-newnode} + +\section{Updating Condor} +\label{sec-adm-update} + +To be written when it's happening... Index: /trunk/ippdor/doc/Full/annex.tex =================================================================== --- /trunk/ippdor/doc/Full/annex.tex (revision 32874) +++ /trunk/ippdor/doc/Full/annex.tex (revision 32874) @@ -0,0 +1,103 @@ +\chapter{Annex} + +\section{Held Jobs} +\label{annex-heldjobs} + +\null\hfill{\parbox{5in}{\scriptsize This example is related to the + DAG shown in section~\ref{sec-oper-tasks}. The A.condor and + B.condor jobs are defined as "echo A" and "echo B" commands.}} + +If \texttt{condor\_submit\_dag} is executed from a local directory +(e.g. \textbf{/export/ippc01.0/ippdor/simple\_dag}), the jobs will be +indefinitely held by Condor. + +\begin{verbatim} +ippdor@ippc01 /export/ippc01.0/ippdor/simple_dag $ condor_submit_dag AB.dag +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71204. +----------------------------------------------------------------------- +\end{verbatim} + +This can be seen with the +\texttt{condor\_q} command: + +\begin{verbatim} +$ condor_q + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu + ID OWNER SUBMITTED RUN_TIME ST PRI SIZE CMD +71204.0 ippdor 11/25 11:41 0+00:01:27 R 0 12.2 condor_dagman +71205.0 ippdor 11/25 11:42 0+00:00:01 H 0 0.0 echo A + +2 jobs; 0 idle, 1 running, 1 held +\end{verbatim} + +``In-depth'' analysis can be performed with \texttt{condor\_q -analyze} command: + +\begin{verbatim} +$ condor_q -analyze 71205.0 + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu +--- +71205.000: Request is held. + +Hold reason: Error from slot7@ipp029.ifa.hawaii.edu: Failed to execute +'/bin/echo' with arguments A: Cannot access specified iwd +"/export/ippc01.0/ippdor/simple_dag" (errno=2: 'No such file or directory') +\end{verbatim} + +The hold reason makes things obvious as soon as you know that +\texttt{iwd} stands for "Initial Working Directory". There is no way +that the Execute node \texttt{ipp029} can access the contents of what is +on the \texttt{ippc01} local file system. + +To fix the problem, stop processing with: +\begin{verbatim} +$ condor_rm 71204.0 +\end{verbatim} + +Note the creation of the \texttt{rescue} file in +\texttt{/export/ippc01.0/ippdor/simple\_dag} + +Now submit the DAG from any NFS directory: +\begin{verbatim} +ippdor@ippc01 ~ $ condor_submit_dag /export/ippc01.0/ippdor/simple_dag/AB.dag + +Running rescue DAG 1 +----------------------------------------------------------------------- +File for submitting this DAG to Condor : /export/ippc01.0/ippdor/simple_dag/AB.dag.condor.sub +Log of DAGMan debugging messages : /export/ippc01.0/ippdor/simple_dag/AB.dag.dagman.out +Log of Condor library output : /export/ippc01.0/ippdor/simple_dag/AB.dag.lib.out +Log of Condor library error messages : /export/ippc01.0/ippdor/simple_dag/AB.dag.lib.err +Log of the life of condor_dagman itself : /export/ippc01.0/ippdor/simple_dag/AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71208. +----------------------------------------------------------------------- +\end{verbatim} + +Note that the "rescue DAG 1" is used. + +... and the DAG should terminate correctly from the \texttt{condor\_q} +command and the contents of the \texttt{AB.dag.dagman.out} file: +\begin{verbatim} +[...] +11/25/11 12:13:58 Done Pre Queued Post Ready Un-Ready Failed +11/25/11 12:13:58 === === === === === === === +11/25/11 12:13:58 2 0 0 0 0 0 0 +11/25/11 12:13:58 0 job proc(s) currently held +11/25/11 12:13:58 All jobs Completed! +11/25/11 12:13:58 Note: 0 total job deferrals because of -MaxJobs limit (0) +11/25/11 12:13:58 Note: 0 total job deferrals because of -MaxIdle limit (0) +11/25/11 12:13:58 Note: 0 total job deferrals because of node category throttles +11/25/11 12:13:58 Note: 0 total PRE script deferrals because of -MaxPre limit (0) +11/25/11 12:13:58 Note: 0 total POST script deferrals because of -MaxPost limit (0) +11/25/11 12:13:58 **** condor_scheduniv_exec.71208.0 (condor_DAGMAN) pid 18455 EXITING WITH STATUS 0 +\end{verbatim} Index: /trunk/ippdor/doc/Full/design.tex =================================================================== --- /trunk/ippdor/doc/Full/design.tex (revision 32874) +++ /trunk/ippdor/doc/Full/design.tex (revision 32874) @@ -0,0 +1,4 @@ +\chapter{Software Design} +\label{chap-software} + +TODO Index: /trunk/ippdor/doc/Full/heldjobs.tex =================================================================== --- /trunk/ippdor/doc/Full/heldjobs.tex (revision 32874) +++ /trunk/ippdor/doc/Full/heldjobs.tex (revision 32874) @@ -0,0 +1,66 @@ +\chapter{Annex} + +\section{Held Jobs} +\label{annex-heldjobs} + +\nul\hfill{\scriptsize This example is related to the DAG shown in + section~\ref{sec-oper-tasks}. The A.condor and B.condor jobs are + defined as "echo A" and "echo B" commands.} + +If \texttt{condor\_submit\_dag} is executed from a local directory +(e.g. \textbf{/export/ippc01.0/ippdor/simple_dag}), the jobs will be +indefinitely held by Condor. + +\begin{verbatim} +ippdor@ippc01 /export/ippc01.0/ippdor/simple_dag $ condor_submit_dag AB.dag +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71204. +----------------------------------------------------------------------- +\end{verbatim} + +This can be seen with the +\texttt{condor\_q} command: + +\begin{verbatim} +$ condor_q + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu + ID OWNER SUBMITTED RUN_TIME ST PRI SIZE CMD +71204.0 ippdor 11/25 11:41 0+00:01:27 R 0 12.2 condor_dagman +71205.0 ippdor 11/25 11:42 0+00:00:01 H 0 0.0 echo A + +2 jobs; 0 idle, 1 running, 1 held +\end{verbatim} + +``In-depth'' analysis can be performed with \texttt{condor\_q -analyze} command: + +\begin{verbatim} +$ condor_q -analyze 71205.0 + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu +--- +71205.000: Request is held. + +Hold reason: Error from slot7@ipp029.ifa.hawaii.edu: Failed to execute +'/bin/echo' with arguments A: Cannot access specified iwd +"/export/ippc01.0/ippdor/simple_dag" (errno=2: 'No such file or directory') +\end{verbatim} + +The hold reason makes things obvious (as soon as you know that +\texttt{iwd} stands for "Initial Working Directory")... There is no +way that the Execute node \texttt{ipp029} can get the contents of what +is on \texttt{ippc01} local file system. + +To fix the problem, stop processing with: +\begin{verbatim} +$ condor_rm 71204.0 +\end{verbatim} + +Note the creation of the \texttt{rescue} file in \texttt{/export/ippc01.0/ippdor/simple\_dag} Index: /trunk/ippdor/doc/Full/ippdor.tex =================================================================== --- /trunk/ippdor/doc/Full/ippdor.tex (revision 32874) +++ /trunk/ippdor/doc/Full/ippdor.tex (revision 32874) @@ -0,0 +1,100 @@ +\documentclass[10pt]{report} + +\usepackage{vmargin} +\setpapersize{USletter} +\setmarginsrb{0.5in}{0.5in}{1.25in}{0.5in}{0pt}{0mm}{0pt}{0mm} +\usepackage{longtable} +\usepackage{hyperref} +\usepackage{makeidx} +\usepackage{enumitem} + +\title{Condor in the IPP} +\author{S.~Chastel\\Version 0.1} +\date{2011-11-23} + +\makeindex + +\newcommand{\CondorSubmit}{\texttt{condor\_submit}} +\newcommand{\CondorSubmitDag}{\texttt{condor\_submit\_dag}} + +\newcommand{\IppPython}[1]{\texttt{\~{ }ippdor/local/share/python/ipp/{#1}}} + +\newcommand{\definition}[3]{% + %#1: What is displayed in the text + %#2: What is displayed in the margin + %#3: Entry in the index + \emph{#1}\marginpar{\scriptsize #2}% + \index{#3}% +} + +\begin{document} +\maketitle + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\chapter*{Introduction} + +This is the documentation detailing how the Condor HTC system is used +to perform processing for the IPP. Note that it is (currently) not +supposed to replace the \texttt{pantasks} scheduling system (mainly +because we choose not to use Condor as a scheduler). + +This documentation consists of five parts: +\begin{enumerate} +\item This introduction; +\item Operational Users Manual (page~\pageref{chap-operation}): its + targets are the operators of the IPP, that is, persons who want to + use the Condor framework to process data for the IPP; +\item Administration Manual (page~\pageref{chap-adm}): for system + administrators. Mainly how to install and upgrade the Condor + framework; +\item Link with the IPP (page~\pageref{chap-ipp}): TODO: Should be + part of Software Organization; +\item Software Organization (page~\pageref{chap-software}): Design, + Classes. How to add new features. +\end{enumerate} + +\emph{Note:} Be aware that this documentation is not a substitute to +the Condor documentation\definition{}{Condor + documentation}{Condor!Documentation} which is available from the +Condor Project Homepage\footnote{when this document was released, it + was: + \href{http://research.cs.wisc.edu/condor/}{http://research.cs.wisc.edu/condor/}}. + +\subsection*{Document History} +\begin{tabular}{|c|c|p{5in}|} + \hline + \textbf{Version} & \textbf{Date} & \textbf{Comment} \\ + \hline + 0.1 & 2011-11-23 & First draft \\ + \hline +\end{tabular} + +\subsection*{Acronyms} +\begin{longtable}{lp{5in}} + IPP & Image Processing Pipeline \\ + IWD & Initial Working Directory \\ + RTCM & Read the Condor Manual \\ +\end{longtable} + +\setcounter{tocdepth}{2} +\tableofcontents + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{operations.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{administration.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{link_ipp.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{design.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{annex.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\printindex + +\end{document} Index: /trunk/ippdor/doc/Full/link_ipp.tex =================================================================== --- /trunk/ippdor/doc/Full/link_ipp.tex (revision 32874) +++ /trunk/ippdor/doc/Full/link_ipp.tex (revision 32874) @@ -0,0 +1,4 @@ +\chapter{Link with the IPP} +\label{chap-ipp} + +TODO Index: /trunk/ippdor/doc/Full/operations.tex =================================================================== --- /trunk/ippdor/doc/Full/operations.tex (revision 32874) +++ /trunk/ippdor/doc/Full/operations.tex (revision 32874) @@ -0,0 +1,622 @@ +\newcommand{\QueueExposuresWarping}{% + \texttt{queue\_exposures\_for\_warping.py}} +\newcommand{\QueueExposuresStacking}{% + \texttt{queue\_exposures\_for\_stacking.py}} + +\chapter{Operators Manual} +\label{chap-operation} + +\section{Condor Framework Availability on the IPP Production Cluster} + +The Condor framework is not available on all hosts of the IPP +production cluster. The purpose of this section is to give some +details about the different nodes which are used and their roles in +Condor. The set of all machines on which Condor is installed is called +the \definition{Condor pool}{Condor pool}{Condor!Pool}. + +The Condor \definition{central manager}{Central + manager}{Condor!Central manager} is \texttt{ippdb03}. From the +Condor documentation (section~3.1), the role of the central manager is +to "[collect] information, and [negotiate] between resources and +resource requests". It "plays a very important part in the Condor pool +and should be reliable" since "[if it] crashes, no further matchmaking +can be performed within the Condor system". Moreover "[all] queries go +to the central manager". + +The other nodes of the IPP production cluster where Condor is +installed are all both Execute and Submit Condor nodes. The complete +list of those nodes is available in the file +\texttt{\~{ }ippdor/local/share/CondorNodes}. Note that a node on which +Condor is installed is not necessary available for processing in +Condor since it can be deactivated (see section~\ref{sec-oper-hosts} +for details). + +An \definition{Execute}{Execute node}{Condor!Execute node} node is a +node where processing can be performed. From the Condor manual +section~3.1, "the only resource that might matter is disk +space"... Until now, it has never been a matter. + +A \definition{Submit}{Submit node}{Condor!Submit node} node is a node +where Condor jobs can be submitted (see +section~\ref{sec-ippdor-run}). It is suggested (again this is from the +Condor manual, section~3.1), that since "every job that you submit +that is currently running on a remote machine generates another +process on your submit machine [, ...] if you have lots of jobs +running, you will need a fair amount of swap space and/or real +memory". Until now, jobs were generally submitted from one of the +\texttt{ippc\emph{xy}} nodes. + +Note: For Condor gurus, there is no Checkpoint Server since all +processing is performed in the \emph{vanilla} Condor universe. + +\section{Condor Dedicated User: \texttt{ippdor}} +\label{sec-oper-login} + +To use the Condor framework you need to log into the IPP production +cluster as the user \texttt{ippdor}. Note that since Condor is not +installed on \texttt{ippc18}, you will have to login again into +another node for the condor commands to be usable. \texttt{ipp004} or +\texttt{ippdb03} are other "open" nodes where you can connect. + +Note: The \texttt{ippdor} account uses \texttt{bash}. + +Theoretically it should be possible to use the Condor framework with +your own login but 1) It has not been tested; 2) No support will be +provided for that; 3) So... RTCM. + +As detailed in section~\ref{sec-adm-ippdor}, the \texttt{ippdor} +account uses files that may be localized on the Condor node. If you +get a \texttt{command not found} error, make sure you are on one of +the node where Condor is installed\footnote{that is, the command + \texttt{"grep \`{ }hostname\`{ } \~{ + }ippdor/local/share/CondorNodes"} should display the host name. If + you don't see anything, read the first paragraph of this section.}. + +\section{Managing Condor Execution on the IPP Production Cluster + Nodes} +\label{sec-oper-hosts} + +Note 1: To install Condor on a new node, see +section~\ref{sec-adm-newnode}. + +Note 2: Except for a few home-grown commands, information was +extracted from the Condor Manual section~3.10. + +\subsection{Node Availability} +\label{sec-oper-hosts-availability} +\definition{}{Node availability}{Node availability}% +\definition{}{}{Condor!Node availability}% +To check if a node can be used by Condor, check the result of the +\texttt{ipp\_condor\_status } command. You should get some +information about the host from the Condor point of view. For +instance, \texttt{ipp060} potentially has 12 potential processors for +Condor execution\footnote{To get more information about what the + columns mean, RTCM}: +\begin{verbatim} +ippdor@ippc20 ~ $ ipp_condor_status ipp060 +Name OpSys Arch State Activity LoadAv Mem ActvtyTime + +slot10@ipp060.ifa. LINUX X86_64 Unclaimed Idle 0.000 4032 0+05:54:29 +slot11@ipp060.ifa. LINUX X86_64 Owner Idle 0.400 4032 0+00:00:06 +slot12@ipp060.ifa. LINUX X86_64 Owner Idle 1.000 4032 0+00:00:07 +slot1@ipp060.ifa.h LINUX X86_64 Owner Idle 0.840 4032 0+00:00:04 +slot2@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:34:09 +slot3@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+06:56:32 +slot4@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:25:44 +slot5@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:33:57 +slot6@ipp060.ifa.h LINUX X86_64 Claimed Idle 0.000 4032 3+11:22:07 +slot7@ipp060.ifa.h LINUX X86_64 Owner Idle 1.000 4032 0+00:00:29 +slot8@ipp060.ifa.h LINUX X86_64 Owner Idle 0.790 4032 0+00:00:03 +slot9@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+06:55:39 + Total Owner Claimed Unclaimed Matched Preempting Backfill + + X86_64/LINUX 12 5 1 6 0 0 0 + + Total 12 5 1 6 0 0 0 +\end{verbatim} + +If you don't get anything, the node has been deactivated for Condor +processing). For instance (while the author writes these lines), +\texttt{ipp065} will not execute any Condor jobs: +\begin{verbatim} +ippdor@ippc20 ~ $ ipp_condor_status ipp065 +ippdor@ippc20 ~ $ +\end{verbatim} + +\emph{Note:} \texttt{ipp\_condor\_status} is an alias of +\texttt{condor\_status} defined in \texttt{\~{ }ippdor/.bashrc}. + +\subsection{Make a Node Unavailable} +\label{sec-oper-hosts-remove} + +To remove a node from the list of execution nodes, execute the command +\texttt{ipp\_condor\_off } from any node where Condor is +installed. No job will be started/executed and submitted from that +host. \texttt{ipp\_condor\_off} is a function defined in \texttt{\~{ + }ippdor/.bashrc}. + +To stop Condor execution and submission from \definition{all}{Stopping + Condor on stare nodes}{Stopping Condor on stare nodes}stare nodes, +the \texttt{stare\_nodes\_off} and \texttt{stare\_nodes\_on} scripts +are provided. As it can be expected, those scripts just call +\texttt{ipp\_condor\_off} or \texttt{ipp\_condor\_on} for each of the +stares nodes. + +\subsection{Make a Node Available} +\label{sec-oper-hosts-available} + +To make a node available for Condor after it has been stopped, execute +the command \texttt{ipp\_condor\_on } from any node where +Condor is installed. + +\emph{Note:} If the following message is displayed, +\begin{verbatim} +bash: condor_master: command not found +\end{verbatim} +it simply means that Condor is not installed on the node where you try +to start it. + +\section{Condor Jobs and DAG} + +\subsection{Running Jobs in Condor} +\label{sec-oper-jobs} +We will call \definition{job}{job}{job} an elementary unit of +processing. It consists of the execution of one instance of an +executable (we will assume that an executable is a program that can be +called directly from the command line). For instance, "/bin/echo One" +and "/bin/echo Two" are two jobs, each job being an instance of the +same executable "/bin/echo". + +In our case, defining a correct Condor job is as simple (not to say +simplistic) as defining: +\begin{itemize} +\item The name of the executable; +\item The Condor universe of execution (in the IPP case, to avoid + recompilation of programs we use the \emph{vanilla} universe); +\item The file where the standard output of the executable will go (on + an NFS directory since we don't know where the executable will be + run); +\item The file where the error output of the executable will go (on + an NFS directory since we don't know where the executable will be + run); +\item The file where Condor will write the log of the job (on a local + directory); +\item The requirements about the node where execution will take place; +\item Some argument to tell that we want to keep the environment + variables; +\item The arguments for each execution of the executable. +\end{itemize} + +With the example given on \texttt{/bin/echo}, the contents of the +condor job file (let's call this file \texttt{echo\_test.condor}) are: +\begin{verbatim} +Executable = /bin/echo +Universe = vanilla +Output = /home/panstarrs/ippdor/tmp/tests/out.echo_test +Error = /home/panstarrs/ippdor/tmp/tests/err.echo_test +Log = /tmp/echo_test.log +Requirements = LoadAvg < 0.3 + +getenv = True + +Arguments = One +Queue + +Arguments = Two +Queue +\end{verbatim} + +To execute those jobs, they just need to be \definition{submitted}{Job + submission}{Job submission} from any Submit node to the Condor +system with: + +\texttt{condor\_submit echo\_test.condor} + +The jobs are (usually) not executed immediately and there is no way of +guessing the order on the jobs execution (if the order of jobs +execution is a matter, look at paragraph~\ref{sec-oper-tasks}). + +Note: From what the author experienced, \texttt{condor\_submit} and +the condor jobs definition file can be executed and placed on any kind +of file system (local and/or NFS) which is different from +\texttt{condor\_submit\_dag}. + +\subsection{Running tasks} +\label{sec-oper-tasks} + +\subsubsection{DAGMan} + +We will call \definition{task}{task}{task} a set of jobs where jobs +chaining depend on each other. Condor provides a mechanism of +organizing jobs according to their dependencies: +\definition{DAGMan}{DAGMan}{DAGMan} (which stands for Directed Acyclic +Graph Manager). Exhaustive documentation about DAGMan can be found in +the Condor documentation section 2.10. In this section I will just +give details about what is used for the IPP. + +\subsubsection{DAG Description File} + +To make it simple, a DAG is a high-level description of how jobs are +sequenced. If the job B has to be performed after the job A, the +dependency is formalized this way: +\begin{verbatim} +JOB A /some/directory/A.condor +JOB B /some/directory/B.condor +PARENT A CHILD B +\end{verbatim} + +Assuming the previous text is saved in the file named \texttt{AB.dag}, +the execution of the DAG can be \definition{submitted}{DAG + submission}{DAG submission} from any Submit node to the Condor +system using: + +\texttt{condor\_submit\_dag AB.dag} + +The output of that command will look like: +\begin{verbatim} +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71201. +----------------------------------------------------------------------- +\end{verbatim} + +All those files are created in the directory of the \texttt{AB.dag} +file. The \texttt{AB.dag.dagman.out} file is the most interesting log +file. + +\subsubsection{Files and Directories Locations} + +The DAG file can be localized on any file system: local or +NFS. However for performance concerns, I recommend to put it on the +local file system of the submit node. Monitor to the log growth rate +(the logs are not huge though... For instance, full processing of 2300 +MD06 exposures from Chip to Warp stages in the IPP created a log of 15 +MB for the \texttt{.out} file and of a total of about 50 kB for the 4 +other files). + +The \texttt{/some/directory} directory (where the condor jobs are +defined) can be placed on any file system. Again, I suggest to put +them on the local file system of the Submit node (mostly for +performance concerns). + +\begin{center} + \fbox{\parbox{6in}{\textbf{Important note:} The + \texttt{condor\_submit\_dag} command \textbf{must} be executed + \textbf{from a NFS file system} or the jobs will be indefinitely + \texttt{Held} (shown with \texttt{H} status in + \texttt{condor\_q}. See section~\ref{annex-heldjobs} for more + details.}} +\end{center} + +One restriction is that there cannot be cycles, i.e. circular +dependencies. For instance, job Egg depends on job Chicken, which +depend on job Egg. If you submit a DAG with a cycle, looking at the +\texttt{dagman.out} file, you will see a message like: +\begin{verbatim} +[...] +11/25/11 12:22:02 ERROR: DAG finished but not all nodes are complete -- + checking for a cycle... +11/25/11 12:22:02 ... ERROR: a cycle exists in the dag, plese check input +[...] +11/25/11 12:22:03 **** condor_scheduniv_exec.71211.0 (condor_DAGMAN) + pid 20996 EXITING WITH STATUS 1 +\end{verbatim} + +\subsubsection{Advanced DAGMan Features} +\label{sec-oper-dagman} + +DAGMan offers other convenient functions for a DAG description. As for +the other Condor features the reader is invited to check the Condor +documentation for in-depth details about them. + +\paragraph{SubDAG:} A graph node of a DAG can be itself a DAG making +the development and the analysis of a processing chain easier. A DAG +in a DAG is called a \definition{SubDAG}{SubDAG}{SubDAG} and can be +added to a DAG using this syntax: +\begin{verbatim} +SUBDAG EXTERNAL +\end{verbatim} + +\paragraph{Script:} Scripts canbe executed before or after a node +execution. They can be added using the following syntax: +\begin{verbatim} +SCRIPT PRE [] +SCRIPT POST [] +\end{verbatim} + +From the Condor documentation, "the \texttt{} specifies the +node to which the script is attached. The \texttt{} +specifies the script to be executed, and it may be followed by any +command line \texttt{} to that script". In the IPP context +we currently use only the \texttt{SCRIPT POST} feature. + +\subsection{Condor Interrupted} +\label{sec-oper-condor-interrupted} + +If the execution of Condor DAG is interrupted, for instance, because +of an operator \texttt{condor\_rm} or because of errors, Condor marks +the jobs or subdags that were successfully completed by adding the +\texttt{DONE} keyword to the DAG file. The resulting DAG file is saved +in a \definition{rescue DAG file}{Rescue DAG file}{Rescue DAG file}, +which is simply a copy of the original DAG file with a +\texttt{rescue} file extension (\texttt{} is +an auto-incrementing integer coded on three digits, starting at +\texttt{001}). + +To \definition{rescue processing}{Rescuing processing}{Rescuing + processing} you just need to execute \textbf{exactly} the same +submission command: Condor will use the last rescue file that was +produced. + +Note: If you know what you are doing you can create you rown rescue +file or add/remove \texttt{DONE} keywords to/from a rescue file. Some +utility programs can do that for you for some IPP stages +(see~\ref{TODO}). + +\section{IPP Tasks} +\label{sec-oper-ipp} + +The IPP defines complex tasks. The purpose of this section is to +provide a user manual about the programs that need to be run to +prepare the set of Condor files which are needed to perform some of +the IPP tasks. Developers should look at chapter~\ref{chap-software} +for in-depth details. + +Each subsection of this section details: 1) The software role; 2) Its +required inputs; 3) Its outputs; 4) Its IPP products. + +Currently the IPP parts that are implemented are: +\begin{itemize}% + \setlength{\itemsep}{0pt}\setlength{\topsep}{0pt}\setlength{\partopsep}{0pt}\setlength{\parsep}{0pt}% +\item Chip to Warp stages (paragraph~\ref{sec-oper-ipp-chip2warp}); +\item Chip to Stack stages (paragraph~\ref{sec-oper-ipp-chip2stack}); +\end{itemize} + +\subsection{Chip to Warp Processing: \QueueExposuresWarping{}} +\label{sec-oper-ipp-chip2warp} + +Since this section is supposed to belong to the operators manual, no +detail about the software will be provided. The developer will read +section~\ref{sec-TODO} for such details. + +\subsubsection{Role} +The role of \QueueExposuresWarping{} is to prepare the set of Condor +and IPP elements which are required to process a set of exposures from +the Chip to the Warp stage. + +The \QueueExposuresWarping{} script has four roles: +\begin{enumerate}[topsep=0pt, partopsep=0pt, itemsep=0pt, parsep=0pt, itemindent=10pt] +\item Assert that the label has not already been used and that the + input exposures exist; +\item Create entries in the \texttt{chipRun} table; +\item Create the DAG files which are necessary for the processing of + all exposures; +\item Display the \texttt{submit\_condor\_dag} command that needs to + be run by the operator. +\end{enumerate} + +Note that the operator has to run the \texttt{submit\_condor\_dag} +command. + +\subsubsection{Submit Node} + +A node needs to be chosen as Submit node. TODO: add details. + +\subsubsection{Inputs} +The inputs are: +\begin{itemize} +\item A \textbf{label} The label used for processing. If the label has + already been used, the program is aborted. As for the processing in + IPP, don't use spaces or non-ASCII characters for the label (for + instance, \texttt{My.New\_Label}); +\item An \textbf{e-mail address} who gets the e-mails (for instance, + \texttt{someone@mail.com}); +\item A \textbf{tessellation ID} (e.g. \texttt{MD04.V3}); +\item A \textbf{list of exposures} to be processed (for instance, + \texttt{o4926g0064o}, \texttt{o4926g0068o}, \texttt{o4926g0069o}, + \texttt{o4926g0071o}, and \texttt{o4940g0237o}). +\end{itemize} + +The \QueueExposuresWarping{} script can be called in two ways: +\begin{itemize} +\item Either they are provided in a file which has the format of a + Properties-style file. With the previous parameters: +\begin{verbatim} +label = My.New_Label +tessId = MD04.V3 +recipient = someone@mail.com +exposure = o4926g0064o +exposure = o4926g0068o +exposure = o4926g0069o +exposure = o4926g0071o +# Comment line +exposure = o4940g0237o # Line followed by a comment +\end{verbatim} + Assume that the file is stored under the name + \texttt{~{ }ippdor/examples/ChipsToWarps.ippdor} + the script is called with (note the \texttt{-f} option flag): +\begin{verbatim} +queue_exposures_for_warping.py -f ~ippdor/examples/01/ChipsToWarps.ippdor +\end{verbatim} +\item Or the input parameters are provided as arguments, that is (the + arguments have to be in this order): +\begin{verbatim} +queue_exposures_for_warping.py My.New_Label MD04.V3 someone@mail.com \ + o4926g0064o o4926g0068o o4926g0069o o4926g0071o o4940g0237o +\end{verbatim} +\end{itemize} + +\subsubsection{Outputs. Entries on GPC1 Database} +The script writes its outputs to the directory named +\texttt{/