Index: /branches/sc_branches/psps_testing/README =================================================================== --- /branches/sc_branches/psps_testing/README (revision 29111) +++ /branches/sc_branches/psps_testing/README (revision 29111) @@ -0,0 +1,13 @@ +1. Running the unit tests: +python -m doctest testers/batch_file.py +python -m doctest testers/batch_manifest_file.py + +2. Coverage +2.1 Coverage run +python-coverage -x testers/batch_file.py +python-coverage -x testers/batch_manifest_file.py + +2.2 Coverage report + +python-coverage -r -o /usr +python-coverage -r -o /usr -m Index: /branches/sc_branches/psps_testing/build.py =================================================================== --- /branches/sc_branches/psps_testing/build.py (revision 29111) +++ /branches/sc_branches/psps_testing/build.py (revision 29111) @@ -0,0 +1,32 @@ +#!/usr/bin/python + +# Setup logging +import logging +datefmt = '%H:%M:%S' +fmt = '[%(asctime)8s/@%(filename)s/%(funcName)s()/%(lineno)03d %(levelname)5s] %(message)s' +logging.basicConfig(level=logging.INFO, format=fmt) +logger = logging.Logger("main") +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter(fmt, datefmt)) +logger.addHandler(handler) +logger.setLevel(logging.INFO) + +# +from testers.batch_file import BatchFileTester + +FILES_TO_CHECK = [ 'data/samples/B00029152.tar.gz', + 'data/samples/B00030978.tar.gz', + 'data/samples/B00031481.tar.gz', + 'data/samples/B00032090.tar.gz', + 'data/samples/B00032701.tar.gz', + 'data/samples/B00033107.tar.gz', + 'data/samples/B00033820.tar.gz', + 'data/samples/B00034120.tar.gz', + 'data/samples/B00034574.tar.gz', + 'data/samples/B00035357.tar.gz', + ] + +if __name__ == '__main__': + for file_to_check in FILES_TO_CHECK: + logger.info('Testing for entry %s' % (file_to_check)) + print BatchFileTester(file_to_check).test() Index: /branches/sc_branches/psps_testing/build.sh =================================================================== --- /branches/sc_branches/psps_testing/build.sh (revision 29111) +++ /branches/sc_branches/psps_testing/build.sh (revision 29111) @@ -0,0 +1,47 @@ +#!/bin/bash + +# +# This is the script used to test the test framework +# To run the tests themselves, run build.py +# + +PYTHON=/usr/bin/python +COVERAGE=/usr/bin/python-coverage + +FILES="testers/__init__.py \ + testers/batch_file.py \ + testers/batch_manifest_file.py \ + testers/fits_file.py \ + utilities/__init__.py \ + utilities/configuration.py \ + utilities/file_manipulation.py" + +case "$1" in + unittest) + for file in $FILES; do + echo "Unit test for $file ($PYTHON -m doctest $file)" + $PYTHON -m doctest $file + done + ;; + coverage) + for file in $FILES; do + echo "Coverage for $file" + $COVERAGE -x $file + done + echo "### Coverage results ###" + $COVERAGE -r -o /usr -m + ;; + all) + $0 unittest + $0 coverage + ;; + clean) + mkdir -p trash + for file in `find . -name \*.pyc`; do echo "Deleting $file"; mv $file trash/; done + for file in `find . -name \*~`; do echo "Deleting $file"; mv $file trash/; done + ;; + *) + echo "Usage: $0 [ all | unittest | coverage]" + exit 1 + ;; +esac Index: /branches/sc_branches/psps_testing/data/psut/README =================================================================== --- /branches/sc_branches/psps_testing/data/psut/README (revision 29111) +++ /branches/sc_branches/psps_testing/data/psut/README (revision 29111) @@ -0,0 +1,7 @@ +This directory contains files used to unittest the testing system. + +The files in the ko subdirectory are supposed to be invalid in +the PSPS framework. +The files in the ok directory are supposed to be valid in the +PSPS framework. + Index: /branches/sc_branches/psps_testing/data/psut/ko/B00000009/BatchManifest.xml =================================================================== --- /branches/sc_branches/psps_testing/data/psut/ko/B00000009/BatchManifest.xml (revision 29111) +++ /branches/sc_branches/psps_testing/data/psut/ko/B00000009/BatchManifest.xml (revision 29111) @@ -0,0 +1,6 @@ + + + + + + Index: /branches/sc_branches/psps_testing/data/samples/original/README =================================================================== --- /branches/sc_branches/psps_testing/data/samples/original/README (revision 29111) +++ /branches/sc_branches/psps_testing/data/samples/original/README (revision 29111) @@ -0,0 +1,3 @@ +Data provided by Roy (2010-08-26). +All should be ingested in PSPS db. +Only P2, i.e. detections. Index: /branches/sc_branches/psps_testing/data/xsd/BatchManifest.xsd =================================================================== --- /branches/sc_branches/psps_testing/data/xsd/BatchManifest.xsd (revision 29111) +++ /branches/sc_branches/psps_testing/data/xsd/BatchManifest.xsd (revision 29111) @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: /branches/sc_branches/psps_testing/doc/icd_changes/icd_changes.tex =================================================================== --- /branches/sc_branches/psps_testing/doc/icd_changes/icd_changes.tex (revision 29111) +++ /branches/sc_branches/psps_testing/doc/icd_changes/icd_changes.tex (revision 29111) @@ -0,0 +1,35 @@ +\documentclass{article} + +\newcommand{\change}[2]{ + \emph{#1}\footnote{#2}} + +\begin{document} +The purpose of this document is to reflect the changes that should be +made to \texttt{PSDC-940-006-01}. Suggested changes are +\emph{emphasized} while the old version is shown in a footnote. + +\section{Section 3.4.1: Batch File Naming Convention} + +Batch files must be uniquely named within a job to facilitate +verification of the job manifest. \change{Batches and jobs are related + through an ID number: the Batch ID and the Job ID numbers are + equals}{It is also preferable to relate the batch to a job.}. + +\change{A batch file must be named using the following case- +sensitive naming convention: +\begin{center} + \texttt{B$<$JobID$>$.tar.gz} +\end{center} +where $<$JobID$>$ is a unique 8-digits positive integer}{A batch file +must be named using the following case-sensitive naming convention: +JOBID\_BATCHID.EXT Where: "JOBID" is the literal 'J' followed by a +sequence number identifying the job to which the batch belongs; e.g, +"J1" for the first job. "BATCHID" is the literal 'B' followed by a +sequence number uniquely identifying the batch, given the context of +the JOBID; i.e., the BATCHID is initialized to 1 for every job; and, +"EXT" is the literal "tar.gz". For example, the first batch file for +the first job should have the filename J1\_B1.jar.gz}. + + + +\end{document} Index: /branches/sc_branches/psps_testing/testers/batch_file.py =================================================================== --- /branches/sc_branches/psps_testing/testers/batch_file.py (revision 29111) +++ /branches/sc_branches/psps_testing/testers/batch_file.py (revision 29111) @@ -0,0 +1,217 @@ +# Strings managed as files +from StringIO import StringIO +# Regular expressions +import re +# OS operations for system calls +import subprocess +import os +from utilities.file_manipulation import FileManipulation +from testers.batch_manifest_file import BatchManifestFileTester +from testers.fits_file import FitsFileTester +# Temporary files and directories +import tempfile +# Global definitions +from utilities.configuration import Configuration + +class BatchFileTester: + """ + The purpose of this class is to test the common features of batch + files. Namely, they are: + - the form of the file name + - the contents of the file (a gzipped tar archive file) and the form + of the files contained in the archive + + >>> print BatchFileTester("data/psut/ok/B00029152.tar.gz").test() + PSDC-940-006-01, 3.4.1: OK + PSDC-940-006-01, 3.4: OK + PSDC-940-006-01, 3.5.1.2: OK + PSDC-940-006-01, 3.5.1.1: OK + PSDC-940-006-01, 3.5.1 (file attributes/name): OK + PSDC-940-006-01, 3.5.1 (file attributes/size): OK + PSDC-940-006-01, 3.5.1 (file attributes/MD5 checksum of file): OK + """ + def __init__(self, filename): + """ + Creates a BatchFileTester. + + >>> bft = BatchFileTester("1/2/3/4/dummy.tar.gz") + >>> print bft._filename + 1/2/3/4/dummy.tar.gz + >>> print bft.expected_basename + dummy/ + """ + self._filename = filename + self.expected_basename = os.path.basename(self._filename + ).replace(Configuration.EXTENSION, + '') + '/' + + def test(self, start_dependant_tests = True): + """ + Runs the different tests. Runs the dependant tests if all + tests are succesful. + """ + report = StringIO() + testsOk = True + (message, status) = self._test_file_name("PSDC-940-006-01, 3.4.1") + report.write(message) + testsOk = testsOk and status + (message, status) = self._test_file_format("PSDC-940-006-01, 3.4") + report.write('\n' + message) + testsOk = testsOk and status + # Now runs the dependant tests + if testsOk and start_dependant_tests: + # Uncompress the archive (since it's valid) to some temp dir + tempDir = tempfile.mkdtemp() + command = '/bin/tar xvfz %s -C %s' % (self._filename, tempDir) + (status, + child_stdout, + child_stderr) = FileManipulation.system(command) + # Run the tests on the batch manifest file + batchManifestFileTester = BatchManifestFileTester(self.expected_basename, + tempDir) + (_report, fits_info) = batchManifestFileTester.test() + report.write('\n' + _report) + # Run the tests on the fits file + fitsFileTester = FitsFileTester(fits_info['filename'], + self.expected_basename, tempDir, + fits_info['filetype']) + (_report, info) = fitsFileTester.test() + report.write('\n' + _report) + # End of test... Delete the temporary directory + FileManipulation.delete_directory(tempDir) + return report.getvalue() + + def _test_file_name(self, requirement): + """ + From PSDC-940-006-01, 3.4.1: + Batch files must be uniquely named within a job to facilitate + verification of the job manifest. Batches and jobs are related + through an ID number: the Batch ID and the Job ID numbers are + equals 1 . A batch file must be named using the following + case- sensitive naming con- vention: B.tar.gz where + is a unique 8-digits positive intege + + >>> batchFileTester = BatchFileTester('B12345678.tar.gz') + >>> print batchFileTester._test_file_name("REQ_ID") + ('REQ_ID: OK', True) + + >>> batchFileTester = BatchFileTester('B1234567.tar.gz') + >>> print batchFileTester._test_file_name("REQ_ID") + ("REQ_ID: KO (invalid archive file basename 'B1234567.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('B123456789.tar.gz') + >>> print batchFileTester._test_file_name("REQ_ID") + ("REQ_ID: KO (invalid archive file basename 'B123456789.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('B123a5678.tar.gz') + >>> print batchFileTester._test_file_name("REQ_ID") + ("REQ_ID: KO (invalid archive file basename 'B123a5678.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('b12345678.tar.gz') + >>> print batchFileTester._test_file_name("REQ_ID") + ("REQ_ID: KO (invalid archive file basename 'b12345678.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('B12345678.tgz') + >>> print batchFileTester._test_file_name("REQ_ID") + ("REQ_ID: KO (invalid archive file basename 'B12345678.tgz')", False) + + >>> batchFileTester = BatchFileTester('B12345678.TAR.GZ') + >>> print batchFileTester._test_file_name("REQ_ID") + ("REQ_ID: KO (invalid archive file basename 'B12345678.TAR.GZ')", False) + """ + report = StringIO() + basename = os.path.basename(self._filename) + if re.match('B\d{8}\.tar.gz', basename) == None: + return ('%s: KO (invalid archive file basename \'%s\')' % (requirement, basename), False) + else: + report.write('%s: OK' % requirement) + return (report.getvalue(), True) + + def _test_file_format(self, requirement): + """ + From "PSDC-940-006-01, 3.4" + + Both the IPP and the PSPS are designed to run on Linux + platforms. Common Linux utilities, tar and gzip, will be used + to group and compress FITS files, respectively, into an + archive file. Each archive file is the basis for a "batch" + transfer. + + Providing batch data in one compressed archive file may result + in a very large file which increases the probability of + corruption, and can make the file awkward to process. So: + + Derived Requirement 3.3.1: The maximum size of a batch file + shall not exceed 100 gigabytes (TBR). + + -> Test 1: The file is a valid tar gzipped file + -> Test 2: The file uncompresses in the directory + where is the file base name + -> Test 3: The tar gzipped file contains exactly one XML file + (named BatchManifest.xml) and exactly one FITS file. + + >>> batchFileTester = BatchFileTester('data/psut/ok/B00029152.tar.gz') + >>> print batchFileTester._test_file_format("REQ_ID") + ('REQ_ID: OK', True) + + >>> batchFileTester = BatchFileTester('This_file_does_not_exist.tar.gz') + >>> print batchFileTester._test_file_format("REQ_ID") + ("REQ_ID: KO (invalid tar gzipped archive file 'This_file_does_not_exist.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('data/psut/ko/B00000001.tar.gz') + >>> print batchFileTester._test_file_format("REQ_ID") + ("REQ_ID: KO (invalid directory name in archive 'data/psut/ko/B00000001.tar.gz': Got tmp/ instead of B00000001/)", False) + + >>> batchFileTester = BatchFileTester('data/psut/ko/B00000002.tar.gz') + >>> print batchFileTester._test_file_format("REQ_ID") + ("REQ_ID: KO (missing XML Manifest file in archive 'data/psut/ko/B00000002.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('data/psut/ko/B00000003.tar.gz') + >>> print batchFileTester._test_file_format("REQ_ID") + ("REQ_ID: KO (missing FITS file in archive 'data/psut/ko/B00000003.tar.gz')", False) + + >>> batchFileTester = BatchFileTester('data/psut/ko/B00000004.tar.gz') + >>> print batchFileTester._test_file_format("REQ_ID") + ("REQ_ID: KO (unexpected file 'B00000004/extra' in archive 'data/psut/ko/B00000004.tar.gz')", False) + """ + (status, + child_stdout, + child_stderr) = FileManipulation.system('/bin/tar tfz ' + self._filename) + # Test 1: if the archive is not valid + if status != 0: + return (requirement + ': KO (invalid tar gzipped archive file \'%s\')' % self._filename, False) + # Now check the results of the standard output which must be: + # / + # /.FITS + # /BatchManifest.xml + # or + # / + # /BatchManifest.xml + # /.FITS + # Note: there are CR/LF that have to be removed at the end of + # stdout lines + status = 0 + fits_file_in_archive = False + xml_file_in_archive = False + for line in child_stdout: + line = line.rstrip() + if status == 0: + if line != self.expected_basename: + return (requirement + ': KO (invalid directory name in archive \'%s\': Got %s instead of %s)' % (self._filename, line, self.expected_basename), False) + status = 1 + elif status == 1: + if re.match('.*\.FITS$', line): + fits_file_in_archive = True + elif re.match('.*\.xml$', line): + xml_file_in_archive = True + else: + return (requirement + ': KO (unexpected file \'%s\' in archive \'%s\')' % (line, self._filename), False) + if not xml_file_in_archive: + return (requirement + ': KO (missing XML Manifest file in archive \'%s\')' % (self._filename), False) + if not fits_file_in_archive: + return (requirement + ': KO (missing FITS file in archive \'%s\')' % (self._filename), False) + return (requirement + ': OK', True) + +if __name__ == '__main__': + import doctest + doctest.testmod() Index: /branches/sc_branches/psps_testing/testers/batch_manifest_file.py =================================================================== --- /branches/sc_branches/psps_testing/testers/batch_manifest_file.py (revision 29111) +++ /branches/sc_branches/psps_testing/testers/batch_manifest_file.py (revision 29111) @@ -0,0 +1,209 @@ +# Strings managed as files +from StringIO import StringIO +# System call and so on +from utilities.file_manipulation import FileManipulation +# Global definitions +from utilities.configuration import Configuration +# Simple XML manipulation +from xml.dom import minidom +# Load some utilities to compute md5 checksum +import hashlib + +class BatchManifestFileTester: + _TMPDIR = '.tmp' + @staticmethod + def _setup_test_environment(basename, tgz_filename): + """ + This method is here just to create the test environment + for the unit tests of this class and nothing else. + >>> BatchManifestFileTester._setup_test_environment("tmp/dummy", "filename.tar.gz") + """ + # Tidy up + BatchManifestFileTester._remove_test_environment() + # Create the temporary directory for tests + FileManipulation.system('mkdir -p %s ' % BatchManifestFileTester._TMPDIR) + FileManipulation.system('/bin/tar xvfz %s -C %s' % (tgz_filename, BatchManifestFileTester._TMPDIR)) + @staticmethod + def _remove_test_environment(): + """ + This method is here just to remove the test environment + for the unit tests of this class and nothing else. + """ + FileManipulation.system('/bin/rm -rf %s' % BatchManifestFileTester._TMPDIR) + + def __init__(self, basename, tempDir): + """ + Creates an instance of a BatchManifestFileTester + + >>> basename = 'B00029152' + >>> tgz_filename = 'data/psut/ok/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> print bmft.manifest + .tmp/B00029152/BatchManifest.xml + >>> # Delete temporary directory + >>> BatchManifestFileTester._remove_test_environment() + """ + self._manifest_relative_directory = tempDir + '/' + basename + self.manifest = ( self._manifest_relative_directory + '/' + + Configuration.BATCH_MANIFEST ) + + def test(self): + """ + Executes the various tests. + TODO: This method should call/be split into individual tests. + + >>> # Unit test: valid file + >>> basename = 'B00029152' + >>> tgz_filename = 'data/psut/ok/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> (report, fits_info) = bmft.test() + >>> print report + PSDC-940-006-01, 3.5.1.2: OK + PSDC-940-006-01, 3.5.1.1: OK + PSDC-940-006-01, 3.5.1 (file attributes/name): OK + PSDC-940-006-01, 3.5.1 (file attributes/size): OK + PSDC-940-006-01, 3.5.1 (file attributes/MD5 checksum of file): OK + >>> print fits_info + {'filetype': u'P2', 'filename': u'00105439.FITS'} + >>> BatchManifestFileTester._remove_test_environment() + + >>> # Unit test: invalid manifest file name in archive + >>> basename = 'B00000005' + >>> tgz_filename = 'data/psut/ko/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> (report, fits_info) = bmft.test() + >>> print report + PSDC-940-006-01, 3.5.1.2: KO (No .tmp/B00000005/BatchManifest.xml file) + >>> print fits_info + None + >>> BatchManifestFileTester._remove_test_environment() + + >>> # Unit test: manifest file does not validate + >>> basename = 'B00000006' + >>> tgz_filename = 'data/psut/ko/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> (report, fits_info) = bmft.test() + >>> print report + PSDC-940-006-01, 3.5.1.2: OK + PSDC-940-006-01, 3.5.1.1: KO (XML validation through XSD failed: command was [/usr/bin/xmllint --schema data/xsd/BatchManifest.xsd --noout .tmp/B00000006/BatchManifest.xml]) + >>> print fits_info + None + >>> BatchManifestFileTester._remove_test_environment() + + >>> # Unit test: manifest file does not validate (no or incorrect FITS file name) + >>> basename = 'B00000007' + >>> tgz_filename = 'data/psut/ko/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> (report, fits_filename) = bmft.test() + >>> print report + PSDC-940-006-01, 3.5.1.2: OK + PSDC-940-006-01, 3.5.1.1: OK + PSDC-940-006-01, 3.5.1 (file attributes/name): KO (no FITS file is named '.tmp/B00000007/00105439.FITS') + >>> print fits_info + None + >>> BatchManifestFileTester._remove_test_environment() + + >>> # Unit test: manifest file does not validate (incorrect size of FITS file) + >>> basename = 'B00000008' + >>> tgz_filename = 'data/psut/ko/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> (report, fits_info) = bmft.test() + >>> print report + PSDC-940-006-01, 3.5.1.2: OK + PSDC-940-006-01, 3.5.1.1: OK + PSDC-940-006-01, 3.5.1 (file attributes/name): OK + PSDC-940-006-01, 3.5.1 (file attributes/size): KO (actual size of FITS file is '0' instead of '10336320') + >>> print fits_info + None + >>> BatchManifestFileTester._remove_test_environment() + + >>> # Unit test: manifest file does not validate (incorrect MD5 sum of FITS file) + >>> basename = 'B00000009' + >>> tgz_filename = 'data/psut/ko/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> BatchManifestFileTester._setup_test_environment(basename, tgz_filename) + >>> bmft = BatchManifestFileTester(basename, '.tmp') + >>> (report, fits_info) = bmft.test() + >>> print report + PSDC-940-006-01, 3.5.1.2: OK + PSDC-940-006-01, 3.5.1.1: OK + PSDC-940-006-01, 3.5.1 (file attributes/name): OK + PSDC-940-006-01, 3.5.1 (file attributes/size): OK + PSDC-940-006-01, 3.5.1 (file attributes/MD5 checksum of file): KO (actual md5sum of FITS file is 'd41d8cd98f00b204e9800998ecf8427e' instead of '852461f5e14ff6ce5f0b0edd7166b515') + >>> print fits_info + None + >>> BatchManifestFileTester._remove_test_environment() + """ + + report = StringIO() + # Check file existence and proper naming in archive + requirement = 'PSDC-940-006-01, 3.5.1.2: ' + if not FileManipulation.exists(self.manifest): + report.write(requirement + 'KO (No %s file)' % (self.manifest)) + return (report.getvalue(), None) + else: + report.write(requirement + 'OK') + # Check validation of XML by XSD through xmllint + requirement = 'PSDC-940-006-01, 3.5.1.1: ' + command = '/usr/bin/xmllint --schema %s --noout %s' + (status, + stdout, + stderr) = FileManipulation.system(command % (Configuration.XSD_FILE, + self.manifest) ) + if status != 0: + report.write('\n' + requirement + 'KO (XML validation through XSD failed: command was [%s])' % (command % (Configuration.XSD_FILE, self.manifest) )) + return (report.getvalue(), None) + else: + report.write('\n' + requirement + 'OK') + # Get XML information concerning FITS file + # Note: We are only interested in the first element + xmldoc = minidom.parse(self.manifest) + fileList = xmldoc.getElementsByTagName('file') + # File existence + requirement = 'PSDC-940-006-01, 3.5.1 (file attributes/name): ' + fitsFilename = fileList[0].attributes['name'].value + fileToLookFor = self._manifest_relative_directory + '/' + fitsFilename + if not FileManipulation.exists(fileToLookFor): + report.write('\n' + requirement + 'KO (no FITS file is named \'%s\')' % (fileToLookFor)) + return (report.getvalue(), None) + report.write('\n' + requirement + 'OK') + # File size match + requirement = 'PSDC-940-006-01, 3.5.1 (file attributes/size): ' + expected_size = fileList[0].attributes['bytes'].value + actual_size = str(FileManipulation.size(fileToLookFor)) + if actual_size != expected_size: + report.write('\n' + requirement + 'KO (actual size of FITS file is \'%s\' instead of \'%s\')' % (actual_size, expected_size)) + return (report.getvalue(), None) + report.write('\n' + requirement + 'OK') + # File md5sum + requirement = 'PSDC-940-006-01, 3.5.1 (file attributes/MD5 checksum of file): ' + expected_md5sum = fileList[0].attributes['md5'].value + actual_md5sum = FileManipulation.md5(fileToLookFor) + if actual_md5sum != expected_md5sum: + report.write('\n' + requirement + 'KO (actual md5sum of FITS file is \'%s\' instead of \'%s\')' % (actual_md5sum, expected_md5sum)) + return (report.getvalue(), None) + report.write('\n' + requirement + 'OK') + # TODO: Perform some validation of the XML content that XSD can't do + # e.g.: minObjId < maxObjId. + # Note: if other objects need to be provided in the output, just add them to the dictionary... + # e.g.: return (..., {'filename': fitsFilename, 'fitsFilesize': fitsFilesize}) + return (report.getvalue(), + { 'filename': fitsFilename, + 'filetype': xmldoc.getElementsByTagName('manifest')[0].attributes['type'].value}) + +if __name__ == '__main__': + import doctest + doctest.testmod() Index: /branches/sc_branches/psps_testing/testers/fits/p2.py =================================================================== --- /branches/sc_branches/psps_testing/testers/fits/p2.py (revision 29111) +++ /branches/sc_branches/psps_testing/testers/fits/p2.py (revision 29111) @@ -0,0 +1,13 @@ + + +class P2FitsTester: + """ + This is the class that tests the contents of a P2 FITS file + provided by IPP. The contents can be checked against the database + through SOAP queries. + """ + def __init__(self, fitsFile): + self.fits = fitsFile + + def test(self): + return ("None", None) Index: /branches/sc_branches/psps_testing/testers/fits_file.py =================================================================== --- /branches/sc_branches/psps_testing/testers/fits_file.py (revision 29111) +++ /branches/sc_branches/psps_testing/testers/fits_file.py (revision 29111) @@ -0,0 +1,84 @@ +from utilities.file_manipulation import FileManipulation +import pyfits +from testers.fits.p2 import P2FitsTester + +""" +This is the class intended to test FITS file provided by IPP to PSPS. + +""" +class FitsFileTester: + class Type: + """ + Definition taken from 5.2 + """ + UNKNOWN = {'id': 00, 'code': ''} + INITIALIZATION = 10 + DETECTION = 20 + STACK = 30 + DIFFERENCE = 40 + OBJECTS = 50 + + _TMPDIR = '.tmp' + @staticmethod + def _setup_test_environment(basename, tgz_filename): + """ + This method is here just to create the test environment + for the unit tests of this class and nothing else. + >>> FitsFileTester._setup_test_environment("tmp/dummy", "filename.tar.gz") + """ + # Tidy up + FitsFileTester._remove_test_environment() + # Create the temporary directory for tests + FileManipulation.system('mkdir -p %s ' % FitsFileTester._TMPDIR) + FileManipulation.system('/bin/tar xvfz %s -C %s' % (tgz_filename, FitsFileTester._TMPDIR)) + @staticmethod + def _remove_test_environment(): + """ + This method is here just to remove the test environment + for the unit tests of this class and nothing else. + """ + FileManipulation.system('/bin/rm -rf %s' % FitsFileTester._TMPDIR) + + def __init__(self, fitsname, basename, tempDir, fitsType): + """ + Creates an instance of a FitsFileTester + + >>> fitsname = '00105439.FITS' + >>> basename = 'B00029152' + >>> tgz_filename = 'data/psut/ok/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> FitsFileTester._setup_test_environment(basename, tgz_filename) + >>> fft = FitsFileTester(fitsname, basename, '.tmp', 'P2') + >>> print fft.filename + .tmp/B00029152/00105439.FITS + >>> # Delete temporary directory + >>> FitsFileTester._remove_test_environment() + """ + self.filename = tempDir + '/' + basename + '/' + fitsname + self.fitsType = fitsType + self.fitsFile = pyfits.open(self.filename) + + def __del__(self): + """ + Closes the FITS file + """ + self.fitsFile.close() + + def test(self): + """ + Tests the contents of the FITS file + + >>> fitsname = '00105439.FITS' + >>> basename = 'B00029152' + >>> tgz_filename = 'data/psut/ok/' + basename + '.tar.gz' + >>> # Create the temporary directory for tests + >>> FitsFileTester._setup_test_environment(basename, tgz_filename) + >>> fft = FitsFileTester(fitsname, basename, '.tmp', 'P2') + >>> fft.test() + >>> # Delete temporary directory + >>> FitsFileTester._remove_test_environment() + """ + if self.fitsType == "P2": + return P2FitsTester(self.fitsFile).test() + else: + raise Exception("Unsupported FITS type [%s]" % fitsType) Index: /branches/sc_branches/psps_testing/utilities/configuration.py =================================================================== --- /branches/sc_branches/psps_testing/utilities/configuration.py (revision 29111) +++ /branches/sc_branches/psps_testing/utilities/configuration.py (revision 29111) @@ -0,0 +1,9 @@ + +class Configuration: + # The extension of expected files + EXTENSION = '.tar.gz' + # The name of the Batch Manifest XML file (defined in + # PSDC-940-006-01, 3.5.1.1 "Batch Manifest File Structure" + BATCH_MANIFEST = 'BatchManifest.xml' + # The location of the validating XSD file + XSD_FILE = 'data/xsd/BatchManifest.xsd' Index: /branches/sc_branches/psps_testing/utilities/file_manipulation.py =================================================================== --- /branches/sc_branches/psps_testing/utilities/file_manipulation.py (revision 29111) +++ /branches/sc_branches/psps_testing/utilities/file_manipulation.py (revision 29111) @@ -0,0 +1,135 @@ +# Strings managed as files +from StringIO import StringIO +# OS operations for system calls +import subprocess +import os +import hashlib + +class FileManipulation: + @staticmethod + def system(command): + """ + Runs a command on the system. + + >>> (status, stdout, stderr) = FileManipulation.system("/bin/ls no_file_here") + >>> print status + 2 + >>> for line in stdout: + ... print line + >>> for line in stderr: + ... print line + /bin/ls: cannot access no_file_here: No such file or directory + + """ + process = subprocess.Popen(command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=True) + (child_stdout, + child_stderr) = (process.stdout, process.stderr) + process.wait() + return (process.returncode, child_stdout, child_stderr) + + @staticmethod + def delete_directory(directory): + """ + A utility function to delete possibly non-empty directories. + + >>> import os + >>> dir_name = "/tmp/__tests_for_FileManipulation.py" + >>> report = FileManipulation.delete_directory(dir_name) + >>> os.mkdir(dir_name) + >>> print FileManipulation.delete_directory(dir_name) + Executing [/bin/rm -rf /tmp/__tests_for_FileManipulation.py] + Command return status: 0 + >>> os.mkdir(dir_name) + >>> FileManipulation.touch(dir_name + '/blah') + >>> print FileManipulation.delete_directory(dir_name) + Executing [/bin/rm -rf /tmp/__tests_for_FileManipulation.py] + Command return status: 0 + """ + logger = StringIO() + command = '/bin/rm -rf %s' % (directory) + logger.write("Executing [%s]\n" % command) + (status, + child_stdout, + child_stderr) = FileManipulation.system(command) + logger.write(" Command return status: %d" % status) + return logger.getvalue() + + @staticmethod + def touch(fname, times = None): + """ + Equivalent of unix touch command. + From http://stackoverflow.com/questions/1158076/implement-touch-using-python + + >>> fileNameForTouchTest = '/tmp/fileNameForTouchTest' + >>> (status, stdout, stderr) = FileManipulation.system('/bin/rm -f ' + fileNameForTouchTest) + >>> FileManipulation.touch(fileNameForTouchTest) + >>> print FileManipulation.exists(fileNameForTouchTest) + True + >>> (status, stdout, stderr) = FileManipulation.system('/bin/rm -f ' + fileNameForTouchTest) + """ + with file(fname, 'a'): + os.utime(fname, times) + + @staticmethod + def exists(path): + """ + Return True if path refers to an existing path. Returns False + for broken symbolic links. + + >>> FileManipulation.exists('utilities/file_manipulation.py') + True + >>> FileManipulation.exists('no_file_with_this_name') + False + """ + return os.path.exists(path) + + @staticmethod + def md5(fileName, excludeLine="", includeLine=""): + """ + Computes md5 hash of the specified file. Shamelessly stolen from: + http://thejaswihr.blogspot.com/2008/06/python-md5-checksum-of-file.html + + >>> FileManipulation.md5('No file with this name on the system') + Traceback (most recent call last): + ... + IOError: [Errno 2] No such file or directory: 'No file with this name on the system' + + >>> fileUnderMD5Test = '/usr/bin/python' + >>> (status, stdout, stderr) = FileManipulation.system('/usr/bin/md5sum ' + fileUnderMD5Test) + >>> lines = '' + >>> for line in stdout: lines = lines + line + >>> expected_md5value = lines.split(' ')[0] + >>> actual_md5value = FileManipulation.md5(fileUnderMD5Test) + >>> print (expected_md5value == actual_md5value) + True + """ + m = hashlib.md5() + fobj = file(fileName, 'rb') + while True: + d = fobj.read(8096) + if not d: + break + m.update(d) + return m.hexdigest() + + @staticmethod + def size(filename): + """ + Returns the size in bytes of a file + + >>> fileNameForSizeTest = '/tmp/fileNameForSizeTest' + >>> (status, stdout, stderr) = FileManipulation.system('/bin/rm -f ' + fileNameForSizeTest) + >>> FileManipulation.touch(fileNameForSizeTest) + >>> print FileManipulation.size(fileNameForSizeTest) + 0 + >>> (status, stdout, stderr) = FileManipulation.system('/bin/rm -f ' + fileNameForSizeTest) + """ + return os.path.getsize(filename) + +if __name__ == '__main__': + import doctest + doctest.testmod()