Index: /trunk/pstamp/scripts/Makefile.am =================================================================== --- /trunk/pstamp/scripts/Makefile.am (revision 16932) +++ /trunk/pstamp/scripts/Makefile.am (revision 16933) @@ -3,5 +3,5 @@ install_files = \ - ppstamp_run.pl \ + pstamp_job_run.pl \ pstamp_listjobs.pl \ pstamp_new_request.sh \ @@ -9,5 +9,6 @@ pstamp_queue_requests.pl \ pstamp_runcommand.sh \ - pstamp_webrequest.pl + pstamp_webrequest.pl \ + pstamp_get_image_job.pl install_SCRIPTS = $(install_files) Index: unk/pstamp/scripts/ppstamp_run.pl =================================================================== --- /trunk/pstamp/scripts/ppstamp_run.pl (revision 16932) +++ (revision ) @@ -1,114 +1,0 @@ -#!/bin/env perl -### -### ppstamp_run.pl -### -### Run the postage stamp extraction tool to process a given job -### - -use warnings; -use strict; - -if (@ARGV != 1) { - die "usage: $0 job_id\n"; -} - -my $job_id = $ARGV[0]; - -my $verbosity = 1; - -use Sys::Hostname; -my $host = hostname(); - -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use IPC::Cmd 0.36 qw( can_run run ); - -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::Stats; -use PS::IPP::Metadata::List qw( parse_md_list ); - -use PS::IPP::Config qw($PS_EXIT_SUCCESS - $PS_EXIT_UNKNOWN_ERROR - $PS_EXIT_SYS_ERROR - $PS_EXIT_CONFIG_ERROR - $PS_EXIT_PROG_ERROR - $PS_EXIT_DATA_ERROR - $PS_EXIT_TIMEOUT_ERROR - metadataLookupStr - metadataLookupBool - caturi - ); - -my $missing_tools; - -my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); -my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); - -if ($missing_tools) { - warn("Can't find required tools."); - exit ($PS_EXIT_CONFIG_ERROR); -} - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -my $psjob; -#Look up the uri for the given job -{ - my $command = "$pstamptool -pendingjob -job_id $job_id"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbosity); - unless ($success) { - die("Unable to perform pstamptool -pendingjob: $error_code"); - } - - if (@$stdout_buf == 0) { - print STDERR "pending pstamp job id $job_id not found\n"; - # exit 1; - exit 0; - } - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - die("Unable to parse metdata config doc"); - - my $jobs = parse_md_list($metadata); - - $psjob = ${@$jobs}[0]; -} - -if (!$psjob) { - print STDERR "postage stamp job $job_id not found\n"; - # TODO: is this always an error, what if the job is no longer pending? - exit 1; -} - -my $uri = $psjob->{uri}; -my $outputBase = $psjob->{outputBase}; -my $argString = $psjob->{args}; - -my $jobStatus; -{ - my $command = "$ppstamp -file $uri $outputBase $argString"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbosity); - - if ($success) { - $jobStatus = $PS_EXIT_SUCCESS; - } else { - # run shifts the unix exit status up by 8 bits for some reason - $jobStatus = $error_code >> 8; - print STDERR "ppstamp failed with error code: $jobStatus\n"; - } -} - - -# stop the job and set the result value -{ - my $command = "$pstamptool -processedjob -job_id $job_id -state stop -result $jobStatus"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbosity); - unless ($success) { - die("Unable to perform $command: $error_code"); - } -} - -exit $jobStatus; Index: /trunk/pstamp/scripts/pstamp_job_run.pl =================================================================== --- /trunk/pstamp/scripts/pstamp_job_run.pl (revision 16933) +++ /trunk/pstamp/scripts/pstamp_job_run.pl (revision 16933) @@ -0,0 +1,126 @@ +#!/bin/env perl +### +### pstamp_job_run.pl +### +### Run a given postage stamp Job +### + +use warnings; +use strict; + +if (@ARGV != 1) { + die "usage: $0 job_id\n"; +} + +my $job_id = $ARGV[0]; + +my $verbosity = 1; + +use Sys::Hostname; +my $host = hostname(); + +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use IPC::Cmd 0.36 qw( can_run run ); + +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::Stats; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config qw($PS_EXIT_SUCCESS + $PS_EXIT_UNKNOWN_ERROR + $PS_EXIT_SYS_ERROR + $PS_EXIT_CONFIG_ERROR + $PS_EXIT_PROG_ERROR + $PS_EXIT_DATA_ERROR + $PS_EXIT_TIMEOUT_ERROR + metadataLookupStr + metadataLookupBool + caturi + ); + +my $missing_tools; + +my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1); +my $ppstamp = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1); +my $pstamp_get_image_job = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit ($PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $psjob; +#Look up the uri for the given job +{ + my $command = "$pstamptool -pendingjob -job_id $job_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbosity); + unless ($success) { + die("Unable to perform pstamptool -pendingjob: $error_code"); + } + + if (@$stdout_buf == 0) { + print STDERR "pending pstamp job id $job_id not found\n"; + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + die("Unable to parse metdata config doc"); + + my $jobs = parse_md_list($metadata); + + $psjob = ${@$jobs}[0]; +} + +if (!$psjob) { + print STDERR "postage stamp job $job_id not found\n"; + exit 1; +} + +my $uri = $psjob->{uri}; +my $outputBase = $psjob->{outputBase}; +my $argString = $psjob->{args}; +my $jobType = $psjob->{jobType}; + +my $jobStatus; +if ($jobType eq "stamp") { + my $command = "$ppstamp -file $uri $outputBase $argString"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbosity); + + if ($success) { + $jobStatus = $PS_EXIT_SUCCESS; + } else { + $jobStatus = $error_code >> 8; + print STDERR "ppstamp failed with error code: $jobStatus\n"; + } +} elsif ($jobType eq "get_image") { + my $command = "$pstamp_get_image_job --uri $uri --out_dir $outputBase"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbosity); + + if ($success) { + $jobStatus = $PS_EXIT_SUCCESS; + } else { + $jobStatus = $error_code >> 8; + print STDERR "ppstamp failed with error code: $jobStatus\n"; + } +} else { + die("unknown jobType $jobType found"); +} + + +# stop the job and set the result value +{ + my $command = "$pstamptool -processedjob -job_id $job_id -state stop -result $jobStatus"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbosity); + unless ($success) { + die("Unable to perform $command: $error_code"); + } +} + +exit $jobStatus; Index: /trunk/pstamp/src/pstampfinish.c =================================================================== --- /trunk/pstamp/src/pstampfinish.c (revision 16932) +++ /trunk/pstamp/src/pstampfinish.c (revision 16933) @@ -182,5 +182,5 @@ } -static bool finishRequest(psrfOptions *options, pstampRequestRow *pReq, psArray *stoppedJobs) +static bool buildResultsFile(psrfOptions *options, pstampRequestRow *pReq, psArray *stoppedJobs) { // build results file in dsRoot/pReq->outFileset @@ -238,5 +238,5 @@ psString command = NULL; - psStringAppend(&command, "dsreg --add --type PSRESPONSE --product %s --fileset %s", + psStringAppend(&command, "dsreg --add --type PSRESULTS --product %s --fileset %s", options->dsProduct, fileset_id); @@ -266,4 +266,14 @@ psError(PS_ERR_UNKNOWN, true, "dsreg failed with status: %d", status); return false; + } + + return true; +} +static bool finishRequest(psrfOptions *options, pstampRequestRow *pReq, psArray *stoppedJobs) +{ + if (pReq->resultsFile) { + if (!buildResultsFile(options, pReq, stoppedJobs)) { + return false; + } } // update the database setting the request state to stop Index: /trunk/pstamp/src/pstampparse.c =================================================================== --- /trunk/pstamp/src/pstampparse.c (revision 16932) +++ /trunk/pstamp/src/pstampparse.c (revision 16933) @@ -72,8 +72,9 @@ psArgumentRemove(argnum, &argc, argv); } + if ((argnum = psArgumentGet(argc, argv, "-req_id"))) { psArgumentRemove(argnum, &argc, argv); if (argnum == argc) { - fprintf(stderr, "value required for mode\n"); + fprintf(stderr, "value required for req_id\n"); usage(); } @@ -598,15 +599,23 @@ static bool -queueOneJob (pspOptions *options, psString uri, psString outputBase, psString commandArgs) +queueOneJob (pspOptions *options, psString job_type, psString uri, psString outputBase, psString commandArgs) { psString cmd = NULL; - psStringAppend(&cmd, "pstamptool -addjob -req_id %ld -uri %s -outputBase ", options->req_id, uri); + psStringAppend(&cmd, "pstamptool -addjob -req_id %ld -job_type %s -uri %s ", + options->req_id, job_type, uri); - if (options->outputDirectory) { - psStringAppend(&cmd, "%s/", options->outputDirectory); - } + if (outputBase != NULL) { + psStringAppend(&cmd, "-outputBase "); - psStringAppend(&cmd, "%s -args '%s' >/dev/null", outputBase, commandArgs); + if (options->outputDirectory) { + psStringAppend(&cmd, "%s/", options->outputDirectory); + } + psStringAppend(&cmd, "%s", outputBase); + } + if (commandArgs) { + psStringAppend(&cmd, " -args '%s'", commandArgs); + } + psStringAppend(&cmd, " > /dev/null"); if (options->verbose) { @@ -626,5 +635,5 @@ static bool -queueJobs (pspOptions *options, psArray *uris, psString user_tag, psString commandArgs) +queueJobs (pspOptions *options, psString job_type, psArray *uris, psString user_tag, psString commandArgs) { int numURIs = psArrayLength(uris); @@ -633,5 +642,5 @@ if (numURIs == 1) { // use the user tag as the outputBase - status = queueOneJob(options, uris->data[0], user_tag, commandArgs); + status = queueOneJob(options, job_type, uris->data[0], user_tag, commandArgs); } else { for (int i = 0; i< numURIs; i++) { @@ -641,5 +650,5 @@ psStringAppend(&outputBase, "%s_%d", user_tag, i); - status = queueOneJob(options, uris->data[i], outputBase, commandArgs); + status = queueOneJob(options, job_type, uris->data[i], outputBase, commandArgs); psFree(outputBase); @@ -653,4 +662,53 @@ } +static bool +turnOffResultsFile(pspOptions *options) +{ + // turn off the creation of a results file for this request + char * query = "UPDATE pstampRequest SET resultsFile = 0 WHERE req_id = %" PRId64; + return p_psDBRunQuery(options->config->database, query, options->req_id); +} + +static bool +queueGetImageJob(pspOptions *options, psArray *uris, psString req_type) +{ + FILE *listFile; + psString fileName = NULL; + psString img_type = psMetadataLookupStr(NULL, options->md, "IMG_TYPE"); + + if (! turnOffResultsFile(options)) { + fprintf(stderr, "failed to update resultsFile for request %" PRId64 "\n", options->req_id); + return false; + } + + if (options->outputDirectory == NULL) { + fprintf(stderr, "outputDirectory is required\n"); + return false; + } + psStringAppend(&fileName, "%s/filelist", options->outputDirectory); + + listFile = fopen(fileName, "w"); + if (listFile == NULL) { + psError(PS_ERR_UNKNOWN, true, "failed to open get_image file list: %s\n", fileName); + return false; + } + + for (int i=0; idata[i], img_type); + } + fclose(listFile); + + psString cmd = NULL; + psStringAppend(&cmd, "pstamptool -addjob -req_id %ld -job_type get_image -uri %s -outputBase %s", + options->req_id, fileName, options->outputDirectory); + + int rstatus = system(cmd); + if (rstatus) { + fprintf(stderr, "%s failed with status: %d\n", cmd, rstatus); + } + + return rstatus == 0; +} + static bool @@ -658,7 +716,14 @@ { bool status; + psString job_type = psMetadataLookupStr(&status, options->md, "JOB_TYPE"); + + if (job_type == NULL) { + // XXX: compatiabilty hack change to error at some point + job_type = "stamp"; + } + psString cmd_mode = psMetadataLookupStr(&status, options->md, "CMD_MODE"); - // if mode is set on the command line ignore the value in the request file + // if cmd_mode is set on the command line ignore the value in the request file if (options->mode == PSP_MODE_UNKNOWN) { if (cmd_mode != NULL) { @@ -682,26 +747,27 @@ return false; } + psString roiString = parseROI(options); -#ifdef notdef - if (!roiString) { - return false; - } -#endif psArray *uris = NULL; + if (!strcmp(req_type, "byid")) { // directly by exp_id, chip_id, warp_id etc + uris = parseByID(options); } else if (!strcmp(req_type, "byexp")) { // images that resulted from a given exposure + uris = parseByExp(options); } else if (!strcmp(req_type, "bycoord")) { // images from a particular region of the sky + if (!roiString) { - fprintf(stderr, "req_type: bycoord with NULL ROI\n"); + fprintf(stderr, "Region of interest required with req_type: bycoord\n"); return false; } uris = parseByCoord(options); } else { + psError(PS_ERR_UNKNOWN, true, "unknown REQ_TYPE: %s", req_type); return false; @@ -728,24 +794,32 @@ psStringAppend(&commandArgs, " -chip %s", class_id); } - psString user_tag = psMetadataLookupStr(&status, options->md, "USER_TAG"); - if (user_tag == NULL) { - psError(PS_ERR_UNKNOWN, true, "no user tag in request file"); - return false; - } if (options->mode == PSP_MODE_LIST_JOB) { if (numURIs == 1) { - printf("%s %s %s\n", (psString) uris->data[0], user_tag, commandArgs); + printf("%s %s %s %s\n", job_type, (psString) uris->data[0], + user_tag ? user_tag : "", commandArgs ? commandArgs : "") ; } else { // add an integer to the user_tag for (int i = 0; i < numURIs; i++) { - printf("%s %s_%d %s\n", (psString) uris->data[i], user_tag, i, commandArgs); + printf("%s %s %s_%d %s\n", job_type, (psString) uris->data[i], user_tag, i, + commandArgs ? commandArgs :""); } } } else if (options->mode == PSP_MODE_QUEUE_JOB) { - if (!queueJobs(options, uris, user_tag, commandArgs)) { - return false; + if (!strcmp(job_type, "stamp")) { + // Postage stamp job + if (user_tag == NULL) { + psError(PS_ERR_UNKNOWN, true, "no user tag in request file"); + return false; + } + + if (!queueJobs(options, job_type, uris, user_tag, commandArgs)) { + return false; + } + } else { + + return queueGetImageJob(options, uris, req_type); } Index: /trunk/pstamp/src/pstamprequest.c =================================================================== --- /trunk/pstamp/src/pstamprequest.c (revision 16932) +++ /trunk/pstamp/src/pstamprequest.c (revision 16933) @@ -105,9 +105,9 @@ psMetadata *md = psMetadataAlloc(); int argnum; - bool gotStyle = false; + bool gotStyle = false; bool needCoord = false; bool gotCenter = false; - bool gotRange = false; - bool needROI = true; + bool gotRange = false; + bool needROI = true; bool makeStamps = true; @@ -116,6 +116,7 @@ psMetadataAdd (md, PS_LIST_TAIL, "VERSION", PS_DATA_STRING, "", STAMP_REQUEST_VERSION); + // These mode arguments should be consistent with pstampparse if ((argnum = psArgumentGet(argc, argv, "-list"))) { - psMetadataAdd (md, PS_LIST_TAIL, "CMD_MODE", PS_DATA_STRING, "", "LIST_URI"); + psMetadataAdd (md, PS_LIST_TAIL, "CMD_MODE", PS_DATA_STRING, "", "list_uri"); psArgumentRemove(argnum, &argc, argv); // we don't need coordinates if we're listing unless mode is -bycoord. May be set true below. @@ -123,4 +124,15 @@ makeStamps = false; } + + if ((argnum = psArgumentGet(argc, argv, "-get_image"))) { + psMetadataAdd (md, PS_LIST_TAIL, "JOB_TYPE", PS_DATA_STRING, "", "get_image"); + psArgumentRemove(argnum, &argc, argv); + // we don't need coordinates if we're listing unless mode is -bycoord. May be set true below. + needROI = false; + makeStamps = false; + } else { + psMetadataAdd (md, PS_LIST_TAIL, "JOB_TYPE", PS_DATA_STRING, "", "stamp"); + } + // get project from command line