Index: trunk/DataStore/scripts/dsget =================================================================== --- trunk/DataStore/scripts/dsget (revision 25285) +++ trunk/DataStore/scripts/dsget (revision 25299) @@ -69,5 +69,5 @@ undef $copies; } - if ($copies < 1) { + elsif ($copies < 1) { die "--copies must be >= 1"; } Index: trunk/dbconfig/add.md =================================================================== --- trunk/dbconfig/add.md (revision 25299) +++ trunk/dbconfig/add.md (revision 25299) @@ -0,0 +1,25 @@ +addRun METADATA + add_id S64 0 # Primary Key AUTO_INCREMENT + cam_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 + label STR 64 + dvodb STR 255 + magicked S64 0 +END + +addProcessedExp METADATA + add_id S64 0 # Primary Key AUTO_INCREMENT + dtime_addstar F32 0.0 + + n_stars S32 0 + + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +addMask METADATA + label STR 64 # Primary Key +END + Index: trunk/dbconfig/changes.txt =================================================================== --- trunk/dbconfig/changes.txt (revision 25285) +++ trunk/dbconfig/changes.txt (revision 25299) @@ -1211,2 +1211,39 @@ ALTER TABLE publishDone ADD COLUMN hostname VARCHAR(64) AFTER path_base; ALTER TABLE publishDone ADD COLUMN dtime_script FLOAT AFTER hostname; + +-- Version 1.1.55 +-- addstar stage tables: + +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + magicked BIGINT, + PRIMARY KEY(add_id), + KEY(add_id), + KEY(cam_id), + KEY(state), + KEY(workdir_state), + KEY(label), + INDEX(add_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + n_stars INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; Index: trunk/dbconfig/ipp.m4 =================================================================== --- trunk/dbconfig/ipp.m4 (revision 25285) +++ trunk/dbconfig/ipp.m4 (revision 25299) @@ -14,4 +14,5 @@ include(chip.md) include(cam.md) +include(add.md) include(fake.md) include(warp.md) Index: trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h (revision 25285) +++ trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h (revision 25299) @@ -55,3 +55,5 @@ EXTERN int EXPIRE_TRAIL_TIME; /* Expire trails after this interval */ +EXTERN int PERSIST_RETAIN; /* Retain bad-slope persistence fits */ + #endif /* _INCLUDED_burnparams_ */ Index: trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25285) +++ trunk/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 25299) @@ -100,4 +100,6 @@ EXPIRE_TRAIL_TIME = 2000; /* Expire a persist after this [sec] */ + PERSIST_RETAIN = 0; /* Retain persists with bad slopes? */ + /* Parse the args */ cellxy = -1; @@ -174,4 +176,8 @@ } else if(strncmp(argv[i], "infits=", 7) == 0) { /* infits=fname */ persistfitsfile = argv[i] + 7; + +/* Keep persistence streaks which had a bad slope? */ + } else if(strncmp(argv[i], "persist=", 8) == 0) {/* persist={t|f} */ + PERSIST_RETAIN = argv[i][8] == 'y' || argv[i][8] == '1' || argv[i][8] == 't'; /* Output file for PSF gallery */ Index: trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 25285) +++ trunk/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 25299) @@ -53,4 +53,12 @@ power laws; downward burns as exponentials. + The fit to downward, persistence burns may have an unreasonable "slope" + (meaning exponential sign), increasing away from the burn origin. + These are discarded unless "persist=t" is invoked. It is possible + for the fit to be fooled by uncataloged stars, so the persistence + can be kept in the output file with "persist=t" (although no fit + correction is applied) and they will propagate from input to output + until they finally achieve a legal fit with negligible amplitude. + Burntool also identifies really blasted areas which are saturated from top to bottom. These cannot be fitted, but are carried along for @@ -231,4 +239,8 @@ 'in' takes precedence. + persist={t|f} + Retain persistence streaks whose fit slope turned out to be + unreasonable. + out=fname Output file for burn streaks @@ -332,4 +344,5 @@ BUGS: 090224: Still in development + 090810: Squished a few memory bugs SEE ALSO: Index: trunk/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c (revision 25285) +++ trunk/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c (revision 25299) @@ -222,6 +222,14 @@ for(k=0; k 0) { + boxbuf[nbox].zero = (double *)calloc(boxbuf[nbox].nfit, sizeof(double)); + boxbuf[nbox].xfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int)); + boxbuf[nbox].yfit = (int *)calloc(boxbuf[nbox].nfit, sizeof(int)); + for(i=0; i 0) zk = box[k].zero[box[k].nfit/2]; for(jp=kp+1; jp 0) zj = box[j].zero[box[j].nfit/2]; if(ABS(yctr[jp]-yctr[kp]) > DIFFERENT_STREAK) { /* Trim back the feebler streak */ @@ -238,6 +241,15 @@ /* First: patched up persists */ for(k=0; k= FIT_MAX_SLOPE || slope < FIT_MIN_SLOPE) { + box->slope = slope; + box->nfit = 0; box->fiterr = FIT_SLOPE_ERROR; return(-1); Index: trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (revision 25285) +++ trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (revision 25299) @@ -1039,4 +1039,5 @@ newval[strlen(newval) - 1] = '\0'; fh_set_str(hu, idx, name, newval, comment); + free(newval); return; } @@ -1519,4 +1520,29 @@ i++; } + return FH_SUCCESS; +} + +static void +pad_header(HeaderUnit hu, int n) +{ + int i; + double idx = 900000000; + + for (i = 0; i < n; i++) + { + fh_set_card(hu, idx++, FH_RESERVE); + } +} + +fh_result +fh_copy(HeaderUnit hu, const HeaderUnit source_) +{ + fh_result result; + int reserve; + + result = fh_merge(hu, source_); + if (result != FH_SUCCESS) return result; + reserve = fh_get_reserve(source_); + if (reserve) pad_header(hu, reserve); return FH_SUCCESS; } @@ -1757,4 +1783,21 @@ } +int +fh_get_reserve(HeaderUnit hu) +{ + HeaderUnitStruct* list = FH_HU(hu); + + if (!list) + { + log_error("invalid argument to fh_get_reserve()"); + return -1; + } + + if (list->reserve) + return list->reserve; + else + return list->reserve_found; +} + fh_result fh_rewrite(HeaderUnit hu) Index: trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h =================================================================== --- trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h (revision 25285) +++ trunk/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h (revision 25299) @@ -174,4 +174,9 @@ * is used to create a new FITS header. NOT for use with fh_rewrite(). */ +int fh_get_reserve(HeaderUnit hu); +/* + * Get the current setting or the number of reserve cards found in + * a header unit. Returns -1 if hu is invalid. + */ fh_result fh_validate(HeaderUnit hu); /* fh_validate.c; for use by fhtool.c */ @@ -386,4 +391,5 @@ double fh_idx(HeaderUnit hu); /* idx of the last card returned by fh_next */ fh_result fh_merge(HeaderUnit hu, const HeaderUnit source); /* source unchanged */ +fh_result fh_copy(HeaderUnit hu, const HeaderUnit source); /* fh_merge+keep reserve */ /* --------------------------------------------------------- @@ -459,4 +465,3 @@ */ - #endif /* _INCLUDED_fh */ Index: trunk/ippScripts/Build.PL =================================================================== --- trunk/ippScripts/Build.PL (revision 25285) +++ trunk/ippScripts/Build.PL (revision 25299) @@ -49,4 +49,5 @@ scripts/chip_imfile.pl scripts/camera_exp.pl + scripts/addstar_run.pl scripts/fake_imfile.pl scripts/warp_overlap.pl Index: trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- trunk/ippScripts/scripts/addstar_run.pl (revision 25299) +++ trunk/ippScripts/scripts/addstar_run.pl (revision 25299) @@ -0,0 +1,250 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +#my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +#my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +#my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +#my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, + $no_op, $redirect, $save_temps, $run_state); +GetOptions( + 'exp_tag=s' => \$exp_tag, # Exposure identifier + 'add_id=s' => \$add_id, # Camtool identifier + 'recipe=s' => \$recipe, # Recipe to use + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'outroot|w=s' => \$outroot, # output file base name + 'camroot|w=s' => \$camroot, # camera stage root name. + 'reduction=s' => \$reduction, # Reduction class + 'dvodb|w=s' => \$dvodb, # output DVO database + 'run-state=s' => \$run_state, # 'new' or 'update' + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --exp_tag --add_id --camera --outroot", + -exitval => 3, + ) unless + defined $exp_tag and + defined $add_id and + defined $outroot and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if (not defined $run_state) { $run_state = 'new'; } +if ($run_state eq 'update') { + $logDest .= '.update'; +} + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $add_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +&my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; + +#my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use +#&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $cmdflags; + +# Get list of component files +my $files; # Array of component files +{ + my $command = "$addtool -pendingexp -add_id $add_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addtool: $error_code", $add_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); +} + +my $chipObjectsExist = 0; + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if ($run_state eq 'update') { + $traceDest .= '.update'; +} + +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} + +my $dtime_addstar = 0; + +unless ($no_op) { + if (defined $dvodbReal and ($run_state eq 'new')) { + ## XXX the camera analysis can either save the full set of + ## detections, or just the image metadata, in the dvodb + + ## get the addstar recipe for this camera and CAMERA reduction + my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR); + + ## allow the dvodb to save only images, or the full detection set + my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -image" if $imagesOnly; + $command .= " -D CATDIR $dvodbReal"; + + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + } +} + + +# This needs to be updated when addtool is written. BROKEN +my $fpaCommand = "$addtool -add_id $add_id"; +if ($run_state eq 'new') { + $fpaCommand .= " -addprocessedexp"; + $fpaCommand .= " -path_base $outroot"; + $fpaCommand .= " $cmdflags"; + $fpaCommand .= " -dtime_addstar $dtime_addstar"; +} else { + $fpaCommand .= " -updaterun -state full"; +} +$fpaCommand .= " -dbname $dbname" if defined $dbname; + +# Add the result into the database +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } +} else { + print "skipping command: $fpaCommand\n"; +} + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $add_id = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + if (defined $add_id and not $no_update) { +# This needs to be updated when addtool is written. BROKEN + my $command = "$addtool -add_id $add_id"; + if ($run_state eq 'new') { + $command .= " -addprocessedexp"; + $command .= " -uri UNKNOWN"; + $command .= " -fault $exit_code"; + $command .= " -path_base $outroot"; + $command .= " -path_base $outroot" if defined $outroot; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + } else { + $command .= " -updateprocessedexp"; + $command .= " -fault $exit_code"; + } + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + exit $exit_code; +} + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippScripts/scripts/camera_exp.pl =================================================================== --- trunk/ippScripts/scripts/camera_exp.pl (revision 25285) +++ trunk/ippScripts/scripts/camera_exp.pl (revision 25299) @@ -93,6 +93,6 @@ &my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; +#my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +#&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use @@ -205,5 +205,5 @@ } -my $dtime_addstar = 0; +#my $dtime_addstar = 0; unless ($no_op) { @@ -297,47 +297,47 @@ # run addstar on the output fpaObjects (if a DVO database is defined) - if (defined $dvodbReal and ($run_state eq 'new')) { - - ## XXX the camera analysis can either save the full set of - ## detections, or just the image metadata, in the dvodb - - ## get the addstar recipe for this camera and CAMERA reduction - $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); - - ## allow the dvodb to save only images, or the full detection set - my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - # XXX this needs to be converted to addstar_client... - - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $command; - $command = "$addstar -D CAMERA $camdir -update"; - $command .= " -image" if $imagesOnly; - $command .= " -D CATDIR $dvodbReal"; - - my $realFile = $ipprc->file_resolve($fpaObjects); - $command .= " $realFile"; - - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script - } +# if (defined $dvodbReal and ($run_state eq 'new')) { + +# ## XXX the camera analysis can either save the full set of +# ## detections, or just the image metadata, in the dvodb + +# ## get the addstar recipe for this camera and CAMERA reduction +# $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; +# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); +# } +# my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or +# &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); + +# ## allow the dvodb to save only images, or the full detection set +# my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + +# # XXX this construct requires the user to have a valid .ptolemyrc +# # XXX which in turn points at ippconfig/dvo.site +# # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) +# # XXX this needs to be converted to addstar_client... + +# my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar +# my $command; +# $command = "$addstar -D CAMERA $camdir -update"; +# $command .= " -image" if $imagesOnly; +# $command .= " -D CATDIR $dvodbReal"; + +# my $realFile = $ipprc->file_resolve($fpaObjects); +# $command .= " $realFile"; + +# my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); +# } +# $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script +# } } } @@ -353,5 +353,5 @@ $fpaCommand .= " -hostname $host" if defined $host; $fpaCommand .= " -dtime_script $dtime_script"; - $fpaCommand .= " -dtime_addstar $dtime_addstar"; +# $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { $fpaCommand .= " -updaterun -state full"; Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25285) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25299) @@ -60,8 +60,8 @@ -my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1, - detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1, - detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1, - detrend.resid.imfile => 1, detrend.resid.exp => 1 ); +my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, + "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1, + "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1, + "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 ); unless ($stages{$stage}) { die "unknown stage $stage for ipp_cleanup.pl\n"; @@ -119,7 +119,8 @@ # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - print STDERR "CHIP: CONFIG_FILE : $config_file\n"; + if (!$config_file or ! -e $config_file) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " @@ -128,4 +129,13 @@ } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } if ($status) { @@ -160,7 +170,12 @@ if ($mode eq "goto_purged") { $command .= " -topurgedimfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -218,4 +233,5 @@ my $status = 1; # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); @@ -225,4 +241,12 @@ $status = 0; } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -246,5 +270,5 @@ } if ($mode eq "goto_scrubbed") { - $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; + $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; } if ($mode eq "goto_purged") { @@ -311,4 +335,13 @@ } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } if ($status) { # delete the temporary image datafiles @@ -337,7 +370,11 @@ if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedskyfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -395,6 +432,5 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); + $config_file =~ s%^file://%%; if (!$config_file or ! -e $config_file) { @@ -404,4 +440,13 @@ } $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + $config_file =~ s%^file://%%; + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } } if ($status) { @@ -428,6 +473,10 @@ if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; } $command .= " -dbname $dbname" if defined $dbname; @@ -485,6 +534,5 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); + $config_file =~ s%^file://%%; if (!$config_file or ! -e $config_file) { @@ -494,4 +542,13 @@ } $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + $config_file =~ s%^file://%%; + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -529,10 +586,15 @@ if ($status) { my $command = "$difftool -diff_id $stage_id"; -# my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -545,5 +607,5 @@ } else { my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; -# my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; @@ -560,7 +622,1208 @@ } if ($stage eq 'fake') { - die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; + print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; + exit(0); + die "--stage_id required for stage fake\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'chiptool' + my $faketool = can_run('faketool') or die "Can't find faketool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + + # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for fakeRun $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); + addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$faketool -fake_id $stage_id -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedimfile"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -tocleanedimfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } + } + exit 0; + } -# fake : faketool : -pendingcleanupimfile (loop over imfiles) +# Detrend stages +if ($stage eq "detrend.process.imfile") { + + die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find chiptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $exp_id = $imfile->{exp_id}; + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.process.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; #silently abort. I need to fix this for propers + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $exp_id = $exp->{exp_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + # I can't find anything to put here + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + # since 'camera' has only a single imfile, we can just update the run + my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.stack.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + my $stacks; # Array reference of component files + my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $stacks = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my @files = (); + foreach my $stack (@{ $stacks }) { + # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. + my $path_base = $stack->{uri}; + my $iteration = $stack->{iteration}; + my $class_id = $stack->{class_id}; + + $path_base =~ s/\.fits$//; # That should do it? + + my $status = 1; + +# if ($mode eq "goto_cleaned") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); + +# $config_file =~ s%^file://%%; +# if (!$config_file or ! -e $config_file) { +# print STDERR "skipping cleanup for $stage $stage_id $path_base" . +# " because config file is missing\n"; +# $status = 0; +# } +# $config_file = 'file://' . $config_file; +# } +# elsif ($mode eq "goto_scrubbed") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); +# $config_file =~ s%^file://%%; +# if ($config_file and -e $config_file) { +# print STDERR "skipping scrubbed for $stage $stage_id $path_base" . +# " because config file is present\n"; +# $status = 0; +# } +# } + + if ($status) { + # delete the temporary image datafiles + # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. + # I understand that they all point to the same filename right now, but that may not be true in + # the future. + addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); + + + addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode +# addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); + } + + $status = &delete_files(\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -updatestacked -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -updatestacked -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updatestacked -data_state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.normstat.imfile") { + print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { +# my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + if ($status) { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR "WTF: $success TTT $error_code QQQ\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; + +} +if ($stage eq "detrend.norm.imfile") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); + + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); + addFilename (\@files, "PPIMAGE.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.norm.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.resid.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $iteration = $imfile->{iteration}; + my $exp_id = $imfile->{exp_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for $stage $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.resid.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; +# my $class_id = $exp->{class_id} + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} + die "ipp_cleanup.pl -stage $stage not yet implemented\n"; @@ -570,9 +1833,26 @@ my $files = shift; # reference to a list of files to unlink +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); + # this script is, of course, very dangerous. foreach my $file (@$files) { print STDERR "unlinking $file\n"; +# print TMPLOG "$stage $stage_id $file"; +# my $ff = $file; +# $ff =~ s%^file://%%; +# unless (-e $ff) { +# print TMPLOG "\t File not found\n"; +# } +# else { +# print TMPLOG "\n"; +# } + $ipprc->file_delete($file); } + +# flock(TMPLOG,8); +# close(TMPLOG); + return 1; } Index: trunk/ippScripts/scripts/stack_skycell.pl =================================================================== --- trunk/ippScripts/scripts/stack_skycell.pl (revision 25285) +++ trunk/ippScripts/scripts/stack_skycell.pl (revision 25299) @@ -65,4 +65,6 @@ my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$| = 1; +print "I've set up: $stack_id\n"; # XXX camera is not known here; cannot use filerules... @@ -107,4 +109,6 @@ &my_die("Stack list contains less than two elements", $stack_id, $PS_EXIT_SYS_ERROR) unless scalar @$files >= 2; + +print "I've loaded my inputs: $stack_id\n"; # Parse the list of input files to get the tesselation, skycell identifiers and camera @@ -146,4 +150,6 @@ } +print "I've configured everything: $stack_id\n"; + # Generate MDC file with the inputs my $tess_base = basename($tess_id); @@ -180,4 +186,6 @@ print $listFile "END\n\n"; } + +print "I've checked everything: $stack_id\n"; # Get the output filenames @@ -200,4 +208,5 @@ my $cmdflags; + # Perform stacking @@ -265,5 +274,6 @@ # &my_die("Couldn't find expected output file: $bin2Name", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name); } - + + print "I've stacked $listName: $stack_id\n"; } @@ -298,4 +308,6 @@ } +print "I've updated teh database: $stack_id\n"; + my $my_die_called = 0; @@ -372,4 +384,5 @@ } +print "I've reached the end of processing with a code of $?: $stack_id\n"; END { Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 25285) +++ trunk/ippTasks/Makefile.am (revision 25299) @@ -13,4 +13,5 @@ chip.pro \ camera.pro \ + addstar.pro \ fake.pro \ warp.pro \ Index: trunk/ippTasks/addstar.pro =================================================================== --- trunk/ippTasks/addstar.pro (revision 25299) +++ trunk/ippTasks/addstar.pro (revision 25299) @@ -0,0 +1,227 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + +book init addPendingExp + +macro addstar.status + book listbook addPendingExp +end + +macro addstar.reset + book init addPendingExp +end + +macro addstar.on + task addstar.exp.load + active true + end + task addstar.exp.run + active true + end +end + +macro addstar.off + task addstar.exp.load + active false + end + task addstar.exp.run + active false + end +end + +# this variable will cycle through the known database names +$addstar_DB = 0 +$addstar_revert_DB = 0 + +# select images ready for addstar analysis +# new entries are added to addPendingExp +# skip already-present entries +task addstar.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the addstar script on pending exposures +task addstar.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages addPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in addPendingExp (pantaskState == INIT) + book getpage addPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword addPendingExp $pageName pantaskState RUN + book getword addPendingExp $pageName camera -var CAMERA + book getword addPendingExp $pageName exp_tag -var EXP_TAG + book getword addPendingExp $pageName add_id -var ADD_ID + book getword addPendingExp $pageName cam_id -var CAM_ID + book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName dbname -var DBNAME + book getword addPendingExp $pageName reduction -var REDUCTION + book getword addPendingExp $pageName state -var RUN_STATE + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + sprintf camroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + stdout $LOGDIR/addstar.exp.log + stderr $LOGDIR/addstar.exp.log + + $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --outroot $outroot --camroot $camroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit addPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword addPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword addPendingExp $options:0 pantaskState TIMEOUT + end +end + +task addstar.revert + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB + $run = $run -dbname $DB:$addstar_revert_DB + $addstar_revert_DB ++ + if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: trunk/ippTasks/pantasks.pro =================================================================== --- trunk/ippTasks/pantasks.pro (revision 25285) +++ trunk/ippTasks/pantasks.pro (revision 25299) @@ -213,4 +213,5 @@ chip.on camera.on + addstar.on fake.on warp.on @@ -225,4 +226,5 @@ chip.off camera.off + addstar.off fake.off warp.off @@ -242,4 +244,5 @@ module chip.pro module camera.pro + module addstar.pro module fake.pro module warp.pro Index: trunk/ippTasks/summit.copy.pro =================================================================== --- trunk/ippTasks/summit.copy.pro (revision 25285) +++ trunk/ippTasks/summit.copy.pro (revision 25299) @@ -449,4 +449,5 @@ end if (("$MD5SUM" != "NULL") && ("$MD5SUM" != "0") && (not($COMPRESS))) +# && (($YEAR > 2008) || (("$YEAR" = "2007") && ($MONTH > 8)))) $run = $run --md5 $MD5SUM end Index: trunk/ippTools/doc/addstar_flow.txt =================================================================== --- trunk/ippTools/doc/addstar_flow.txt (revision 25299) +++ trunk/ippTools/doc/addstar_flow.txt (revision 25299) @@ -0,0 +1,34 @@ +### Set up the addRun row during the previous stage +#>> camtool.c @ addprocessedexpMode(pxConfig *config) +camtool -cam_id $cam_id -addprocessedexp ... + +### Catch that things now need to be added. This should check for magic_status as well, probably. +#>> add.pro / addtool.c / addtoolConfig.c +addtool -pendingsmf +#addMe METADATA +# add_id S64 5 +#END + +### Run the addstar script. +#>> addstar_run.pl +addstar_run.pl --add_id $add_id --camera $camera --camroot=$camroot --dvodb $DVODB + addtool -pendingsmf -add_id $add_id +#addMe METADATA +# add_id S64 5 +# workdir STR /path/to/workdir +# +#END + + ppConfigDump -camera $camera -recipe ADDSTAR $recipt_addstar -dump-recipe ADDSTAR - + addstar -D CAMERA $camdir -update -D CATDIR $dvodbReal $realFile + addtool -add_id $add_id -addprocessedsmf ... + addtool -add_id $add_id -updaterun -state full + +### Reverting +#>> addtool.c +addtool -revertprocessedsmf ... + + + + + Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 25285) +++ trunk/ippTools/share/Makefile.am (revision 25299) @@ -4,4 +4,8 @@ dist_pkgdata_DATA = \ + addtool_find_pendingexp.sql \ + addtool_queue_cam_id.sql \ + addtool_reset_faulted_runs.sql \ + addtool_revertprocessedexp.sql \ camtool_donecleanup.sql \ camtool_find_chip_id.sql \ Index: trunk/ippTools/share/addtool_find_pendingexp.sql =================================================================== --- trunk/ippTools/share/addtool_find_pendingexp.sql (revision 25299) +++ trunk/ippTools/share/addtool_find_pendingexp.sql (revision 25299) @@ -0,0 +1,26 @@ +SELECT + addRun.*, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM addRun +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN chipProcessedImfile + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id +LEFT JOIN addProcessedExp + USING(add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + camRun.state = 'full' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addMask.label IS NULL + Index: trunk/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- trunk/ippTools/share/addtool_queue_cam_id.sql (revision 25299) +++ trunk/ippTools/share/addtool_queue_cam_id.sql (revision 25299) @@ -0,0 +1,14 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- dvodb + 0 -- magicked + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: trunk/ippTools/share/addtool_reset_faulted_runs.sql =================================================================== --- trunk/ippTools/share/addtool_reset_faulted_runs.sql (revision 25299) +++ trunk/ippTools/share/addtool_reset_faulted_runs.sql (revision 25299) @@ -0,0 +1,8 @@ +UPDATE addRun, addProcessedExp, camRun, chipRun, rawExp +SET addRun.state = 'new' +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 Index: trunk/ippTools/share/addtool_revertprocessedexp.sql =================================================================== --- trunk/ippTools/share/addtool_revertprocessedexp.sql (revision 25299) +++ trunk/ippTools/share/addtool_revertprocessedexp.sql (revision 25299) @@ -0,0 +1,8 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun, camRun, chipRun, rawExp +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 Index: trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25285) +++ trunk/ippTools/share/dettool_pendingcleanup_normalizedimfile.sql (revision 25299) @@ -1,3 +1,3 @@ -SELECT +SELECT DISTINCT detNormalizedImfile.*, detRunSummary.data_state Index: trunk/ippTools/share/dettool_pendingcleanup_stacked.sql =================================================================== --- trunk/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25285) +++ trunk/ippTools/share/dettool_pendingcleanup_stacked.sql (revision 25299) @@ -1,3 +1,3 @@ -SELECT +SELECT DISTINCT detStackedImfile.*, detRunSummary.data_state Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 25285) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 25299) @@ -451,4 +451,38 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + magicked BIGINT, + PRIMARY KEY(add_id), + KEY(add_id), + KEY(cam_id), + KEY(state), + KEY(workdir_state), + KEY(label), + INDEX(add_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + n_stars INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE fakeRun ( fake_id BIGINT AUTO_INCREMENT, @@ -1086,5 +1120,6 @@ KEY(magic_id), KEY(label), - FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: trunk/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_drop_tables.sql (revision 25285) +++ trunk/ippTools/share/pxadmin_drop_tables.sql (revision 25299) @@ -19,4 +19,7 @@ DROP TABLE IF EXISTS detRun; DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS addRun; +DROP TABLE IF EXISTS addProcessedExp; +DROP TABLE IF EXISTS addMask; DROP TABLE IF EXISTS fakeRun; DROP TABLE IF EXISTS fakeProcessedImfile; Index: trunk/ippTools/src/Makefile.am =================================================================== --- trunk/ippTools/src/Makefile.am (revision 25285) +++ trunk/ippTools/src/Makefile.am (revision 25299) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + addtool \ caltool \ camtool \ @@ -28,4 +29,5 @@ pkginclude_HEADERS = \ + pxadd.h \ pxadmin.h \ pxcam.h \ @@ -42,4 +44,5 @@ noinst_HEADERS = \ + addtool.h \ caltool.h \ camtool.h \ @@ -68,4 +71,5 @@ libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) libpxtools_la_SOURCES = \ + pxadd.c \ pxcam.c \ pxchip.c \ @@ -96,4 +100,10 @@ pstamptool.c \ pstamptoolConfig.c + +addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +addtool_SOURCES = \ + addtool.c \ + addtoolConfig.c caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: trunk/ippTools/src/addtool.c =================================================================== --- trunk/ippTools/src/addtool.c (revision 25299) +++ trunk/ippTools/src/addtool.c (revision 25299) @@ -0,0 +1,1116 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = addtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(ADDTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(ADDTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(ADDTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(ADDTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(ADDTOOL_MODE_BLOCK, blockMode); + MODECASE(ADDTOOL_MODE_MASKED, maskedMode); + MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + + // find the cam_id of all the exposures that we want to queue up. + psString query = pxDataGet("addtool_find_cam_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + + // loop over our list of addRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + addRunRow *row = addRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + row->cam_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + "RECIPE", + dvodb ? dvodb : row->dvodb + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set addRun.state to state + if (!pxaddRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set addRun.label to label + if (!pxaddRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* static bool pendingimfileMode(pxConfig *config) */ +/* { */ +/* PS_ASSERT_PTR_NON_NULL(config, false); */ + +/* psMetadata *where = psMetadataAlloc(); */ +/* pxaddGetSearchArgs (config, where); */ +/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ +/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ + +/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ + +/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ +/* if (!query) { */ +/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ +/* return false; */ +/* } */ + +/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ +/* if (psListLength(where->list)) { */ +/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ +/* psStringAppend(&query, " AND %s", whereClause); */ +/* psFree(whereClause); */ +/* } */ +/* psFree(where); */ + +/* if (!p_psDBRunQuery(config->dbh, query)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* psFree(query); */ +/* return false; */ +/* } */ +/* psFree(query); */ + +/* psArray *output = p_psDBFetchResult(config->dbh); */ +/* if (!output) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* if (!psArrayLength(output)) { */ +/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ +/* psFree(output); */ +/* return true; */ +/* } */ + +/* // negate simple so the default is true */ +/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ +/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ +/* psFree(output); */ +/* return false; */ +/* } */ + +/* psFree(output); */ + +/* return true; */ +/* } */ + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + addRunRow *pendingRow = addRunObjectFromMetadata(output->data[0]); + psFree(output); + addProcessedExpRow *row = addProcessedExpRowAlloc( + pendingRow->add_id, + dtime_addstar, + n_stars, + path_base, + 0 + ); + + if (!addProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set addRun.state = 'full' + if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if fault != 0 + // Also, we can run fake even if tess_id is not defined +/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ +/* psFree(row); */ +/* psFree(pendingRow); */ +/* if (!psDBCommit(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* return true; */ +/* } */ + psFree(row); + // else continue on... + +/* if (!pxfakeQueueByAddID(config, */ +/* pendingRow->add_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->expgroup, */ +/* pendingRow->dvodb, */ +/* pendingRow->tess_id, */ +/* pendingRow->end_stage */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; + } + + psString query = pxDataGet("addtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + // we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); + } + psFree(where); + + // order by add_id so that the postage stamp parser can easliy find the 'latest' astrometry + psStringAppend(&query, " ORDER BY add_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); +/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("addtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); +/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!addMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM addMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM addMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + + psString query = pxDataGet("addtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (add_id) { + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + + ExportTable tables [] = { + {"addRun", "addtool_export_run.sql"}, + {"addProcessedExp", "addtool_export_processed_exp.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "addRun")) { + if (!pxSetStateCleaned("addRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + fprintf (stdout, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); + + psMetadataItem *item = psMetadataLookup (input, "addRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); + addRunInsertObject (config->dbh, addRun); + + // fprintf (stdout, "---- add run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + item = psMetadataLookup (input, "addProcessedExp"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + for (int i = 0; i < item->data.list->n; i++) { + psMetadataItem *entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); + addProcessedExpInsertObject (config->dbh, addProcessedExp); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + + return true; +} Index: trunk/ippTools/src/addtool.h =================================================================== --- trunk/ippTools/src/addtool.h (revision 25299) +++ trunk/ippTools/src/addtool.h (revision 25299) @@ -0,0 +1,46 @@ +/* + * addtool.h + * + * Copyright (C) 2006 Joshua Hoblitt, Christopher Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ADDTOOL_H +#define ADDTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + ADDTOOL_MODE_NONE = 0x0, + ADDTOOL_MODE_DEFINEBYQUERY, + ADDTOOL_MODE_UPDATERUN, + ADDTOOL_MODE_PENDINGEXP, + ADDTOOL_MODE_ADDPROCESSEDEXP, + ADDTOOL_MODE_PROCESSEDEXP, + ADDTOOL_MODE_REVERTPROCESSEDEXP, + ADDTOOL_MODE_UPDATEPROCESSEDEXP, + ADDTOOL_MODE_BLOCK, + ADDTOOL_MODE_MASKED, + ADDTOOL_MODE_UNBLOCK, + ADDTOOL_MODE_PENDINGCLEANUPRUN, + ADDTOOL_MODE_PENDINGCLEANUPEXP, + ADDTOOL_MODE_DONECLEANUP, + ADDTOOL_MODE_EXPORTRUN, + ADDTOOL_MODE_IMPORTRUN +} addtoolMode; + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // ADDTOOL_H Index: trunk/ippTools/src/addtoolConfig.c =================================================================== --- trunk/ippTools/src/addtoolConfig.c (revision 25299) +++ trunk/ippTools/src/addtoolConfig.c (revision 25299) @@ -0,0 +1,212 @@ +/* + * addtoolConfig.c + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ + + // -revertprocessedexp + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exp", ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exps", ADDTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exp", ADDTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: trunk/ippTools/src/camtool.c =================================================================== --- trunk/ippTools/src/camtool.c (revision 25285) +++ trunk/ippTools/src/camtool.c (revision 25299) @@ -656,4 +656,21 @@ return false; } + + if (!pxaddQueueByCamID(config, + pendingRow->cam_id, + pendingRow->workdir, + pendingRow->label, + pendingRow->reduction, + pendingRow->dvodb + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); + psFree(pendingRow); + return false; + } + psFree(pendingRow); Index: trunk/ippTools/src/dettoolConfig.c =================================================================== --- trunk/ippTools/src/dettoolConfig.c (revision 25285) +++ trunk/ippTools/src/dettoolConfig.c (revision 25299) @@ -70,4 +70,5 @@ psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebytagArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddTime(definebytagArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -138,4 +139,13 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_min", 0, "define min moon angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_angle_max", 0, "define max moon angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_min", 0, "define min moon alt", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_alt_max", 0, "define max moon alt", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_min", 0, "define min moon phase", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class for processing", NULL); Index: trunk/ippTools/src/pxadd.c =================================================================== --- trunk/ippTools/src/pxadd.c (revision 25299) +++ trunk/ippTools/src/pxadd.c (revision 25299) @@ -0,0 +1,262 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" + +bool pxaddSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ + + return true; +} + +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); +/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ +/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ +/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ + + return true; +} + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + + +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE addRun SET addRun.label = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, label, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + +// Need to think more about this to see what we want it to do. BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("addtool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + // fprintf(stderr,"%s",query); + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQueryF(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + dvodb ? dvodb : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: trunk/ippTools/src/pxadd.h =================================================================== --- trunk/ippTools/src/pxadd.h (revision 25299) +++ trunk/ippTools/src/pxadd.h (revision 25299) @@ -0,0 +1,44 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADD_H +#define PXADD_H 1 + +#include + +#include "pxtools.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxaddSetSearchArgs (psMetadata *md); +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); + +// Likely BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb); + + +#endif // PXADD_H Index: trunk/ippTools/src/pxtools.h =================================================================== --- trunk/ippTools/src/pxtools.h (revision 25285) +++ trunk/ippTools/src/pxtools.h (revision 25299) @@ -37,4 +37,5 @@ #include "pxtoolsErrorCodes.h" +#include "pxadd.h" #include "pxcam.h" #include "pxchip.h" Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 25285) +++ trunk/ippTools/src/regtool.c (revision 25299) @@ -331,4 +331,5 @@ PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); psString query = pxDataGet("regtool_processedimfile.sql"); @@ -352,4 +353,9 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + // add the ORDER BY statement if desired + if (ordered_by_date) { + psStringAppend(&query, " ORDER BY dateobs"); } @@ -923,6 +929,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "object", "=="); - PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); - PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); + PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<"); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); Index: trunk/ippTools/src/warptool.c =================================================================== --- trunk/ippTools/src/warptool.c (revision 25285) +++ trunk/ippTools/src/warptool.c (revision 25299) @@ -234,6 +234,6 @@ PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); - PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); - PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">="); + PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<"); PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); Index: trunk/ippconfig/gpc1/format_20090220.config =================================================================== --- trunk/ippconfig/gpc1/format_20090220.config (revision 25285) +++ trunk/ippconfig/gpc1/format_20090220.config (revision 25299) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - FPA.TEMP STR DETTEM + # FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -501,4 +501,6 @@ # How to translation PS concepts into database lookups DATABASE METADATA + # this rule is fragile : does not match the camera + FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: trunk/ippconfig/recipes/masks.16bit.config =================================================================== --- trunk/ippconfig/recipes/masks.16bit.config (revision 25285) +++ trunk/ippconfig/recipes/masks.16bit.config (revision 25299) @@ -20,4 +20,5 @@ # Mask values which represent non-astronomical structures +BURNTOOL U16 0x0080 # Pixel may contain uncorrected streak. CR U16 0x0100 # Pixel contains a cosmic ray SPIKE U16 0x0200 # Pixel contains a diffraction spike Index: trunk/ippconfig/recipes/ppImage.config =================================================================== --- trunk/ippconfig/recipes/ppImage.config (revision 25285) +++ trunk/ippconfig/recipes/ppImage.config (revision 25299) @@ -16,4 +16,5 @@ MASK.SATURATED BOOL TRUE # Mask the saturated pixels MASK.LOW BOOL TRUE # Mask pixels below valid range +MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails VARIANCE.BUILD BOOL FALSE # Build internal variance image PATTERN BOOL FALSE # Fit and remove pattern noise? @@ -54,4 +55,6 @@ ## if we use multithreaded detrending, detrend this number of rows per thread SCAN.ROWS S32 100 + +BURNTOOL.TRAILS U16 0x01 # Non-linearity correction Index: trunk/ippconfig/recipes/psastro.config =================================================================== --- trunk/ippconfig/recipes/psastro.config (revision 25285) +++ trunk/ippconfig/recipes/psastro.config (revision 25299) @@ -3,4 +3,7 @@ PSASTRO.ONLY.REFSTARS BOOL FALSE # skip all but refstar matches PSASTRO.SAVE.REFMATCH BOOL FALSE # save refstar matches as table in output smf file + +# select which WCS style to use on output images. +PSASTRO.WCS.USECDKEYS BOOL FALSE # perform single-chip astrometry? Index: trunk/ppImage/src/Makefile.am =================================================================== --- trunk/ppImage/src/Makefile.am (revision 25285) +++ trunk/ppImage/src/Makefile.am (revision 25299) @@ -54,4 +54,5 @@ ppImageDefineFile.c \ ppImageSetMaskBits.c \ + ppImageBurntoolMask.c \ ppImageParityFlip.c \ ppImageCheckCTE.c \ Index: trunk/ppImage/src/ppImage.h =================================================================== --- trunk/ppImage/src/ppImage.h (revision 25285) +++ trunk/ppImage/src/ppImage.h (revision 25299) @@ -26,4 +26,5 @@ bool doMaskBuild; // Build internal mask bool doVarianceBuild; // Build internal variance map + bool doMaskBurntool; // mask potential burntool trails bool doMaskSat; // mask saturated pixels bool doMaskLow; // mask low pixels @@ -74,5 +75,5 @@ psImageMaskType darkMask; // Mask value to give bad dark pixels psImageMaskType blankMask; // Mask value to give blank pixels - + psImageMaskType burntoolMask; // Suspect pixels that fall where a burntool trail is expected. // non-linear correction parameters psDataType nonLinearType; @@ -82,5 +83,5 @@ // options for the analysis pmOverscanOptions *overscan; // Overscan options - + int burntoolTrails; // binning parameters int xBin1; // x-binning, scale 1 @@ -156,4 +157,6 @@ bool ppImageCheckCTE(pmConfig *config, ppImageOptions *options, pmFPAview *view); +bool ppImageBurntoolMask(pmConfig *config, ppImageOptions *options, pmFPAview *view, pmReadout *mask); + // Record which detrend file was used for the detrending bool ppImageDetrendRecord( Index: trunk/ppImage/src/ppImageBurntoolMask.c =================================================================== --- trunk/ppImage/src/ppImageBurntoolMask.c (revision 25299) +++ trunk/ppImage/src/ppImageBurntoolMask.c (revision 25299) @@ -0,0 +1,100 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +/* #define PPIMAGE_BURNTOOL_DEBUG 1 */ + +#include "ppImage.h" + +bool ppImageBurntoolMask (pmConfig *config, ppImageOptions *options, pmFPAview *view,pmReadout *mask) { + bool status = true; + int burntool_cell; + /* Find input filename */ + pmFPAfile *inputFile = psMetadataLookupPtr(&status , config->files, "PPIMAGE.INPUT"); + if (!status) { + psError(PS_ERR_IO,false, "Unable to identify inputFile"); + return(false); + } + psFits *fits = inputFile->fits; + + /* Read input header, and find the burntool data table. */ + if (!psFitsMoveExtName(fits,"burntool_areas")) { + psError(PS_ERR_IO,false, "Unable to find extension burntool_areas"); + return(false); + } + long Nrows = psFitsTableSize(fits); + + long row = 0; + + psLogMsg ("ppImageBurntoolMask", 4, "Inside burntool mask %ld", Nrows); + + /* Redirects and Memory juggling. */ + view->readout = 0; + psImage *image = mask->mask; + + + /* Set the maskValue from the recipes. */ + psImageMaskType maskValue = options->burntoolMask; +#ifdef PPIMAGE_BURNTOOL_DEBUG + psLogMsg("ppImageBurntoolMask", 4, "Status: %ld %d\n",Nrows,maskValue); +#endif + + burntool_cell = view->cell; + burntool_cell = (view->cell % 8) * 8 + (view->cell - (view->cell % 8)) / 8; + psLogMsg("ppImageBurntoolMask", 4, "Cell mapping: %d %d %d\n",view->cell,burntool_cell,-1); + for (row = 0; row < Nrows; row++) { + psMetadata *rowMD = psFitsReadTableRow(fits,row); + + if (psMetadataLookupS32(&status,rowMD,"cell") == burntool_cell) { + if (((options->burntoolTrails & 0x01)&&(psMetadataLookupS32(&status,rowMD,"nfit") == 0))|| + ((options->burntoolTrails & 0x02)&&(psMetadataLookupS32(&status,rowMD,"up") == 1))|| + ((options->burntoolTrails & 0x04)&&(psMetadataLookupS32(&status,rowMD,"up") == 0))) { + /* If the fit fails, burntool reports zero here. This + signifies that it expected to see a trail (else why + fit) but did not find it when it attempted to + correct. */ +#ifdef PPIMAGE_BURNTOOL_DEBUG + psLogMsg ("ppImageBurntoolMask", 4, "Masking! %d (%d %d %d) %d %d", + psMetadataLookupS32(&status,rowMD,"cell"), + ((options->burntoolTrails & 0x0001)&&(psMetadataLookupS32(&status,rowMD,"nfit") == 0)), + ((options->burntoolTrails & 0x02)&&(psMetadataLookupS32(&status,rowMD,"up") == 1)), + ((options->burntoolTrails & 0x04)&&(psMetadataLookupS32(&status,rowMD,"up") == 0)), + options->burntoolTrails, + maskValue + ); +#endif + for (int i = psMetadataLookupS32(&status,rowMD,"sxfit"); + i <= psMetadataLookupS32(&status,rowMD,"exfit"); + i++) { + + if (psMetadataLookupS32(&status,rowMD,"up") == 0) { + for (int j = 0; j <= psMetadataLookupS32(&status,rowMD,"y1p"); j++) { +/* #ifdef PPIMAGE_BURNTOOL_DEBUG */ +/* psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", */ +/* i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); */ +/* #endif */ + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + else { + for (int j = psMetadataLookupS32(&status,rowMD,"y1m"); j < image->numRows ; j++) { +/* #ifdef PPIMAGE_BURNTOOL_DEBUG */ +/* psLogMsg("ppImageBurntoolMask", 4, "Noisy!: %d %d %d %d\n", */ +/* i,j,image->data.PS_TYPE_IMAGE_MASK_DATA[j][i],maskValue); */ +/* #endif */ + image->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskValue; + } + } + } + + } + } + psFree(rowMD); + } + + return(status); +} + + + + Index: trunk/ppImage/src/ppImageDetrendReadout.c =================================================================== --- trunk/ppImage/src/ppImageDetrendReadout.c (revision 25285) +++ trunk/ppImage/src/ppImageDetrendReadout.c (revision 25299) @@ -26,4 +26,8 @@ pmMaskBadPixels(input, mask, options->maskValue); } + if (options->doMaskBurntool) { + ppImageBurntoolMask(config,options,view,input); + } + # if 0 Index: trunk/ppImage/src/ppImageOptions.c =================================================================== --- trunk/ppImage/src/ppImageOptions.c (revision 25285) +++ trunk/ppImage/src/ppImageOptions.c (revision 25299) @@ -21,4 +21,5 @@ options->doMaskSat = false; // mask saturated pixels options->doMaskLow = false; // mask low pixels + options->doMaskBurntool = false; // mask potential burntool trails options->doVarianceBuild = false; // Build internal variance options->doMask = false; // Mask bad pixels @@ -64,5 +65,6 @@ options->blankMask = 0x00; // Blank (no data, cell gap) pixels (supplied to pmChipMosaic, pmFPAMosaic) options->markValue = 0x00; // A safe bit for internal marking - + options->burntoolMask = 0x00; // Suspect pixels that fall where a burntool trail is expected. + options->burntoolTrails = 0x00; // Which types of burntool areas to mask. // crosstalk options options->doCrosstalkMeasure = false; // measure crosstalk @@ -219,4 +221,5 @@ options->doMaskSat = psMetadataLookupBool(NULL, recipe, "MASK.SATURATED"); options->doMaskLow = psMetadataLookupBool(NULL, recipe, "MASK.LOW"); + options->doMaskBurntool = psMetadataLookupBool(NULL, recipe, "MASK.BURNTOOL"); options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD"); @@ -245,13 +248,19 @@ options->applyParity = psMetadataLookupBool(NULL, recipe, "APPLY.CELL.PARITY"); + options->burntoolTrails = psMetadataLookupU16(&status, recipe, "BURNTOOL.TRAILS"); + if (!status) { + psWarning("BURNTOOL.TRAILS not found in recipe: setting to default value.\n"); + } + // binned image options options->xBin1 = psMetadataLookupS32(&status, recipe, "BIN1.XBIN"); if (!status) { psWarning("BIN1.XBIN not found in recipe: setting to default value.\n"); + options->xBin1 = 4; } options->yBin1 = psMetadataLookupS32(&status, recipe, "BIN1.YBIN"); if (!status) { psWarning("BIN1.YBIN not found in recipe: setting to default value.\n"); - options->yBin1 = 16; + options->yBin1 = 4; } Index: trunk/ppImage/src/ppImageSetMaskBits.c =================================================================== --- trunk/ppImage/src/ppImageSetMaskBits.c (revision 25285) +++ trunk/ppImage/src/ppImageSetMaskBits.c (revision 25299) @@ -38,4 +38,8 @@ psAssert (options->lowMask, "low mask not set"); + // mask for suspect regions due to burntool + options->burntoolMask = pmConfigMaskGet("BURNTOOL",config); + psAssert (options->burntoolMask, "burntool mask not set"); + // save MASK and MARK on the PSPHOT recipe psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSPHOT_RECIPE); Index: trunk/psModules/src/astrom/pmAstrometryWCS.c =================================================================== --- trunk/psModules/src/astrom/pmAstrometryWCS.c (revision 25285) +++ trunk/psModules/src/astrom/pmAstrometryWCS.c (revision 25299) @@ -289,4 +289,6 @@ // test the CDELTi varient if (pcKeys) { + wcs->wcsCDkeys = 0; + wcs->cdelt1 = psMetadataLookupF64 (&status, header, "CDELT1"); wcs->cdelt2 = psMetadataLookupF64 (&status, header, "CDELT2"); @@ -334,4 +336,6 @@ // test the CDi_j varient if (cdKeys) { + wcs->wcsCDkeys = 1; + wcs->trans->x->coeff[1][0] = psMetadataLookupF64 (&status, header, "CD1_1"); // == PC1_1 wcs->trans->x->coeff[0][1] = psMetadataLookupF64 (&status, header, "CD1_2"); // == PC1_2 @@ -375,42 +379,59 @@ // XXX make it optional to write out CDi_j terms, or other versions // apply CDELT1,2 (degrees / pixel) to yield PCi,j terms of order unity - double cdelt1 = wcs->cdelt1; - double cdelt2 = wcs->cdelt2; - psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT1", PS_META_REPLACE, "", cdelt1); - psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT2", PS_META_REPLACE, "", cdelt2); - - // test the PC00i00j varient: - psMetadataAddF64 (header, PS_LIST_TAIL, "PC001001", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0] / cdelt1); // == PC1_1 - psMetadataAddF64 (header, PS_LIST_TAIL, "PC001002", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1] / cdelt2); // == PC1_2 - psMetadataAddF64 (header, PS_LIST_TAIL, "PC002001", PS_META_REPLACE, "", wcs->trans->y->coeff[1][0] / cdelt1); // == PC2_1 - psMetadataAddF64 (header, PS_LIST_TAIL, "PC002002", PS_META_REPLACE, "", wcs->trans->y->coeff[0][1] / cdelt2); // == PC2_2 - - // Elixir-style polynomial terms - // XXX currently, Elixir/DVO cannot accept mixed orders - // XXX need to respect the masks - // XXX is wcs->cdelt1,2 always consistent? - int fitOrder = wcs->trans->x->nX; - if (fitOrder > 1) { + + if (!(wcs->wcsCDkeys)) { + + double cdelt1 = wcs->cdelt1; + double cdelt2 = wcs->cdelt2; + psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT1", PS_META_REPLACE, "", cdelt1); + psMetadataAddF64 (header, PS_LIST_TAIL, "CDELT2", PS_META_REPLACE, "", cdelt2); + + // test the PC00i00j varient: + psMetadataAddF64 (header, PS_LIST_TAIL, "PC001001", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0] / cdelt1); // == PC1_1 + psMetadataAddF64 (header, PS_LIST_TAIL, "PC001002", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1] / cdelt2); // == PC1_2 + psMetadataAddF64 (header, PS_LIST_TAIL, "PC002001", PS_META_REPLACE, "", wcs->trans->y->coeff[1][0] / cdelt1); // == PC2_1 + psMetadataAddF64 (header, PS_LIST_TAIL, "PC002002", PS_META_REPLACE, "", wcs->trans->y->coeff[0][1] / cdelt2); // == PC2_2 + + // Elixir-style polynomial terms + // XXX currently, Elixir/DVO cannot accept mixed orders + // XXX need to respect the masks + // XXX is wcs->cdelt1,2 always consistent? + int fitOrder = wcs->trans->x->nX; + if (fitOrder > 1) { for (int i = 0; i <= fitOrder; i++) { - for (int j = 0; j <= fitOrder; j++) { - if (i + j < 2) - continue; - if (i + j > fitOrder) - continue; - sprintf (name, "PCA1X%1dY%1d", i, j); - psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->x->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j)); - sprintf (name, "PCA2X%1dY%1d", i, j); - psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->y->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j)); - } + for (int j = 0; j <= fitOrder; j++) { + if (i + j < 2) + continue; + if (i + j > fitOrder) + continue; + sprintf (name, "PCA1X%1dY%1d", i, j); + psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->x->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j)); + sprintf (name, "PCA2X%1dY%1d", i, j); + psMetadataAddF64 (header, PS_LIST_TAIL, name, PS_META_REPLACE, "", wcs->trans->y->coeff[i][j] / pow(cdelt1, i) / pow(cdelt2, j)); + } } psMetadataAddS32 (header, PS_LIST_TAIL, "NPLYTERM", PS_META_REPLACE, "", fitOrder); - } - - // remove any existing 'CDi_j style' wcs keywords - if (psMetadataLookup(header, "CD1_1")) { + } + + // remove any existing 'CDi_j style' wcs keywords + if (psMetadataLookup(header, "CD1_1")) { psMetadataRemoveKey(header, "CD1_1"); psMetadataRemoveKey(header, "CD1_2"); psMetadataRemoveKey(header, "CD2_1"); psMetadataRemoveKey(header, "CD2_2"); + } + } + if (wcs->wcsCDkeys) { + psMetadataAddF64 (header, PS_LIST_TAIL, "CD1_1", PS_META_REPLACE, "", wcs->trans->x->coeff[1][0]); + psMetadataAddF64 (header, PS_LIST_TAIL, "CD1_2", PS_META_REPLACE, "", wcs->trans->x->coeff[0][1]); + psMetadataAddF64 (header, PS_LIST_TAIL, "CD2_1", PS_META_REPLACE, "", wcs->trans->y->coeff[1][0]); + psMetadataAddF64 (header, PS_LIST_TAIL, "CD2_2", PS_META_REPLACE, "", wcs->trans->y->coeff[0][1]); + + if (psMetadataLookup(header, "PC001001")) { + psMetadataRemoveKey(header, "PC001001"); + psMetadataRemoveKey(header, "PC001002"); + psMetadataRemoveKey(header, "PC002001"); + psMetadataRemoveKey(header, "PC002002"); + } } @@ -535,4 +556,6 @@ fpa->toSky->R -= 2.0*M_PI; + fpa->wcsCDkeys = wcs->wcsCDkeys; + psTrace ("psastro", 5, "toFPA: %f %f (%f,%f),(%f,%f)\n", chip->toFPA->x->coeff[0][0], chip->toFPA->y->coeff[0][0], @@ -682,4 +705,5 @@ wcs->cdelt2 = hypot (wcs->trans->y->coeff[1][0], wcs->trans->y->coeff[0][1]); + wcs->wcsCDkeys = fpa->wcsCDkeys; psFree (toTPA); @@ -922,4 +946,5 @@ wcs->trans = psPlaneTransformAlloc (nXorder, nYorder); wcs->toSky = NULL; + wcs->wcsCDkeys = 0; memset (wcs->ctype1, 0, PM_ASTROM_WCS_TYPE_SIZE); Index: trunk/psModules/src/astrom/pmAstrometryWCS.h =================================================================== --- trunk/psModules/src/astrom/pmAstrometryWCS.h (revision 25285) +++ trunk/psModules/src/astrom/pmAstrometryWCS.h (revision 25299) @@ -23,4 +23,5 @@ double crpix1, crpix2; double cdelt1, cdelt2; + bool wcsCDkeys; psProjection *toSky; psPlaneTransform *trans; Index: trunk/psModules/src/camera/pmFPA.h =================================================================== --- trunk/psModules/src/camera/pmFPA.h (revision 25285) +++ trunk/psModules/src/camera/pmFPA.h (revision 25299) @@ -48,4 +48,5 @@ psPlaneTransform *toTPA; ///< Transformation from focal plane to tangent plane, or NULL psProjection *toSky; ///< Projection from tangent plane to sky, or NULL + bool wcsCDkeys; // Information psMetadata *concepts; ///< FPA-level concepts Index: trunk/psastro/src/psastroChipAstrom.c =================================================================== --- trunk/psastro/src/psastroChipAstrom.c (revision 25285) +++ trunk/psastro/src/psastroChipAstrom.c (revision 25299) @@ -97,5 +97,6 @@ // write the elapsed time here; this will be updated in psastroMosaicAstrometry, if called psMetadataAddF32 (updates, PS_LIST_TAIL, "DT_ASTR", PS_META_REPLACE, "elapsed psastro time", psTimerMark ("psastroAnalysis")); - + + fpa->wcsCDkeys = psMetadataLookupBool(&status, recipe , "PSASTRO.WCS.USECDKEYS"); pmAstromWriteWCS (updates, fpa, chip, NONLIN_TOL); Index: trunk/tools/ipp_apply_burntool.pl =================================================================== --- trunk/tools/ipp_apply_burntool.pl (revision 25285) +++ trunk/tools/ipp_apply_burntool.pl (revision 25299) @@ -172,7 +172,7 @@ if ($RAWTABLES) { - $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN); + $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt persist=t", $REALRUN); } else { - $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN); + $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt persist=t", $REALRUN); } if ($status) {