Index: trunk/dbconfig/changes.txt =================================================================== --- trunk/dbconfig/changes.txt (revision 27148) +++ trunk/dbconfig/changes.txt (revision 27149) @@ -1527,2 +1527,13 @@ ALTER TABLE publishClient ADD COLUMN magicked TINYINT DEFAULT 1 AFTER stage; +ALTER TABLE rawExp ADD COLUMN state VARCHAR(64) AFTER workdir, ADD KEY(state); +UPDATE rawExp SET state = 'full'; + +ALTER TABLE rawImfile ADD COLUMN data_state VARCHAR(64) AFTER uri, ADD KEY(data_state); +UPDATE rawImfile SET data_state = 'full'; + +-- Not updating schema_version to not break distribution bundles. +-- UPDATE dbversion set schema_version = '1.1.65', updated= CURRENT_TIMESTAMP(); + +-- Version 1.1.66 + Index: trunk/dbconfig/raw.md =================================================================== --- trunk/dbconfig/raw.md (revision 27148) +++ trunk/dbconfig/raw.md (revision 27149) @@ -10,4 +10,5 @@ filelevel STR 64 workdir STR 255 # destination for output files + state STR 64 # Key reduction STR 64 # Reduction class dvodb STR 255 @@ -77,4 +78,5 @@ class_id STR 64 # Primary Key uri STR 255 + data_state STR 64 # Key exp_type STR 64 # This field is used to set the per exp filelevel. Thus the values for all of Index: trunk/ippScripts/Build.PL =================================================================== --- trunk/ippScripts/Build.PL (revision 27148) +++ trunk/ippScripts/Build.PL (revision 27149) @@ -101,4 +101,5 @@ scripts/whichimage scripts/automate_stacks.pl + scripts/lossy_compress_imfile.pl scripts/ipp_apply_burntool.pl scripts/make_burntool_pcontrol.pl Index: trunk/ippScripts/MANIFEST =================================================================== --- trunk/ippScripts/MANIFEST (revision 27148) +++ trunk/ippScripts/MANIFEST (revision 27149) @@ -37,4 +37,5 @@ scripts/ds9_cmf_regions.pl scripts/automate_stacks.pl +scripts/lossy_compress_imfile.pl scripts/ipp_apply_burntool.pl scripts/make_burntool_pcontrol.pl Index: trunk/ippScripts/scripts/automate_stacks.pl =================================================================== --- trunk/ippScripts/scripts/automate_stacks.pl (revision 27148) +++ trunk/ippScripts/scripts/automate_stacks.pl (revision 27149) @@ -426,9 +426,13 @@ my ($trash,$start,$end); ($trash,$trash,$start,$end) = (split /\s+/,$row); - $metadata_out{"btBegin${N}"} = $start; - $metadata_out{"btEnd${N}"} = $end; - $metadata_out{"btCounter${N}"} = 0; - $N++; - print STDERR "define_burntool: $row\n"; + + for (my $class_counter = 0; $class_counter < 60; $class_counter++) { + $metadata_out{"bt${N}Begin"} = $start; + $metadata_out{"bt${N}End"} = $end; + $metadata_out{"bt${N}Class"} = $class_counter; + $metadata_out{"bt${N}Status"} = 'NEW'; + $N++; + } + print STDERR "define_burntool: $row\n"; } $metadata_out{btN} = $N - 1; @@ -793,5 +797,5 @@ print STDOUT "autoStack METADATA\n"; print STDOUT " date STR $date\n"; - foreach my $k (keys %metadata_out) { + foreach my $k (sort(keys %metadata_out)) { print STDOUT " $k STR $metadata_out{$k}\n"; } Index: trunk/ippScripts/scripts/lossy_compress_imfile.pl =================================================================== --- trunk/ippScripts/scripts/lossy_compress_imfile.pl (revision 27149) +++ trunk/ippScripts/scripts/lossy_compress_imfile.pl (revision 27149) @@ -0,0 +1,204 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + + +use Digest::MD5::File qw( file_md5_hex ); +use File::Basename qw( basename); +use IPC::Run; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $regtool = can_run( 'regtool' ) or (warn "Can't find regtool" and $missing_tools = 1); +my $fpack = can_run( 'fpack' ) or (warn "Can't find fpack" and $missing_tools = 1); +my ($exp_id, $class_id, $exp_name, $uri, $bytes, $md5sum, $dbname, $state, $verbose, $no_update, $no_op, $logfile); +my ($camera); +GetOptions( + 'exp_id|e=s' => \$exp_id, + 'class_id|i=s' => \$class_id, + 'exp_name|n=s' => \$exp_name, + 'camera=s' => \$camera, + 'uri|u=s' => \$uri, + 'bytes=s' => \$bytes, + 'md5sum=s' => \$md5sum, + 'dbname|d=s' => \$dbname, + 'state=s' => \$state, + 'verbose' => \$verbose, + 'no-update' => \$no_update, + 'no-op' => \$no_op, + 'logfile=s' => \$logfile, + ) or pod2usage ( 2 ); + + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --exp_id --class_id --exp_name --uri --camera --state", -exitval => 3) unless + defined $exp_id and + defined $class_id and + defined $exp_name and + defined $uri and + defined $camera and + defined $state; + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $exp_name, $class_id, $uri, $PS_EXIT_CONFIG_ERROR ); +if (defined($logfile)) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $exp_name, $class_id, $uri, $PS_EXIT_SYS_ERROR ); +} + +my $RECIPE = "LOSSYCOMP"; + +if ($missing_tools) { + warn ("Can't find required tools"); + exit($PS_EXIT_CONFIG_ERROR); +} + + + +my $now_time = localtime(); +printf STDERR "\nstarting compression: %s\n", $now_time if $verbose; + + +# If goto_compress +# - verify disk file matches +# - create compressed version +# - update database +# + +if ($state eq 'goto_compressed') { + # Find the actual filename for this run: + &my_die("Couldn't find input file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + unless ($ipprc->file_exists($uri)); + my $uriReal = $ipprc->file_resolve( $uri ); + + # Create a compressed version: + my $compUri = $uri . ".fz"; + &my_die("Output compressed file already exists: $compUri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + if ($ipprc->file_exists($compUri)); + my $compReal= $ipprc->file_resolve( $compUri, 'create'); + my $compress_command = "$fpack -h -s 8 -S $uriReal > $compReal"; + print STDERR "$compReal $uriReal $compress_command\n"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $compress_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to compress file: $uri -> $compUri: $error_code", $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } + my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state compressed"; + $database_command .= " -dbname $dbname" if defined $dbname; + + if ($no_update || $no_op) { + print STDERR "NOUPDATE: $database_command\n"; + } + else { + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $database_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to update database file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } + } + exit(0); +} +# If goto_lossy +# - verify disk file matches +# - verify compressed version && create if not found +# - swap nebulous keys +# - update gpc1 database with bytes/md5sum and set state +# - remove original version + +if ($state eq 'goto_lossy') { + # Confirm we have both files + &my_die("Couldn't find original file: $uri\n", $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + unless($ipprc->file_exists($uri)); + my $compUri = $uri . ".fz"; + unless($ipprc->file_exists($compUri)) { + &my_die("Couldn't find compressed version of the file: $compUri\n", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + # If that die is removed, this will compress things as well. + &my_die("Output compressed file already exists: $compUri\n", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR) + if ($ipprc->file_exists($compUri)); + my $compReal= $ipprc->file_resolve( $compUri, 'create'); + my $compress_command = "$fpack -h -s 8 -S $uriReal > $compReal"; + print STDERR "$compReal $uriReal $compress_command\n"; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $compress_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to compress file: $uri -> $compUri: $error_code", $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } + } + + # Swap keys + my $compReal = $ipprc->file_resolve( $compUri ); + my ($new_bytes, $new_md5) = (-s $compReal,file_md5_hex($compReal)); + my $neb = $ipprc->nebulous(); + + unless ($no_op) { + $neb->replicate($compUri); + $neb->swap($uri,$compUri) or + &my_die("Nebulous swap failed between $uri and $compUri", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + } + # Update database + + my $database_command = "$regtool -updateprocessedimfile -exp_id $exp_id -class_id $class_id -set_state lossy "; + $database_command .= " -set_bytes $new_bytes -set_md5sum $new_md5 "; + + $database_command .= " -dbname $dbname" if defined $dbname; + + if ($no_update || $no_op) { + print STDERR "NOUPDATE: $database_command\n"; + } + else { + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = + run(command => $database_command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + $neb->swap($uri,$compUri) or + &my_die("DB update failed and Nebulous swap-back failed between $uri and $compUri", + $exp_id,$exp_name,$class_id,$uri,$PS_EXIT_SYS_ERROR); + &my_die("Unable to update database file: $uri -> $compUri: $error_code", + $exp_id,$exp_name,$class_id,$uri, $PS_EXIT_SYS_ERROR); + } + # remove original version +# $neb->delete($uri); + } + exit(0); +} + +die "lossy_compress_imfile.pl -state $state not recognized\n"; + + + +sub my_die { + my $msg = shift; # Warning message on die + my $exp_id = shift; # exp_id + my $exp_name = shift; # exp_name + my $class_id = shift; # Class identifier + my $uri = shift; # uri for the file + my $exit_code = shift; # Exit code + # outputImage and path_base are globals + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 27148) +++ trunk/ippTasks/Makefile.am (revision 27149) @@ -31,5 +31,6 @@ dqstats.pro \ science.cleanup.pro \ - nightly_stacks.pro + nightly_stacks.pro \ + lossy_compress.pro other_files = \ Index: trunk/ippTasks/lossy_compress.pro =================================================================== --- trunk/ippTasks/lossy_compress.pro (revision 27149) +++ trunk/ippTasks/lossy_compress.pro (revision 27149) @@ -0,0 +1,215 @@ +## lossy_compress.pro : tasks to lossy compress old raw images that have negligible science value : -*- sh -*- +## use the books compPendingImfile and compPendingExp + +# test for required global variables +check.globals + +book init compPendingImfile +book init compPendingExp + +macro lossycomp.reset + book init compPendingImfile + book init compPendingExp +end + +macro lossycomp.status + book list + book listbook compPendingImfile + book listbook compPendingExp +end + +macro lossycomp.on + task lossycomp.imfile.load + active true + end + task lossycomp.imfile.run + active true + end + task lossycomp.exp.finish + active true + end +end + +macro lossycomp.off + task lossycomp.imfile.load + active false + end + task lossycomp.imfile.run + active false + end + task lossycomp.exp.finish + active false + end +end + +# these variables will cycle through the known database names +$compPendingImfile_DB = 0 +$comp_finish_DB = 0 + +# select images ready to be compressed +task lossycomp.imfile.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.imfile.load.log + + #select entries from the current DB, cycle to the next DB if possible + task.exec + $run = regtool -pendingcompressimfile + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$compPendingImfile_DB + $run = $run -dbname $DB:$compPendingImfile_DB + $compPendingImfile_DB ++ + if ($compPendingImfile_DB >= $DB:n) set compPendingImfile_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + # convert 'stdout' to book format + ipptool2book stdout compPendingImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle compPendingImfile + if ($VERBOSE > 2) + book listbook compPendingImfile + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup compPendingImfile + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +# run the lossy_compress_imfile.pl script on the pending images +task lossycomp.imfile.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 30 + + task.exec + if ($NETWORK == 0) break + book npages compPendingImfile -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + periods -exec 0.05 + + # look for new images + book getpage compPendingImfile 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword compPendingImfile $pageName pantaskState RUN + book getword compPendingImfile $pageName exp_id -var EXP_ID + book getword compPendingImfile $pageName exp_name -var EXP_NAME + book getword compPendingImfile $pageName exp_tag -var EXP_TAG + book getword compPendingImfile $pageName tmp_class_id -var TMP_CLASS_ID + book getword compPendingImfile $pageName class_id -var CLASS_ID + book getword compPendingImfile $pageName camera -var CAMERA + book getword compPendingImfile $pageName uri -var URI + book getword compPendingImfile $pageName bytes -var BYTES + book getword compPendingImfile $pageName md5sum -var MD5SUM + book getword compPendingImfile $pageName workdir -var WORKDIR + book getword compPendingImfile $pageName data_state -var STATE + book getword compPendingImfile $pageName dbname -var DBNAME + + # specify choice of remote host + set.host.for.camera $CAMERA $TMP_CLASS_ID + + # set logfile name + if ("$WORKDIR" == "NULL") + sprintf logfile "compress_log/%s.%d.lossycomp.%s.log" $EXP_NAME $EXP_ID $CLASS_ID + else + sprintf logfile "%s/%s/%s.lossycomp.%s.log" $WORKDIR $EXP_TAG $EXP_TAG $TMP_CLASS_ID + end + + stdout $LOGDIR/lossycomp.imfile.run.log + stdout $LOGDIR/lossycomp.imfile.run.log + + $run = lossy_compress_imfile.pl --exp_id $EXP_ID --class_id $CLASS_ID --exp_name $EXP_NAME --uri $URI --camera $CAMERA --state $STATE --logfile $logfile --bytes $BYTES --md5sum $MD5SUM + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit compPendingImfile $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword compPendingImfile $options:0 pantaskState CRASH + end + task.exit timeout + showcommand timeout + book setword compPendingImfile $options:0 pantaskState TIMEOUT + end +end + +# select exposures ready for lossy_compress_exp.pl +task lossycomp.exp.finish + host local + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/lossycomp.exp.load.log + + task.exec + $run = regtool -finishcompressexp -limit 10 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$comp_finish_DB + $run = $run -dbname $DB:$comp_finish_DB + $comp_finish_DB ++ + if ($comp_finish_DB >= $DB:n) set comp_finish_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit $EXIT_SUCCESS + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 27148) +++ trunk/ippTools/share/Makefile.am (revision 27149) @@ -256,4 +256,8 @@ regtool_revertprocessedimfile.sql \ regtool_updateprocessedimfile.sql \ + regtool_pendingcompressimfile.sql \ + regtool_finishcompressexp.sql \ + regtool_updatebyquery.sql \ + regtool_updatebyqueryimfile.sql \ stacktool_definebyquery_insert.sql \ stacktool_definebyquery_insert_random_part1.sql \ Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 27148) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 27149) @@ -117,4 +117,5 @@ filelevel VARCHAR(64), workdir VARCHAR(255), + state VARCHAR(64), reduction VARCHAR(64), dvodb VARCHAR(255), @@ -189,4 +190,5 @@ class_id VARCHAR(64), uri VARCHAR(255), + data_state VARCHAR(64), exp_type VARCHAR(64), filelevel VARCHAR(64), Index: trunk/ippTools/share/regtool_finishcompressexp.sql =================================================================== --- trunk/ippTools/share/regtool_finishcompressexp.sql (revision 27149) +++ trunk/ippTools/share/regtool_finishcompressexp.sql (revision 27149) @@ -0,0 +1,89 @@ +select DISTINCT + exp_id, + exp_name, + camera, + telescope, + dateobs, + exp_tag, + exp_type, + filelevel, + workdir, + state, + reduction, + dvodb, + tess_id, + end_stage, + filter, + comment, + obs_mode, + obs_group, + airmass, + ra, + decl, + exp_time, + sat_pixel_frac, + bg, + bg_stdev, + bg_mean_stdev, + alt, + az, + ccd_temp, + posang, + m1_x, + m1_y, + m1_z, + m1_tip, + m1_tilt, + m2_x, + m2_y, + m2_z, + m2_tip, + m2_tilt, + env_temperature, + env_humidity, + env_wind_speed, + env_wind_dir, + teltemp_m1, + teltemp_m1cell, + teltemp_m2, + teltemp_spider, + teltemp_truss, + teltemp_extra, + pon_time, + user_1, + user_2, + user_3, + user_4, + user_5, + object, + sun_angle, + sun_alt, + moon_angle, + moon_alt, + moon_phase, + hostname, + fault, + epoch, + magicked +FROM + (SELECT + rawExp.*, + count(rawImfile.class_id) AS complete, + count(newImfile.tmp_class_id) AS total + FROM rawExp + JOIN newImfile + ON rawExp.exp_id = newImfile.exp_id + LEFT JOIN rawImfile + ON rawExp.exp_id = rawImfile.exp_id + AND rawImfile.tmp_class_id = newImfile.tmp_class_id + WHERE + ((rawExp.state = 'goto_compressed' + AND rawImfile.data_state = 'compressed')OR + (rawExp.state = 'goto_lossy' + AND rawImfile.data_state = 'lossy')) + GROUP BY + rawExp.exp_id + ) as foo + WHERE (foo.complete = foo.total) + + Index: trunk/ippTools/share/regtool_pendingcompressimfile.sql =================================================================== --- trunk/ippTools/share/regtool_pendingcompressimfile.sql (revision 27149) +++ trunk/ippTools/share/regtool_pendingcompressimfile.sql (revision 27149) @@ -0,0 +1,4 @@ +SELECT * from rawImfile + WHERE (data_state = 'goto_compressed' OR data_state = 'goto_lossy') +-- where hook %s +-- limit hook %s Index: trunk/ippTools/share/regtool_updatebyquery.sql =================================================================== --- trunk/ippTools/share/regtool_updatebyquery.sql (revision 27149) +++ trunk/ippTools/share/regtool_updatebyquery.sql (revision 27149) @@ -0,0 +1,5 @@ +UPDATE rawExp + SET state = '%s' +WHERE 1 +-- where hook %s +-- limit hook %s Index: trunk/ippTools/share/regtool_updatebyqueryimfile.sql =================================================================== --- trunk/ippTools/share/regtool_updatebyqueryimfile.sql (revision 27149) +++ trunk/ippTools/share/regtool_updatebyqueryimfile.sql (revision 27149) @@ -0,0 +1,5 @@ +UPDATE rawImfile JOIN rawExp USING(exp_id) + SET data_state = '%s' +WHERE 1 +-- where hook %s +-- limit hook %s Index: trunk/ippTools/share/regtool_updateprocessedimfile.sql =================================================================== --- trunk/ippTools/share/regtool_updateprocessedimfile.sql (revision 27148) +++ trunk/ippTools/share/regtool_updateprocessedimfile.sql (revision 27149) @@ -1,4 +1,4 @@ UPDATE rawImfile - SET burntool_state = %d + SET %s WHERE rawImfile.exp_id = %lld Index: trunk/ippTools/src/pxregister.c =================================================================== --- trunk/ippTools/src/pxregister.c (revision 27148) +++ trunk/ippTools/src/pxregister.c (revision 27149) @@ -58,2 +58,67 @@ return true; } + +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawExp state: %s", state); + return false; + } + + char *query = "UPDATE rawExp SET state = '%s' WHERE exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state) +{ + if (!exp_id) { + psError(PS_ERR_UNKNOWN, true, "0 is not a valid exp_id"); + return false; + } + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!( + (strncmp(state, "full", 5) == 0) + || (strncmp(state, "new", 4) == 0) + || (strncmp(state, "goto_compressed", 14) == 0) + || (strncmp(state, "compressed", 11) == 0) + || (strncmp(state, "goto_lossy", 11) == 0) + || (strncmp(state, "lossy", 6) == 0) + ) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid rawImfile data_state: %s", state); + return false; + } + + char *query = "UPDATE rawImfile SET data_state = '%s' WHERE class_id = %s AND exp_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, class_id, exp_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for exp_id %"PRId64, exp_id); + return false; + } + + return true; +} Index: trunk/ippTools/src/pxregister.h =================================================================== --- trunk/ippTools/src/pxregister.h (revision 27148) +++ trunk/ippTools/src/pxregister.h (revision 27149) @@ -26,4 +26,6 @@ bool pxnewExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawExpSetState(pxConfig *config, psS64 exp_id, const char *state); +bool pxrawImfileSetState(pxConfig *config, psS64 exp_id, const char *class_id, const char *state); #endif // PXREGISTER_H Index: trunk/ippTools/src/regtool.c =================================================================== --- trunk/ippTools/src/regtool.c (revision 27148) +++ trunk/ippTools/src/regtool.c (revision 27149) @@ -37,4 +37,5 @@ static bool revertprocessedimfileMode(pxConfig *config); static bool updateprocessedimfileMode(pxConfig *config); +static bool pendingcompressimfileMode(pxConfig *config); // exp static bool pendingexpMode(pxConfig *config); @@ -44,4 +45,8 @@ static bool updateprocessedexpMode(pxConfig *config); static bool cleardupexpMode(pxConfig *config); +static bool finishcompressexpMode(pxConfig *config); + +static bool updatebyqueryMode(pxConfig *config); + static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -77,4 +82,7 @@ MODECASE(REGTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryMode); + MODECASE(REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileMode); + MODECASE(REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpMode); MODECASE(REGTOOL_MODE_CLEARDUPEXP, cleardupexpMode); MODECASE(REGTOOL_MODE_EXPORTRUN, exportrunMode); @@ -189,4 +197,5 @@ PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", false, false); PXOPT_LOOKUP_STR(obs_group, config->args, "-obs_group", false, false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false); PXOPT_LOOKUP_F32(airmass, config->args, "-airmass", false, false); @@ -255,4 +264,5 @@ class_id, uri, + data_state, exp_type, filelevel, @@ -336,5 +346,5 @@ PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "=="); - PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); + PXOPT_COPY_STR(config->args, where, "-data_state", "data_state", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -491,9 +501,11 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); -/* PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */ PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); - - if ((fault == INT16_MAX) && !isfinite(burntool_state)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected"); + PXOPT_LOOKUP_S32(set_bytes, config->args, "-set_bytes", false, false); + PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); return false; } @@ -514,25 +526,122 @@ return false; } - } - - if (isfinite(burntool_state)) { - psString query = pxDataGet("regtool_updateprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); + psFree (where); + return(true); } psFree (where); + + psString setvalues = NULL; + if (burntool_state != INT16_MAX) { + psStringAppend(&setvalues,"rawImfile.burntool_state = %d",burntool_state); + } + if (set_bytes != INT32_MAX) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.bytes = %d",set_bytes); + } + if (set_md5sum) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.md5sum = '%s'",set_md5sum); + } + if (set_state) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); + } + + psString query = pxDataGet("regtool_updateprocessedimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // printf(query,setvalues,exp_id,class_id); + if (!p_psDBRunQueryF(config->dbh, query, setvalues, exp_id,class_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(setvalues); + return false; + } + psFree(setvalues); + psFree(query); return true; } +static bool pendingcompressimfileMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (whereClause) { + psStringPrepend(&whereClause, "\n AND "); + } + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_pendingcompressimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // printf(query,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "regPendingCompressImfile", !simple)) { + psError(PXTOOLS_ERR_PROG, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + static bool pendingexpMode(pxConfig *config) @@ -626,4 +735,5 @@ PXOPT_LOOKUP_STR(exp_type, config->args, "-exp_type", false, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); @@ -787,4 +897,5 @@ filelevel, workdir, + state, reduction, dvodb, @@ -930,6 +1041,7 @@ PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "=="); PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); @@ -964,5 +1076,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); - + psString where2 = NULL; if (!pxspaceAddWhere(config, &where2, "rawExp")) { @@ -974,12 +1086,12 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - + if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); } psFree(where); - + if (where2) { psStringAppend(&query, " %s ", where2); @@ -988,55 +1100,55 @@ if (faulted) { - // list only faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault != 0"); + // list only faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault != 0"); } else { - // don't list faulted rows - psStringAppend(&query, " %s", "AND rawExp.fault = 0"); - } - + // don't list faulted rows + psStringAppend(&query, " %s", "AND rawExp.fault = 0"); + } + // treat limit == 0 as "no limit" if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; } psFree(query); - + psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; } if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - + psTrace("regtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); - + return true; } @@ -1044,42 +1156,42 @@ static bool revertprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); - PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); - - psString query = pxDataGet("regtool_revertprocessedexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(where); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } else { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id_begin", "exp_id", ">="); + PXOPT_COPY_S64(config->args, where, "-exp_id_end", "exp_id", "<="); + + psString query = pxDataGet("regtool_revertprocessedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); - - int numUpdated = psDBAffectedRows(config->dbh); - - psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); - - return true; + return false; + } + psFree(query); + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated); + + return true; } @@ -1087,21 +1199,205 @@ static bool updateprocessedexpMode(pxConfig *config) { - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); - psFree(where); - return false; - } + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree(where); - - return true; -} - + return false; + } + psFree(where); + + return true; +} + + +static bool finishcompressexpMode(pxConfig *config) { + + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_finishcompressexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " %s ", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s ", limitString); + psFree(limitString); + } + // printf(query); + // return(false); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (strcmp(row->state,"goto_compressed") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "compressed")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + else if (strcmp(row->state,"goto_lossy") == 0) { + if (!pxrawExpSetState(config,row->exp_id, "lossy")) { + psError(PS_ERR_UNKNOWN, false, "failed to set rawExp.state for exp_id: %" PRId64, row->exp_id); + psFree(row); + psFree(output); + return(false); + } + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + + psFree(output); + return(true); +} + +static bool updatebyqueryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", true, false); + + // XX test this out; need to make this consistent with the list in regtoolConfig.c + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause,"\n AND "); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("regtool_updatebyquery.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + psString limitString = NULL; + if (limit) { + limitString = psDBGenerateLimitSQL(limit); + psStringPrepend(&limitString, "\n"); + } + // printf(query,set_state,whereClause,limitString); + if (!p_psDBRunQueryF(config->dbh, query, set_state, whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(query); + // Now up date the imfiles. + query = pxDataGet("regtool_updatebyqueryimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return(false); + } + if (!p_psDBRunQueryF(config->dbh, query, set_state,whereClause, limitString ? limitString : "")) { + psError(PXTOOLS_ERR_PROG, false, "database error"); + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + return(false); + } + + psFree(limitString); + psFree(query); + psFree(whereClause); + psFree(where); + + return true; +} + + static bool cleardupexpMode(pxConfig *config) Index: trunk/ippTools/src/regtool.h =================================================================== --- trunk/ippTools/src/regtool.h (revision 27148) +++ trunk/ippTools/src/regtool.h (revision 27149) @@ -36,4 +36,7 @@ REGTOOL_MODE_UPDATEPROCESSEDEXP, REGTOOL_MODE_CLEARDUPEXP, + REGTOOL_MODE_UPDATEBYQUERY, + REGTOOL_MODE_PENDINGCOMPRESSIMFILE, + REGTOOL_MODE_FINISHCOMPRESSEXP, REGTOOL_MODE_EXPORTRUN, REGTOOL_MODE_IMPORTRUN Index: trunk/ippTools/src/regtoolConfig.c =================================================================== --- trunk/ippTools/src/regtoolConfig.c (revision 27148) +++ trunk/ippTools/src/regtoolConfig.c (revision 27149) @@ -72,6 +72,6 @@ ADD_OPT(Str, addprocessedimfileArgs, "-comment", "define comment ", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-obs_mode", "define observing mode (data usage goal)", NULL); - ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); - + ADD_OPT(Str, addprocessedimfileArgs, "-obs_group", "define observing group (set of associated observations)", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-data_state", "define the data_state", "full"); // Note: RA & DEC are supplied here in radians, but the query options use degrees. This // can be justified by the fact that this option (addprocessedimfile) is a software @@ -139,4 +139,5 @@ ADD_OPT(Str, processedimfileArgs, "-obs_mode", "search by obs_mod", NULL); ADD_OPT(Str, processedimfileArgs, "-exp_type", "search by exposure type", NULL); + ADD_OPT(Str, processedimfileArgs, "-data_state", "search by data_state", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); @@ -162,7 +163,11 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", 0); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", INT16_MAX); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); - + ADD_OPT(S32, updateprocessedimfileArgs, "-set_bytes", "set bytes", INT32_MAX); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); + ADD_OPT(Str, updateprocessedimfileArgs, "-set_state", "set data state", NULL); + + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -185,4 +190,5 @@ ADD_OPT(Str, addprocessedexpArgs, "-exp_type", "define exposure type", NULL); ADD_OPT(Str, addprocessedexpArgs, "-workdir", "define the \"default\" workdir for this exposure", NULL); + ADD_OPT(Str, addprocessedexpArgs, "-state", "define the state for this exposure", NULL); ADD_OPT(Str, addprocessedexpArgs, "-dvodb", "define the dvodb for the next processing step", NULL); ADD_OPT(Str, addprocessedexpArgs, "-tess_id", "define the tess_id for the next processing step", NULL); @@ -253,4 +259,5 @@ psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(processedexpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-state", 0, "search by exposure state", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", "object"); @@ -302,5 +309,5 @@ psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", INT16_MAX); - + psMetadataAddStr(updatedprocessedexpArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -313,8 +320,71 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - // -cleardupexp psMetadata *cleardupexpArgs = psMetadataAlloc(); + // -updatebyquery + psMetadata *updatebyqueryArgs = psMetadataAlloc(); + + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "set the state", NULL); + + psMetadataAddS64(updatebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(updatebyqueryArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(updatebyqueryArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(updatebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); + psMetadataAddStr(updatebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by obs_mode", NULL); + + psMetadataAddU64(updatebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(updatebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -pendingcompressimfile + psMetadata *pendingcompressimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcompressimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(pendingcompressimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddBool(pendingcompressimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcompressimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + + // -finishcompressexp + psMetadata *finishcompressexpArgs = psMetadataAlloc(); + psMetadataAddS64(finishcompressexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddBool(finishcompressexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(finishcompressexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -331,4 +401,7 @@ PXOPT_ADD_MODE("-updateprocessedexp", "", REGTOOL_MODE_UPDATEPROCESSEDEXP, updatedprocessedexpArgs); PXOPT_ADD_MODE("-cleardupexp", "", REGTOOL_MODE_CLEARDUPEXP, cleardupexpArgs); + PXOPT_ADD_MODE("-updatebyquery", "", REGTOOL_MODE_UPDATEBYQUERY, updatebyqueryArgs); + PXOPT_ADD_MODE("-pendingcompressimfile", "", REGTOOL_MODE_PENDINGCOMPRESSIMFILE, pendingcompressimfileArgs); + PXOPT_ADD_MODE("-finishcompressexp", "", REGTOOL_MODE_FINISHCOMPRESSEXP, finishcompressexpArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", REGTOOL_MODE_IMPORTRUN, importrunArgs);