Index: /branches/czw_branch/cleanup/dbconfig/add.md =================================================================== --- /branches/czw_branch/cleanup/dbconfig/add.md (revision 25147) +++ /branches/czw_branch/cleanup/dbconfig/add.md (revision 25147) @@ -0,0 +1,25 @@ +addRun METADATA + add_id S64 0 # Primary Key AUTO_INCREMENT + cam_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + state STR 64 # key + workdir STR 255 + workdir_state STR 64 + label STR 64 + dvodb STR 255 + magicked S64 0 +END + +addProcessedExp METADATA + add_id S64 0 # Primary Key AUTO_INCREMENT + dtime_addstar F32 0.0 + + n_stars S32 0 + + path_base STR 255 + fault S16 0 # Key NOT NULL +END + +addMask METADATA + label STR 64 # Primary Key +END + Index: /branches/czw_branch/cleanup/dbconfig/ipp.m4 =================================================================== --- /branches/czw_branch/cleanup/dbconfig/ipp.m4 (revision 25146) +++ /branches/czw_branch/cleanup/dbconfig/ipp.m4 (revision 25147) @@ -14,4 +14,5 @@ include(chip.md) include(cam.md) +include(add.md) include(fake.md) include(warp.md) Index: /branches/czw_branch/cleanup/ippScripts/Build.PL =================================================================== --- /branches/czw_branch/cleanup/ippScripts/Build.PL (revision 25146) +++ /branches/czw_branch/cleanup/ippScripts/Build.PL (revision 25147) @@ -49,4 +49,5 @@ scripts/chip_imfile.pl scripts/camera_exp.pl + scripts/addstar_run.pl scripts/fake_imfile.pl scripts/warp_overlap.pl Index: /branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl (revision 25146) +++ /branches/czw_branch/cleanup/ippScripts/scripts/addstar_run.pl (revision 25147) @@ -40,5 +40,5 @@ } -my ( $exp_tag, $add_id, $camera, $outroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $recipe, $dbname, $reduction, $dvodb, $verbose, $no_update, $no_op, $redirect, $save_temps, $run_state); GetOptions( @@ -49,4 +49,5 @@ 'dbname|d=s' => \$dbname, # Database name 'outroot|w=s' => \$outroot, # output file base name + 'camroot|w=s' => \$camroot, # camera stage root name. 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database @@ -111,69 +112,11 @@ my $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_PROG_ERROR); - + # extract the metadata for the files into a hash list $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - -# # since I can't figure out how to do input and output within PERL, I'm writing to a temp file -# my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.add.$add_id.stats.XXXX", UNLINK => !$save_temps ); -# print "saving stats to $statName\n"; -# foreach my $line (@$stdout_buf) { -# print $statFile $line; -# } -# close $statFile; - -# # parse the stats in the metadata file -# $command = "$ppStatsFromMetadata $statName - CAMERA_EXP_IMFILE"; -# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# warn("Unable to perform ppStatsFromMetadata: $error_code\n"); -# exit($error_code); -# } - -# foreach my $line (@$stdout_buf) { -# $cmdflags .= " $line"; -# } -# chomp $cmdflags; -} - -# ### not needed to have such an extensive temp file name. -# my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b1.list.XXXX", UNLINK => !$save_temps ); # For binning 1 -# my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b2.list.XXXX", UNLINK => !$save_temps ); # For binning 2 -# my ($list3File, $list3Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b3.list.XXXX", UNLINK => !$save_temps ); # For astrometry - -# ### XXX for the moment, always generate the bright-star mask -# my ($list4File, $list4Name) = tempfile( "/tmp/$exp_tag.cm.$cam_id.b4.list.XXXX", UNLINK => !$save_temps ); # For astrometry - -# # XXX we perform astrometry iff photometry output exists + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); +} + my $chipObjectsExist = 0; -# foreach my $file (@$files) { -# # use the path_base as OUTPUT root and convert the filenames with ipprc->filename: -# my $class_id = $file->{class_id}; - -# # If there is only one chip, we use this name for the input to addstar -# # we expect the chip analysis stage to produce psphot output (cmf file) and two binned images -# my $chipObjects = $ipprc->filename("PSPHOT.OUTPUT", $file->{path_base}, $class_id); -# my $chipMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id); - -# # if any of the output chip photometry files exist, we can run psastro / addstar below -# if ($ipprc->file_exists($chipObjects)) { -# $chipObjectsExist = 1; -# } else { -# if ($verbose) { print "skipping $chipObjects (not found)\n"; } -# next; -# } - -# print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n"); -# print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n"); -# print $list3File ($chipObjects . "\n"); -# print $list4File ($chipMask . "\n"); -# } -# close $list1File; -# close $list2File; -# close $list3File; -# close $list4File; # Output products @@ -181,14 +124,9 @@ # the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs -# my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -# my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); -# my $fpaStats = $ipprc->filename("PSASTRO.STATS", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); -#my $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR); if ($run_state eq 'update') { $traceDest .= '.update'; -# $fpaStats .= '.update'; } @@ -203,139 +141,47 @@ unless ($no_op) { - - ## build the output JPEG images first so we get them even if the astrometry fails - - # Make the jpeg for binning 1 -# if ($run_state eq 'new') { -# my $command = "$ppImage -list $list1Name $outroot"; # Command to run -# $command .= " -recipe PPIMAGE $recipe1"; -# $command .= " -dbname $dbname" if defined $dbname; - -# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); -# } -# &my_die("Unable to find expected output file: $jpeg1", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg1); -# } - -# # Make the jpeg for binning 2 -# if ($run_state eq 'new') { -# my $command = "$ppImage -list $list2Name $outroot"; # Command to run -# $command .= " -recipe PPIMAGE $recipe2"; -# $command .= " -dbname $dbname" if defined $dbname; - -# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform ppImage: $error_code", $cam_id, $error_code); -# } -# &my_die("Unable to find expected output file: $jpeg2", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($jpeg2); -# } - - # only run psastro / addstar if any of the output chip astrometry files exist (should we test for successful astrometry?) -# if ($chipObjectsExist) { - # run psastro on the chipObjects, producing fpaObjects -# my $command; -# $command = "$psastro"; -# $command .= " -list $list3Name"; -# $command .= " -masklist $list4Name"; -# $command .= " $outroot"; -# $command .= " -recipe PSASTRO $recipe_psastro"; -# $command .= " -tracedest $traceDest -log $logDest"; -# $command .= " -dbname $dbname" if defined $dbname; - -# my $do_stats; -# if ($run_state eq 'new') { -# $command .= " -stats $fpaStats -recipe PPSTATS CAMSTATS"; -# $command .= " -dumpconfig $configuration"; -# $do_stats = 1; -# } elsif ($run_state eq 'update') { -# $command .= " -ipprc $configuration"; -# } else { -# &my_die("invalid value for run-state: $run_state", $cam_id, $PS_EXIT_CONFIG_ERROR); -# } - -# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code); -# } - -# my $quality; # Quality flag -# if ($do_stats) { -# my $fpaStatsReal = $ipprc->file_resolve($fpaStats); -# &my_die("Couldn't find expected output file: $fpaStats", $cam_id, $PS_EXIT_SYS_ERROR) unless -f $fpaStatsReal; - -# # parse stats from metadata -# $command = "$ppStatsFromMetadata $fpaStatsReal - CAMERA_EXP_FPA"; -# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = -# run(command => $command, verbose => $verbose); -# unless ($success) { -# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); -# &my_die("Unable to perform ppStatsFromMetadata: $error_code", $cam_id, $error_code); -# } -# foreach my $line (@$stdout_buf) { -# $cmdflags .= " $line"; -# } -# chomp $cmdflags; - -# ($quality) = $cmdflags =~ /-quality (\d+)/; -# } - -# if (!$quality) { -# &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects); -# } - -# # run addstar on the output fpaObjects (if a DVO database is defined) - if (defined $dvodbReal and ($run_state eq 'new')) { - - ## XXX the camera analysis can either save the full set of - ## detections, or just the image metadata, in the dvodb - - ## get the addstar recipe for this camera and CAMERA reduction - my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR); - - ## allow the dvodb to save only images, or the full detection set - my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - # XXX this needs to be converted to addstar_client... - - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - $command = "$addstar -D CAMERA $camdir -update"; - $command .= " -image" if $imagesOnly; - $command .= " -D CATDIR $dvodbReal"; - - my $realFile = $ipprc->file_resolve($fpaObjects); - $command .= " $realFile"; - - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script - } -} - - -my $dtime_script = (DateTime->now->mjd - $mjd_start) * 86400; + if (defined $dvodbReal and ($run_state eq 'new')) { + ## XXX the camera analysis can either save the full set of + ## detections, or just the image metadata, in the dvodb + + ## get the addstar recipe for this camera and CAMERA reduction + my $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $add_id, $PS_EXIT_SYS_ERROR); + } + my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $add_id, $PS_EXIT_SYS_ERROR); + + ## allow the dvodb to save only images, or the full detection set + my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + + # XXX this construct requires the user to have a valid .ptolemyrc + # XXX which in turn points at ippconfig/dvo.site + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + # XXX this needs to be converted to addstar_client... + + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + $command = "$addstar -D CAMERA $camdir -update"; + $command .= " -image" if $imagesOnly; + $command .= " -D CATDIR $dvodbReal"; + + my $realFile = $ipprc->file_resolve($fpaObjects); + $command .= " $realFile"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + } +} + # This needs to be updated when addtool is written. BROKEN @@ -345,6 +191,4 @@ $fpaCommand .= " -path_base $outroot"; $fpaCommand .= " $cmdflags"; - $fpaCommand .= " -hostname $host" if defined $host; - $fpaCommand .= " -dtime_script $dtime_script"; $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { @@ -390,5 +234,4 @@ $command .= " -fault $exit_code"; } - $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; system ($command); Index: /branches/czw_branch/cleanup/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/czw_branch/cleanup/ippScripts/scripts/camera_exp.pl (revision 25146) +++ /branches/czw_branch/cleanup/ippScripts/scripts/camera_exp.pl (revision 25147) @@ -93,6 +93,6 @@ &my_die("Unrecognised JPEG recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe2; -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; +#my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +#&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use @@ -205,5 +205,5 @@ } -my $dtime_addstar = 0; +#my $dtime_addstar = 0; unless ($no_op) { @@ -297,47 +297,47 @@ # run addstar on the output fpaObjects (if a DVO database is defined) - if (defined $dvodbReal and ($run_state eq 'new')) { - - ## XXX the camera analysis can either save the full set of - ## detections, or just the image metadata, in the dvodb - - ## get the addstar recipe for this camera and CAMERA reduction - $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); - } - my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); - - ## allow the dvodb to save only images, or the full detection set - my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); - - # XXX this construct requires the user to have a valid .ptolemyrc - # XXX which in turn points at ippconfig/dvo.site - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - # XXX this needs to be converted to addstar_client... - - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $command; - $command = "$addstar -D CAMERA $camdir -update"; - $command .= " -image" if $imagesOnly; - $command .= " -D CATDIR $dvodbReal"; - - my $realFile = $ipprc->file_resolve($fpaObjects); - $command .= " $realFile"; - - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script - } +# if (defined $dvodbReal and ($run_state eq 'new')) { + +# ## XXX the camera analysis can either save the full set of +# ## detections, or just the image metadata, in the dvodb + +# ## get the addstar recipe for this camera and CAMERA reduction +# $command = "$ppConfigDump -camera $camera -recipe ADDSTAR $recipe_addstar -dump-recipe ADDSTAR -"; +# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform ppConfigDump: $error_code", $cam_id, $PS_EXIT_SYS_ERROR); +# } +# my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or +# &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_SYS_ERROR); + +# ## allow the dvodb to save only images, or the full detection set +# my $imagesOnly = metadataLookupBool($recipeData, 'IMAGES.ONLY'); + +# # XXX this construct requires the user to have a valid .ptolemyrc +# # XXX which in turn points at ippconfig/dvo.site +# # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) +# # XXX this needs to be converted to addstar_client... + +# my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar +# my $command; +# $command = "$addstar -D CAMERA $camdir -update"; +# $command .= " -image" if $imagesOnly; +# $command .= " -D CATDIR $dvodbReal"; + +# my $realFile = $ipprc->file_resolve($fpaObjects); +# $command .= " $realFile"; + +# my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + +# my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $command, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform addstar: $error_code", $cam_id, $error_code); +# } +# $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script +# } } } @@ -353,5 +353,5 @@ $fpaCommand .= " -hostname $host" if defined $host; $fpaCommand .= " -dtime_script $dtime_script"; - $fpaCommand .= " -dtime_addstar $dtime_addstar"; +# $fpaCommand .= " -dtime_addstar $dtime_addstar"; } else { $fpaCommand .= " -updaterun -state full"; Index: /branches/czw_branch/cleanup/ippTasks/Makefile.am =================================================================== --- /branches/czw_branch/cleanup/ippTasks/Makefile.am (revision 25146) +++ /branches/czw_branch/cleanup/ippTasks/Makefile.am (revision 25147) @@ -13,4 +13,5 @@ chip.pro \ camera.pro \ + addstar.pro \ fake.pro \ warp.pro \ Index: /branches/czw_branch/cleanup/ippTasks/addstar.pro =================================================================== --- /branches/czw_branch/cleanup/ippTasks/addstar.pro (revision 25147) +++ /branches/czw_branch/cleanup/ippTasks/addstar.pro (revision 25147) @@ -0,0 +1,227 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + +book init addPendingExp + +macro addstar.status + book listbook addPendingExp +end + +macro addstar.reset + book init addPendingExp +end + +macro addstar.on + task addstar.exp.load + active true + end + task addstar.exp.run + active true + end +end + +macro addstar.off + task addstar.exp.load + active false + end + task addstar.exp.run + active false + end +end + +# this variable will cycle through the known database names +$addstar_DB = 0 +$addstar_revert_DB = 0 + +# select images ready for addstar analysis +# new entries are added to addPendingExp +# skip already-present entries +task addstar.exp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -pendingexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the addstar script on pending exposures +task addstar.exp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10 + + task.exec + book npages addPendingExp -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images in addPendingExp (pantaskState == INIT) + book getpage addPendingExp 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword addPendingExp $pageName pantaskState RUN + book getword addPendingExp $pageName camera -var CAMERA + book getword addPendingExp $pageName exp_tag -var EXP_TAG + book getword addPendingExp $pageName add_id -var ADD_ID + book getword addPendingExp $pageName cam_id -var CAM_ID + book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE + book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName dbname -var DBNAME + book getword addPendingExp $pageName reduction -var REDUCTION + book getword addPendingExp $pageName state -var RUN_STATE + + # specify choice of remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + + # set the WORKDIR variable + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + + # notes on how this works: + # -- raw workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb:///@HOST@-vol0/gpc1/20080130 (need to supply volname?, or are we re-defining this each time?) + # -- out workdir examples: + # file://data/ipp004.0/gpc1/20080130 + # neb:///ipp004-vol0/gpc1/20080130 + + ## generate outroot specific to this exposure (& chip) + sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + sprintf camroot "%s/%s/%s.cm.%s" $WORKDIR $EXP_TAG $EXP_TAG $CAM_ID + + stdout $LOGDIR/addstar.exp.log + stderr $LOGDIR/addstar.exp.log + + $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --outroot $outroot --camroot $camroot --redirect-output --run-state $RUN_STATE + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$DVODB" != "NULL") + $run = $run --dvodb $DVODB + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # success + task.exit default + process_exit addPendingExp $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword addPendingExp $options:0 pantaskState CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword addPendingExp $options:0 pantaskState TIMEOUT + end +end + +task addstar.revert + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB + $run = $run -dbname $DB:$addstar_revert_DB + $addstar_revert_DB ++ + if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/czw_branch/cleanup/ippTasks/pantasks.pro =================================================================== --- /branches/czw_branch/cleanup/ippTasks/pantasks.pro (revision 25146) +++ /branches/czw_branch/cleanup/ippTasks/pantasks.pro (revision 25147) @@ -213,4 +213,5 @@ chip.on camera.on + addstar.on fake.on warp.on @@ -225,4 +226,5 @@ chip.off camera.off + addstar.off fake.off warp.off @@ -242,4 +244,5 @@ module chip.pro module camera.pro + module addstar.pro module fake.pro module warp.pro Index: /branches/czw_branch/cleanup/ippTools/doc/addstar_flow.txt =================================================================== --- /branches/czw_branch/cleanup/ippTools/doc/addstar_flow.txt (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/doc/addstar_flow.txt (revision 25147) @@ -0,0 +1,34 @@ +### Set up the addRun row during the previous stage +#>> camtool.c @ addprocessedexpMode(pxConfig *config) +camtool -cam_id $cam_id -addprocessedexp ... + +### Catch that things now need to be added. This should check for magic_status as well, probably. +#>> add.pro / addtool.c / addtoolConfig.c +addtool -pendingsmf +#addMe METADATA +# add_id S64 5 +#END + +### Run the addstar script. +#>> addstar_run.pl +addstar_run.pl --add_id $add_id --camera $camera --camroot=$camroot --dvodb $DVODB + addtool -pendingsmf -add_id $add_id +#addMe METADATA +# add_id S64 5 +# workdir STR /path/to/workdir +# +#END + + ppConfigDump -camera $camera -recipe ADDSTAR $recipt_addstar -dump-recipe ADDSTAR - + addstar -D CAMERA $camdir -update -D CATDIR $dvodbReal $realFile + addtool -add_id $add_id -addprocessedsmf ... + addtool -add_id $add_id -updaterun -state full + +### Reverting +#>> addtool.c +addtool -revertprocessedsmf ... + + + + + Index: /branches/czw_branch/cleanup/ippTools/share/Makefile.am =================================================================== --- /branches/czw_branch/cleanup/ippTools/share/Makefile.am (revision 25146) +++ /branches/czw_branch/cleanup/ippTools/share/Makefile.am (revision 25147) @@ -4,4 +4,8 @@ dist_pkgdata_DATA = \ + addtool_find_pendingexp.sql \ + addtool_queue_cam_id.sql \ + addtool_reset_faulted_runs.sql \ + addtool_revertprocessedexp.sql \ camtool_donecleanup.sql \ camtool_find_chip_id.sql \ Index: /branches/czw_branch/cleanup/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- /branches/czw_branch/cleanup/ippTools/share/addtool_queue_cam_id.sql (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/share/addtool_queue_cam_id.sql (revision 25147) @@ -0,0 +1,14 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + cam_id, -- cam_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- label + '%s', -- dvodb + 0 -- magicked + FROM camRun + WHERE + camRun.state = 'full' + AND camRun.cam_id = %lld Index: /branches/czw_branch/cleanup/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/czw_branch/cleanup/ippTools/share/pxadmin_create_tables.sql (revision 25146) +++ /branches/czw_branch/cleanup/ippTools/share/pxadmin_create_tables.sql (revision 25147) @@ -451,4 +451,38 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE addRun ( + add_id BIGINT AUTO_INCREMENT, + cam_id BIGINT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(64), + label VARCHAR(64), + dvodb VARCHAR(255), + magicked BIGINT, + PRIMARY KEY(add_id), + KEY(add_id), + KEY(cam_id), + KEY(state), + KEY(workdir_state), + KEY(label), + INDEX(add_id, cam_id), + FOREIGN KEY(cam_id) REFERENCES camRun(cam_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addProcessedExp ( + add_id BIGINT AUTO_INCREMENT, + dtime_addstar FLOAT, + n_stars INT, + path_base VARCHAR(255), + fault SMALLINT NOT NULL, + PRIMARY KEY(add_id), + FOREIGN KEY(add_id) REFERENCES addRun(add_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE addMask ( + label VARCHAR(64), + PRIMARY KEY(label) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE fakeRun ( fake_id BIGINT AUTO_INCREMENT, @@ -1086,5 +1120,5 @@ KEY(magic_id), KEY(label), - FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id) + FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY(inv_magic_id) REFERENCES magicRun(magic_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/czw_branch/cleanup/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/czw_branch/cleanup/ippTools/share/pxadmin_drop_tables.sql (revision 25146) +++ /branches/czw_branch/cleanup/ippTools/share/pxadmin_drop_tables.sql (revision 25147) @@ -19,4 +19,7 @@ DROP TABLE IF EXISTS detRun; DROP TABLE IF EXISTS detInputExp; +DROP TABLE IF EXISTS addRun; +DROP TABLE IF EXISTS addProcessedExp; +DROP TABLE IF EXISTS addMask; DROP TABLE IF EXISTS fakeRun; DROP TABLE IF EXISTS fakeProcessedImfile; Index: /branches/czw_branch/cleanup/ippTools/src/Makefile.am =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/Makefile.am (revision 25146) +++ /branches/czw_branch/cleanup/ippTools/src/Makefile.am (revision 25147) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + addtool \ caltool \ camtool \ @@ -28,4 +29,5 @@ pkginclude_HEADERS = \ + pxadd.h \ pxadmin.h \ pxcam.h \ @@ -42,4 +44,5 @@ noinst_HEADERS = \ + addtool.h \ caltool.h \ camtool.h \ @@ -68,4 +71,5 @@ libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) libpxtools_la_SOURCES = \ + pxadd.c \ pxcam.c \ pxchip.c \ @@ -96,4 +100,10 @@ pstamptool.c \ pstamptoolConfig.c + +addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +addtool_SOURCES = \ + addtool.c \ + addtoolConfig.c caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: /branches/czw_branch/cleanup/ippTools/src/addtool.c =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/addtool.c (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/src/addtool.c (revision 25147) @@ -0,0 +1,1116 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = addtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(ADDTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(ADDTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(ADDTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(ADDTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(ADDTOOL_MODE_BLOCK, blockMode); + MODECASE(ADDTOOL_MODE_MASKED, maskedMode); + MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + + // find the cam_id of all the exposures that we want to queue up. + psString query = pxDataGet("addtool_find_cam_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + + // loop over our list of addRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + addRunRow *row = addRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + row->cam_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + "RECIPE", + dvodb ? dvodb : row->dvodb + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set addRun.state to state + if (!pxaddRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set addRun.label to label + if (!pxaddRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* static bool pendingimfileMode(pxConfig *config) */ +/* { */ +/* PS_ASSERT_PTR_NON_NULL(config, false); */ + +/* psMetadata *where = psMetadataAlloc(); */ +/* pxaddGetSearchArgs (config, where); */ +/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ +/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ + +/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ + +/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ +/* if (!query) { */ +/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ +/* return false; */ +/* } */ + +/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ +/* if (psListLength(where->list)) { */ +/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ +/* psStringAppend(&query, " AND %s", whereClause); */ +/* psFree(whereClause); */ +/* } */ +/* psFree(where); */ + +/* if (!p_psDBRunQuery(config->dbh, query)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* psFree(query); */ +/* return false; */ +/* } */ +/* psFree(query); */ + +/* psArray *output = p_psDBFetchResult(config->dbh); */ +/* if (!output) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* if (!psArrayLength(output)) { */ +/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ +/* psFree(output); */ +/* return true; */ +/* } */ + +/* // negate simple so the default is true */ +/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ +/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ +/* psFree(output); */ +/* return false; */ +/* } */ + +/* psFree(output); */ + +/* return true; */ +/* } */ + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + addRunRow *pendingRow = addRunObjectFromMetadata(output->data[0]); + psFree(output); + addProcessedExpRow *row = addProcessedExpRowAlloc( + pendingRow->add_id, + dtime_addstar, + n_stars, + path_base, + 0 + ); + + if (!addProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set addRun.state = 'full' + if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if fault != 0 + // Also, we can run fake even if tess_id is not defined +/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ +/* psFree(row); */ +/* psFree(pendingRow); */ +/* if (!psDBCommit(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* return true; */ +/* } */ + psFree(row); + // else continue on... + +/* if (!pxfakeQueueByAddID(config, */ +/* pendingRow->add_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->expgroup, */ +/* pendingRow->dvodb, */ +/* pendingRow->tess_id, */ +/* pendingRow->end_stage */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; + } + + psString query = pxDataGet("addtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + // we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); + } + psFree(where); + + // order by add_id so that the postage stamp parser can easliy find the 'latest' astrometry + psStringAppend(&query, " ORDER BY add_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); +/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("addtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); +/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!addMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM addMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM addMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + + psString query = pxDataGet("addtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (add_id) { + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + + ExportTable tables [] = { + {"addRun", "addtool_export_run.sql"}, + {"addProcessedExp", "addtool_export_processed_exp.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "addRun")) { + if (!pxSetStateCleaned("addRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + fprintf (stdout, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); + + psMetadataItem *item = psMetadataLookup (input, "addRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); + addRunInsertObject (config->dbh, addRun); + + // fprintf (stdout, "---- add run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + item = psMetadataLookup (input, "addProcessedExp"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + for (int i = 0; i < item->data.list->n; i++) { + psMetadataItem *entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); + addProcessedExpInsertObject (config->dbh, addProcessedExp); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + + return true; +} Index: /branches/czw_branch/cleanup/ippTools/src/addtool.h =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/addtool.h (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/src/addtool.h (revision 25147) @@ -0,0 +1,46 @@ +/* + * addtool.h + * + * Copyright (C) 2006 Joshua Hoblitt, Christopher Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ADDTOOL_H +#define ADDTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + ADDTOOL_MODE_NONE = 0x0, + ADDTOOL_MODE_DEFINEBYQUERY, + ADDTOOL_MODE_UPDATERUN, + ADDTOOL_MODE_PENDINGEXP, + ADDTOOL_MODE_ADDPROCESSEDEXP, + ADDTOOL_MODE_PROCESSEDEXP, + ADDTOOL_MODE_REVERTPROCESSEDEXP, + ADDTOOL_MODE_UPDATEPROCESSEDEXP, + ADDTOOL_MODE_BLOCK, + ADDTOOL_MODE_MASKED, + ADDTOOL_MODE_UNBLOCK, + ADDTOOL_MODE_PENDINGCLEANUPRUN, + ADDTOOL_MODE_PENDINGCLEANUPEXP, + ADDTOOL_MODE_DONECLEANUP, + ADDTOOL_MODE_EXPORTRUN, + ADDTOOL_MODE_IMPORTRUN +} addtoolMode; + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // ADDTOOL_H Index: /branches/czw_branch/cleanup/ippTools/src/addtoolConfig.c =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/addtoolConfig.c (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/src/addtoolConfig.c (revision 25147) @@ -0,0 +1,212 @@ +/* + * addtoolConfig.c + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ + + // -revertprocessedexp + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exp", ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exps", ADDTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exp", ADDTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /branches/czw_branch/cleanup/ippTools/src/camtool.c =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/camtool.c (revision 25146) +++ /branches/czw_branch/cleanup/ippTools/src/camtool.c (revision 25147) @@ -656,4 +656,21 @@ return false; } + + if (!pxaddQueueByCamID(config, + pendingRow->cam_id, + pendingRow->workdir, + pendingRow->label, + pendingRow->reduction, + pendingRow->dvodb + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); + psFree(pendingRow); + return false; + } + psFree(pendingRow); Index: /branches/czw_branch/cleanup/ippTools/src/pxadd.c =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/pxadd.c (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/src/pxadd.c (revision 25147) @@ -0,0 +1,262 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" + +bool pxaddSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ + + return true; +} + +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); +/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ +/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ +/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ + + return true; +} + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + + +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE addRun SET addRun.label = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, label, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + +// Need to think more about this to see what we want it to do. BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("addtool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + // fprintf(stderr,"%s",query); + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQueryF(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + dvodb ? dvodb : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: /branches/czw_branch/cleanup/ippTools/src/pxadd.h =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/pxadd.h (revision 25147) +++ /branches/czw_branch/cleanup/ippTools/src/pxadd.h (revision 25147) @@ -0,0 +1,44 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADD_H +#define PXADD_H 1 + +#include + +#include "pxtools.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxaddSetSearchArgs (psMetadata *md); +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); + +// Likely BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb); + + +#endif // PXADD_H Index: /branches/czw_branch/cleanup/ippTools/src/pxtools.h =================================================================== --- /branches/czw_branch/cleanup/ippTools/src/pxtools.h (revision 25146) +++ /branches/czw_branch/cleanup/ippTools/src/pxtools.h (revision 25147) @@ -37,4 +37,5 @@ #include "pxtoolsErrorCodes.h" +#include "pxadd.h" #include "pxcam.h" #include "pxchip.h"