Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 25821) +++ /trunk/ippScripts/Build.PL (revision 25822) @@ -59,4 +59,5 @@ scripts/magic_destreak.pl scripts/magic_destreak_revert.pl + scripts/magic_destreak_cleanup.pl scripts/ippdb.pl scripts/ipp_cleanup.pl Index: /trunk/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /trunk/ippScripts/scripts/magic_destreak_cleanup.pl (revision 25822) +++ /trunk/ippScripts/scripts/magic_destreak_cleanup.pl (revision 25822) @@ -0,0 +1,267 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename dirname ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + +use PS::IPP::Config 1.01 qw( :standard ); +use Nebulous::Client; +use DBI; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ($magic_ds_id, $camera); +my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); + +GetOptions( + 'magic_ds_id=s' => \$magic_ds_id,# Magic destreak run identifier + 'camera=s' => \$camera, # camera for evaluating file rules + 'save-temps' => \$save_temps, # Save temporary files? + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --magic_ds_id --camera", + -exitval => 3) unless + defined $magic_ds_id and + defined $camera; +# defined $stage and +# defined $stage_id; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; + +if (0) { +my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' ); +&my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer; + +my $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); }; +if ($@ or not defined $nebulous) { + &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer", $magic_ds_id, $PS_EXIT_CONFIG_ERROR); +} +} + +$dbname = metadataLookupStr( $ipprc->{_siteConfig}, 'DBNAME' ) if !$dbname; +&my_die ("Unable to find DBNAME in site or command line arguments", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbname; + +my $dbuser = metadataLookupStr( $ipprc->{_siteConfig}, 'DBUSER' ); +&my_die ("Unable to find DBUSER in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbuser; + +my $dbpassword = metadataLookupStr( $ipprc->{_siteConfig}, 'DBPASSWORD' ); +&my_die ("Unable to find DBPASSWORD in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbpassword; + +my $dbserver = metadataLookupStr( $ipprc->{_siteConfig}, 'DBSERVER' ); +&my_die ("Unable to find DBSERVER in site", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$dbserver; + +my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; +my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) or die "Cannot connect to mysql server\n"; + +my $q1 = "SELECT magicDSRun.*, camera, camProcessedExp.path_base AS cam_path_base, magicRun.inverse" + . " FROM magicDSRun JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) LEFT JOIN camProcessedExp USING(cam_id)" + . " WHERE magic_ds_id = $magic_ds_id"; +my $q2 = "SELECT * from magicDSFile where magic_ds_id = $magic_ds_id"; + +# my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +my $stmt1 = $dbh->prepare($q1); +$stmt1->execute(); +my $nrows = $stmt1->rows; +&my_die ("Unable to find magicDSRun $magic_ds_id", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if !$nrows; +my $run = $stmt1->fetchrow_hashref(); + +my $state = $run->{state}; +my $stage = $run->{stage}; +my $stage_id = $run->{stage_id}; +my $cam_path_base = $run->{cam_path_base}; +my $inverse = $run->{inverse}; + + +&my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_purged"; +&my_die("purge not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw"; + + +my $stmt2 = $dbh->prepare($q2); +$stmt2->execute(); + +my $num_components = 0; +while (my $comp = $stmt2->fetchrow_hashref()) { + $num_components++; + + my $component = $comp->{component}; + my $backup_path_base = $comp->{backup_path_base}; + my $recovery_path_base = $comp->{recovery_path_base}; + my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom); + my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom); + + if ($stage eq "chip") { + # we use the mask output from the camera stage for input and replace + # the output of the chip stage with that mask as well. + my $cammask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component); + + if ($backup_path_base) { + $bimage = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $component); + $bch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $backup_path_base, $component); + $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $component); + # This is kludgey but correct + + my $prefix; + if ($backup_path_base =~ /\.SR/) { + $prefix = ""; + } else { + $prefix = "SR_"; + } + + $bmask = dirname($backup_path_base) . "/" . $prefix . basename($cammask); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component); + $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component); + $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component); + # This is kludgey but correct + $rmask = dirname($recovery_path_base) . "/" . basename($cammask); + } + } elsif ($stage eq "camera") { + if ($backup_path_base) { + $bastrom = $ipprc->filename("PSASTRO.OUTPUT", $backup_path_base); + } + } elsif ($stage eq "warp") { + if ($backup_path_base) { + $bimage = $ipprc->filename("PSWARP.OUTPUT", $backup_path_base); + $bmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $backup_path_base); + $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base); + $rmask = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base); + $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base); + $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base); + } + } elsif ($stage eq "diff") { + my $name = "PPSUB.OUTPUT"; + if ($backup_path_base) { + $bimage = $ipprc->filename($name, $backup_path_base); + $bmask = $ipprc->filename("$name.MASK", $backup_path_base); + $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename($name, $recovery_path_base); + $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); + $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); + $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); + } + } + + delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom); + + if ($stage eq "diff" and $inverse) { + my $name = "PPSUB.INVERSE"; + if ($backup_path_base) { + $bimage = $ipprc->filename($name, $backup_path_base); + $bmask = $ipprc->filename("$name.MASK", $backup_path_base); + $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base); + $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); + } + if ($recovery_path_base) { + $rimage = $ipprc->filename($name, $recovery_path_base); + $rmask = $ipprc->filename("$name.MASK", $recovery_path_base); + $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base); + $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base); + } + delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources, undef); + } +} + +if (!$no_update and ($num_components > 0)) { + my $result = $dbh->do("DELETE FROM magicDSFile WHERE magic_ds_id = ?", undef, $magic_ds_id); + # my $result = $stmt3->do($magic_ds_id); + my_die("attempt to delete magicDSFiles failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0"; +} + +if (!$no_update) { + my $result = $dbh->do("UPDATE magicDSRun SET state = 'purged' WHERE magic_ds_id = ?", undef, $magic_ds_id); + my_die("attempt to update magicDSRun failed", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR) if $result eq "0E0"; +} else { + print STDERR "skipping update of magicDSRun\n"; +} + + +### Pau. + +sub delete_files { + foreach my $file (@_) { + if ($file) { + if ($ipprc->file_exists($file)) { + if (!$no_update) { + print STDERR "deleting $file\n" if $verbose; + $ipprc->file_delete($file) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR); + } else { + print STDERR "skipping delete $file\n"; + } + } else { + print STDERR "$file not found\n" if $verbose; + } + } + } +} + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $magic_ds_id = shift; # Magic DS identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + my $command = "$magicdstool -updaterun -set_state purge_fault.$exit_code"; + $command .= " -magic_ds_id $magic_ds_id"; + $command .= " -dbname $dbname" if defined $dbname; + + # Add the processed file to the database + unless (1 or $no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + carp("failed to update database for $magic_ds_id"); + } + } else { + print "Skipping command: $command\n"; + } + + carp($msg); + exit $exit_code; +} + +__END__ Index: /trunk/ippTasks/destreak.pro =================================================================== --- /trunk/ippTasks/destreak.pro (revision 25821) +++ /trunk/ippTasks/destreak.pro (revision 25822) @@ -10,4 +10,5 @@ book init magicToDS book init magicDSToRevert +book init magicDSToCleanup ### indexes into Database lists @@ -16,4 +17,5 @@ $magicDSToRevert_DB = 0 $magicDSCompletedRevert_DB = 0 +$magicDSToCleanup_DB = 0 #list of stages @@ -32,4 +34,5 @@ book listbook magicToDS book listbook magicDSToRevert + book init magicDSToCleanup end @@ -38,4 +41,5 @@ book init magicToDS book init magicDSToRevert + book init magicDSToCleanup end @@ -71,4 +75,13 @@ end +macro destreak.cleanup.on + task destreak.cleanup.load + active true + end + task destreak.cleanup.run + active true + end +end + ### Turn tasks off macro destreak.off @@ -92,4 +105,12 @@ end task destreak.completed.revert + active false + end +end +macro destreak.cleanup.off + task destreak.cleanup.load + active false + end + task destreak.cleanup.run active false end @@ -489,2 +510,125 @@ end +task destreak.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 20 + npending 1 + active false + + stdout NULL + stderr $LOGSUBDIR/destreak.cleanup.load.log + + task.exec + $run = magicdstool -tocleanup + if ($DB:n == 0) + option DEFAULT + else + + # save the DB name for the exit tasks + option $DB:$magicDSToCleanup_DB + $run = $run -dbname $DB:$magicDSToCleanup_DB + + # only bump database number after we have gone through all of the stages + $magicDSToCleanup_DB ++ + if ($magicDSToCleanup_DB >= $DB:n) set magicDSToCleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout magicDSToCleanup -key magic_ds_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook magicDSToCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup magicDSToCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task destreak.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + active false + + task.exec + stdout $LOGSUBDIR/destreak.cleanup.run.log + stderr $LOGSUBDIR/destreak.cleanup.run.log + + book npages magicDSToCleanup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new images (pantaskState == INIT) + book getpage magicDSToCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword magicDSToCleanup $pageName pantaskState RUN + book getword magicDSToCleanup $pageName magic_ds_id -var MAGIC_DS_ID + book getword magicDSToCleanup $pageName camera -var CAMERA + book getword magicDSToCleanup $pageName outroot -var OUTROOT + book getword magicDSToCleanup $pageName dbname -var DBNAME + + sprintf logfile "%s/mds.%s.cleanup.log" $OUTROOT $MAGIC_DS_ID + + host anyhost + + $run = magic_destreak_cleanup.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --logfile $logfile + + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit 0 + process_exit magicDSToCleanup $options:0 $JOB_STATUS + end + + # locked list + task.exit default + showcommand failure + process_exit magicDSToCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword magicDSToCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword magicDSToCleanup $options:0 pantaskState TIMEOUT + end +end + Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 25821) +++ /trunk/ippTools/share/Makefile.am (revision 25822) @@ -36,4 +36,5 @@ chiptool_revertprocessedimfile.sql \ chiptool_run.sql \ + chiptool_runstate.sql \ chiptool_export_imfile.sql \ chiptool_export_processed_imfile.sql \ @@ -190,4 +191,5 @@ magicdstool_getskycells.sql \ magicdstool_revertdestreakedfile.sql \ + magicdstool_tocleanup.sql \ magicdstool_todestreak_camera.sql \ magicdstool_todestreak_chip.sql \ @@ -272,4 +274,5 @@ warptool_revertoverlap.sql \ warptool_revertwarped_delete.sql \ + warptool_runstate.sql \ warptool_scmap.sql \ warptool_tooverlap.sql \ Index: /trunk/ippTools/src/chiptool.c =================================================================== --- /trunk/ippTools/src/chiptool.c (revision 25821) +++ /trunk/ippTools/src/chiptool.c (revision 25822) @@ -57,4 +57,5 @@ static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -97,4 +98,5 @@ MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(CHIPTOOL_MODE_RUNSTATE, runstateMode); default: psAbort("invalid option (this should not happen)"); @@ -1508,2 +1510,82 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("chiptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /trunk/ippTools/src/chiptool.h =================================================================== --- /trunk/ippTools/src/chiptool.h (revision 25821) +++ /trunk/ippTools/src/chiptool.h (revision 25822) @@ -47,5 +47,6 @@ CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, - CHIPTOOL_MODE_IMPORTRUN + CHIPTOOL_MODE_IMPORTRUN, + CHIPTOOL_MODE_RUNSTATE } chiptoolMode; Index: /trunk/ippTools/src/chiptoolConfig.c =================================================================== --- /trunk/ippTools/src/chiptoolConfig.c (revision 25821) +++ /trunk/ippTools/src/chiptoolConfig.c (revision 25822) @@ -271,4 +271,15 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -281,4 +292,5 @@ PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); + PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-advanceexp", "advance completed exposures", CHIPTOOL_MODE_ADVANCEEXP, advanceexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); Index: /trunk/ippTools/src/magicdstool.c =================================================================== --- /trunk/ippTools/src/magicdstool.c (revision 25821) +++ /trunk/ippTools/src/magicdstool.c (revision 25822) @@ -43,4 +43,5 @@ static bool torevertMode(pxConfig *config); static bool completedrevertMode(pxConfig *config); +static bool tocleanupMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state); @@ -76,4 +77,5 @@ MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); + MODECASE(MAGICDSTOOL_MODE_TOCLEANUP, tocleanupMode); default: psAbort("invalid option (this should not happen)"); @@ -1221,2 +1223,76 @@ +static bool tocleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magicdstool_tocleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /trunk/ippTools/src/magicdstool.h =================================================================== --- /trunk/ippTools/src/magicdstool.h (revision 25821) +++ /trunk/ippTools/src/magicdstool.h (revision 25822) @@ -36,4 +36,5 @@ MAGICDSTOOL_MODE_TOREVERT, MAGICDSTOOL_MODE_COMPLETEDREVERT, + MAGICDSTOOL_MODE_TOCLEANUP, } MAGICDStoolMode; Index: /trunk/ippTools/src/magicdstoolConfig.c =================================================================== --- /trunk/ippTools/src/magicdstoolConfig.c (revision 25821) +++ /trunk/ippTools/src/magicdstoolConfig.c (revision 25822) @@ -157,4 +157,12 @@ psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tocleanup + psMetadata *tocleanupArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-stage", 0, "define output directory", NULL); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -185,4 +193,6 @@ PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + PXOPT_ADD_MODE("-tocleanup", "destreak runs to clean up", + MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs); Index: /trunk/ippTools/src/warptool.c =================================================================== --- /trunk/ippTools/src/warptool.c (revision 25821) +++ /trunk/ippTools/src/warptool.c (revision 25822) @@ -58,4 +58,5 @@ static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -108,4 +109,5 @@ MODECASE(WARPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); default: @@ -1915,2 +1917,82 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("warptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /trunk/ippTools/src/warptool.h =================================================================== --- /trunk/ippTools/src/warptool.h (revision 25821) +++ /trunk/ippTools/src/warptool.h (revision 25822) @@ -53,5 +53,6 @@ WARPTOOL_MODE_UPDATESKYFILE, WARPTOOL_MODE_EXPORTRUN, - WARPTOOL_MODE_IMPORTRUN + WARPTOOL_MODE_IMPORTRUN, + WARPTOOL_MODE_RUNSTATE, } warptoolMode; Index: /trunk/ippTools/src/warptoolConfig.c =================================================================== --- /trunk/ippTools/src/warptoolConfig.c (revision 25821) +++ /trunk/ippTools/src/warptoolConfig.c (revision 25822) @@ -318,4 +318,16 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); +// psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psFree(now); @@ -336,4 +348,5 @@ PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-runstate", "", WARPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs);