Index: /trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_cleanup.pl (revision 29670) +++ /trunk/ippScripts/scripts/ipp_cleanup.pl (revision 29671) @@ -18,5 +18,5 @@ # Parse the command-line arguments -my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist); +my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile); GetOptions('stage=s' => \$stage, # which analysis stage to clean? 'camera|i=s' => \$camera, # user-supplied camera name @@ -27,5 +27,6 @@ 'verbose' => \$verbose, # Print to stdout 'no-op' => \$no_op, # pretend but don't actually inject - 'helplist' => \$helplist # give help listing + 'helplist' => \$helplist, # give help listing + 'logfile=s' => \$logfile # destination for stdout and stderr ) or pod2usage( 2 ); @@ -53,8 +54,11 @@ } +$ipprc->redirect_output($logfile) or + &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; + # set this to 1 to enable checking for files on dead nodes # it is off for now because the implementation is a hack # See comments below. -my $check_for_gone = 0; +my $check_for_gone = 1; my $error_state; @@ -66,4 +70,5 @@ my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, + "chip_bg" => 1, "warp_bg" => 1, "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); @@ -1648,4 +1653,290 @@ } +if ($stage eq "chip_bg") { + + die "--stage_id required for stage chip_bg\n" if !$stage_id; + + &my_die("only mode goto_cleaned is supported for stage chip_bg", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR) + if $mode ne 'goto_cleaned'; + + ### select the imfiles for this entry + + # this stage uses 'bgtool' + my $bgtool = can_run('bgtool') or die "Can't find bgtool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$bgtool -pendingcleanupchipimfile -chip_bg_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); + } + + # XXX: Is not having any files to process really a bug? + if (@$stdout_buf == 0) { + my $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); + } + exit 0; + } + + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + $status = 0 unless defined $path_base and $path_base ne "NULL"; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id); + + unless ($ipprc->file_exists($config_file)) { + if (file_gone($config_file)) { + print STDERR "forcing cleanup for chipRun $stage_id $class_id " + . " because config file ($config_file) is gone\n"; + } else { + print STDERR "skipping cleanup for chipRun $stage_id $class_id " + . " because config file ($config_file) is missing\n"; + $status = 0; + } + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $class_id " + . " because config file ($config_file) is present\n"; + $status = 0; + } + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the image datafiles + addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPBACKGROUND.STATS", $path_base, $class_id); #clean? + addFilename (\@files, "PPBACKGOROUND.CONFIG", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$bgtool -chip_bg_id $stage_id -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -tocleanedchipimfile"; + } + elsif ($mode eq "goto_purged") { + $command .= " -topurgedchipimfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedchipimfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + + } else { + + # if an error happens for one chip, the chipBackgroundRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$bgtool -updatechipimfile -chip_bg_id $stage_id -class_id $class_id -set_data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); + } + + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + + } + } + exit 0; +} +if ($stage eq "warp_bg") { + + die "--stage_id required for stage warp_bg\n" if !$stage_id; + + &my_die("only mode goto_cleaned is supported for stage warp_bg", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR) + if $mode ne 'goto_cleaned'; + + ### select the imfiles for this entry + + # this stage uses 'bgtool' + my $bgtool = can_run('bgtool') or die "Can't find bgtool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$bgtool -pendingcleanupwarpskyfile -warp_bg_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); + } + + # XXX: Is not having any files to process really a bug? + if (@$stdout_buf == 0) { + my $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); + } + exit 0; + } + + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and skycell_id for each + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + $status = 0 unless defined $path_base and $path_base ne "NULL"; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + if (file_gone($config_file)) { + print STDERR "forcing cleanup for warpBackgroundSkyfile $stage_id $skycell_id " + . " because config file ($config_file) is gone\n"; + } else { + print STDERR "skipping cleanup for warpBackgroundSkyfile $stage_id $skycell_id " + . " because config file ($config_file) is missing\n"; + $status = 0; + } + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $skycell_id " + . " because config file ($config_file) is present\n"; + $status = 0; + } + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the image datafiles + addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id); + addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PSWARP.STATS", $path_base, $skycell_id); #clean? + addFilename (\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$bgtool -warp_bg_id $stage_id -skycell_id $skycell_id"; + # recall that only goto_cleaned is supported (currently) + if ($mode eq "goto_cleaned") { + $command .= " -tocleanedwarpskyfile"; + } + elsif ($mode eq "goto_purged") { + $command .= " -topurgedwarpskyfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedwarpskyfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code); + } + + } else { + + # if an error happens for one skycell, the warpBackgroundRun will stay in goto_*, but the skycells will go to error_* (matching the goto_*) + my $command = "$bgtool -updatewarpskyfile -warp_bg_id $stage_id -skycell_id $skycell_id -set_data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); + } + + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code); + } + } + } + exit 0; +} die "ipp_cleanup.pl -stage $stage not yet implemented\n"; Index: /trunk/ippTasks/science.cleanup.pro =================================================================== --- /trunk/ippTasks/science.cleanup.pro (revision 29670) +++ /trunk/ippTasks/science.cleanup.pro (revision 29671) @@ -8,4 +8,6 @@ diff.cleanup.on stack.cleanup.on + chip.bg.cleanup.on + warp.bg.cleanup.on end @@ -934,2 +936,308 @@ +book init chipBGPendingCleanup + +macro chip.bg.cleanup.status + book listbook chipBGPendingCleanup +end + +macro chip.bg.cleanup.reset + book init chipBGPendingCleanup +end + +macro chip.bg.cleanup.on + task chip.bg.cleanup.load + active true + end + task chip.bg.cleanup.run + active true + end +end + +macro chip.bg.cleanup.off + task chip.bg.cleanup.load + active false + end + task chip.bg.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$chip_bg_cleanup_DB = 0 + +task chip.bg.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/chip.bg.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -pendingcleanupchiprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_bg_cleanup_DB + $run = $run -dbname $DB:$chip_bg_cleanup_DB + $chip_bg_cleanup_DB ++ + if ($chip_bg_cleanup_DB >= $DB:n) set chip_bg_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipBGPendingCleanup -key chip_bg_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipBGPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup chipBGPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task chip.bg.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + active false + + task.exec + book npages chipBGPendingCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in chipBGPendingCleanup (pantaskState == INIT) + book getpage chipBGPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipBGPendingCleanup $pageName pantaskState RUN + book getword chipBGPendingCleanup $pageName camera -var CAMERA + book getword chipBGPendingCleanup $pageName state -var CLEANUP_MODE + book getword chipBGPendingCleanup $pageName chip_bg_id -var CHIP_BG_ID + book getword chipBGPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword chipBGPendingCleanup $pageName exp_tag -var EXP_TAG + book getword chipBGPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and chip (class_id) + set.host.for.camera $CAMERA FPA + set.workdir.by.camera $CAMERA FPA $WORKDIR_TEMPLATE $default_host WORKDIR + sprintf logfile "%s/%s/%s.bgc.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID + + stdout $LOGDIR/chip.bg.cleanup.log + stderr $LOGDIR/chip.bg.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage chip_bg --stage_id $CHIP_BG_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipBGPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword chipBGPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipBGPendingCleanup $options:0 pantaskState TIMEOUT + end +end + +book init warpBGPendingCleanup + +macro warp.bg.cleanup.status + book listbook warpBGPendingCleanup +end + +macro warp.bg.cleanup.reset + book init warpBGPendingCleanup +end + +macro warp.bg.cleanup.on + task warp.bg.cleanup.load + active true + end + task warp.bg.cleanup.run + active true + end +end + +macro warp.bg.cleanup.off + task warp.bg.cleanup.load + active false + end + task warp.bg.cleanup.run + active false + end +end + +# this variable will cycle through the known database names +$warp_bg_cleanup_DB = 0 + +task warp.bg.cleanup.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 300 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/warp.bg.cleanup.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -pendingcleanupwarprun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_bg_cleanup_DB + $run = $run -dbname $DB:$warp_bg_cleanup_DB + $warp_bg_cleanup_DB ++ + if ($warp_bg_cleanup_DB >= $DB:n) set warp_bg_cleanup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpBGPendingCleanup -key warp_bg_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpBGPendingCleanup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup warpBGPendingCleanup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# run the ipp_cleanup.pl script on pending images +task warp.bg.cleanup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + active false + + task.exec + book npages warpBGPendingCleanup -var N + if ($N == 0) + periods -exec $RUNEXEC + break + end + if ($NETWORK == 0) break + + # look for new images in warpBGPendingCleanup (pantaskState == INIT) + book getpage warpBGPendingCleanup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpBGPendingCleanup $pageName pantaskState RUN + book getword warpBGPendingCleanup $pageName camera -var CAMERA + book getword warpBGPendingCleanup $pageName state -var CLEANUP_MODE + book getword warpBGPendingCleanup $pageName warp_bg_id -var WARP_BG_ID + book getword warpBGPendingCleanup $pageName workdir -var WORKDIR_TEMPLATE + book getword warpBGPendingCleanup $pageName exp_tag -var EXP_TAG + book getword warpBGPendingCleanup $pageName dbname -var DBNAME + + # specify choice of local or remote host based on camera and warp + set.workdir.by.camera $CAMERA $WARP_BG_ID $WORKDIR_TEMPLATE $default_host WORKDIR + sprintf logfile "%s/%s/%s.bgw.%s.cleanup.log" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID + + stdout $LOGDIR/warp.bg.cleanup.log + stderr $LOGDIR/warp.bg.cleanup.log + + # XXX is everything listed here needed? + $run = ipp_cleanup.pl --stage warp_bg --stage_id $WARP_BG_ID --camera $CAMERA --mode $CLEANUP_MODE --logfile $logfile + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpBGPendingCleanup $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword warpBGPendingCleanup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpBGPendingCleanup $options:0 pantaskState TIMEOUT + end +end Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 29670) +++ /trunk/ippTools/share/Makefile.am (revision 29671) @@ -21,4 +21,8 @@ bgtool_advancechip.sql \ bgtool_advancewarp.sql \ + bgtool_change_chiprun_state.sql \ + bgtool_change_imfile_data_state.sql \ + bgtool_change_warprun_state.sql \ + bgtool_change_skyfile_data_state.sql \ bgtool_chipinputs.sql \ bgtool_chip.sql \ @@ -29,4 +33,8 @@ bgtool_listchip.sql \ bgtool_listwarp.sql \ + bgtool_pendingcleanupchiprun.sql \ + bgtool_pendingcleanupchipimfile.sql \ + bgtool_pendingcleanupwarprun.sql \ + bgtool_pendingcleanupwarpskyfile.sql \ bgtool_revertchip.sql \ bgtool_revertwarp.sql \ Index: /trunk/ippTools/share/bgtool_change_chiprun_state.sql =================================================================== --- /trunk/ippTools/share/bgtool_change_chiprun_state.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_change_chiprun_state.sql (revision 29671) @@ -0,0 +1,10 @@ +UPDATE chipBackgroundRun +SET chipBackgroundRun.state = '%s', + chipBackgroundRun.magicked = IF(chipBackgroundRun.magicked != 0, -1, 0) +WHERE chip_bg_id = %lld +AND (SELECT COUNT(class_id) + FROM chipBackgroundImfile + WHERE + chipBackgroundImfile.chip_bg_id = chipBackgroundRun.chip_bg_id + AND data_state != '%s' + ) = 0 Index: /trunk/ippTools/share/bgtool_change_imfile_data_state.sql =================================================================== --- /trunk/ippTools/share/bgtool_change_imfile_data_state.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_change_imfile_data_state.sql (revision 29671) @@ -0,0 +1,5 @@ +UPDATE chipBackgroundImfile + JOIN chipBackgroundRun USING(chip_bg_id) +SET data_state = '%s', + chipBackgroundImfile.magicked = IF(chipBackgroundImfile.magicked != 0, -1, 0) +WHERE chip_bg_id = %lld AND class_id = '%s' Index: /trunk/ippTools/share/bgtool_change_skyfile_data_state.sql =================================================================== --- /trunk/ippTools/share/bgtool_change_skyfile_data_state.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_change_skyfile_data_state.sql (revision 29671) @@ -0,0 +1,5 @@ +UPDATE warpBackgroundSkyfile + JOIN warpBackgroundRun USING(warp_bg_id) +SET data_state = '%s', + warpBackgroundSkyfile.magicked = IF(warpBackgroundSkyfile.magicked != 0, -1, 0) +WHERE warp_bg_id = %lld AND skycell_id = '%s' Index: /trunk/ippTools/share/bgtool_change_warprun_state.sql =================================================================== --- /trunk/ippTools/share/bgtool_change_warprun_state.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_change_warprun_state.sql (revision 29671) @@ -0,0 +1,10 @@ +UPDATE warpBackgroundRun +SET state = '%s', + warpBackgroundRun.magicked = IF(warpBackgroundRun.magicked != 0, -1, 0) +WHERE warp_bg_id = %lld +AND (SELECT COUNT(skycell_id) + FROM warpBackgroundSkyfile + WHERE + warpBackgroundSkyfile.warp_bg_id = warpBackgroundRun.warp_bg_id + AND data_state != '%s' + ) = 0 Index: /trunk/ippTools/share/bgtool_pendingcleanupchipimfile.sql =================================================================== --- /trunk/ippTools/share/bgtool_pendingcleanupchipimfile.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_pendingcleanupchipimfile.sql (revision 29671) @@ -0,0 +1,15 @@ +SELECT + chipBackgroundImfile.*, + chipBackgroundRun.state, + chipBackgroundRun.workdir, + chipBackgroundRun.label +FROM chipBackgroundRun +JOIN chipBackgroundImfile + USING(chip_bg_id) +WHERE + ((chipBackgroundRun.state = 'goto_cleaned' AND (chipBackgroundImfile.data_state = 'full' + OR chipBackgroundImfile.data_state = 'update')) +OR + (chipBackgroundRun.state = 'goto_scrubbed' AND chipBackgroundImfile.data_state != 'scrubbed') +OR + (chipBackgroundRun.state = 'goto_purged' AND chipBackgroundImfile.data_state != 'purged')) Index: /trunk/ippTools/share/bgtool_pendingcleanupchiprun.sql =================================================================== --- /trunk/ippTools/share/bgtool_pendingcleanupchiprun.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_pendingcleanupchiprun.sql (revision 29671) @@ -0,0 +1,12 @@ +SELECT + chipBackgroundRun.*, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + rawExp.camera +FROM chipBackgroundRun + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE (chipBackgroundRun.state = 'goto_cleaned' + OR chipBackgroundRun.state = 'goto_purged' + OR chipBackgroundRun.state = 'goto_scrubbed' + ) + Index: /trunk/ippTools/share/bgtool_pendingcleanupwarprun.sql =================================================================== --- /trunk/ippTools/share/bgtool_pendingcleanupwarprun.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_pendingcleanupwarprun.sql (revision 29671) @@ -0,0 +1,13 @@ +SELECT warpBackgroundRun.*, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + rawExp.camera +FROM warpBackgroundRun + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE (warpBackgroundRun.state = 'goto_cleaned' + OR warpBackgroundRun.state = 'goto_purged' + OR warpBackgroundRun.state = 'goto_scrubbed') + Index: /trunk/ippTools/share/bgtool_pendingcleanupwarpskyfile.sql =================================================================== --- /trunk/ippTools/share/bgtool_pendingcleanupwarpskyfile.sql (revision 29671) +++ /trunk/ippTools/share/bgtool_pendingcleanupwarpskyfile.sql (revision 29671) @@ -0,0 +1,15 @@ +SELECT + warpBackgroundSkyfile.*, + warpBackgroundRun.state, + warpBackgroundRun.workdir, + warpBackgroundRun.label +FROM warpBackgroundRun +JOIN warpBackgroundSkyfile + USING(warp_bg_id) +WHERE + ((warpBackgroundRun.state = 'goto_cleaned' AND (warpBackgroundSkyfile.data_state = 'full' + OR warpBackgroundSkyfile.data_state = 'update')) +OR + (warpBackgroundRun.state = 'goto_scrubbed' AND warpBackgroundSkyfile.data_state != 'scrubbed') +OR + (warpBackgroundRun.state = 'goto_purged' AND warpBackgroundSkyfile.data_state != 'purged')) Index: /trunk/ippTools/src/bgtool.c =================================================================== --- /trunk/ippTools/src/bgtool.c (revision 29670) +++ /trunk/ippTools/src/bgtool.c (revision 29671) @@ -49,12 +49,20 @@ static bool revertwarpMode(pxConfig *config); static bool listwarpMode(pxConfig *config); -static bool tocleanchipMode(pxConfig *config); -static bool cleanedchipMode(pxConfig *config); -static bool tocleanwarpMode(pxConfig *config); -static bool cleanedwarpMode(pxConfig *config); + +static bool pendingcleanupchiprunMode(pxConfig *config); +static bool pendingcleanupchipimfileMode(pxConfig *config); +static bool tocleanedchipimfileMode(pxConfig *config); +static bool updatechipimfileMode(pxConfig *config); + +static bool pendingcleanupwarprunMode(pxConfig *config); +static bool pendingcleanupwarpskyfileMode(pxConfig *config); +static bool tocleanedwarpskyfileMode(pxConfig *config); + static bool exportchipMode(pxConfig *config); static bool importchipMode(pxConfig *config); static bool exportwarpMode(pxConfig *config); static bool importwarpMode(pxConfig *config); + +static bool validDataState(psString data_state); // Tables to import/export @@ -112,8 +120,11 @@ MODECASE(BGTOOL_MODE_REVERTWARP, revertwarpMode); MODECASE(BGTOOL_MODE_LISTWARP, listwarpMode); - MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode); - MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode); - MODECASE(BGTOOL_MODE_TOCLEANWARP, tocleanwarpMode); - MODECASE(BGTOOL_MODE_CLEANEDWARP, cleanedwarpMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, pendingcleanupchiprunMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, pendingcleanupchipimfileMode); + MODECASE(BGTOOL_MODE_TOCLEANEDCHIPIMFILE, tocleanedchipimfileMode); + MODECASE(BGTOOL_MODE_UPDATECHIPIMFILE, updatechipimfileMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPWARPRUN, pendingcleanupwarprunMode); + MODECASE(BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, pendingcleanupwarpskyfileMode); + MODECASE(BGTOOL_MODE_TOCLEANEDWARPSKYFILE, tocleanedwarpskyfileMode); MODECASE(BGTOOL_MODE_EXPORTCHIP, exportchipMode); MODECASE(BGTOOL_MODE_IMPORTCHIP, importchipMode); @@ -914,4 +925,5 @@ return true; } + static bool listchipMode(pxConfig *config) { @@ -978,5 +990,5 @@ } -static bool tocleanchipMode(pxConfig *config) +static bool pendingcleanupchiprunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); @@ -987,6 +999,7 @@ psMetadata *where = psMetadataAlloc(); pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); - - psString query = pxDataGet("bgtool_tocleanchip.sql"); + pxAddLabelSearchArgs(config, where, "-data_group", "chipBackgroundRun.data_group", "=="); + + psString query = pxDataGet("bgtool_pendingcleanupchiprun.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1035,24 +1048,17 @@ return true; } - -static bool cleanedchipMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); +static bool pendingcleanupchipimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - if (!pxIsValidCleanedState(state)) { - psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); - return false; - } - - if (!psListLength(where->list)) { - psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); - return false; - } - - psString query = pxDataGet("bgtool_cleanedchip.sql"); + psString query = pxDataGet("bgtool_pendingcleanupchipimfile.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1060,5 +1066,5 @@ } - { + if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); @@ -1067,6 +1073,142 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psError(psErrorCodeLast(), false, "database error"); + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool change_imfile_data_state(pxConfig *config, psString data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, class_id are required + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("bgtool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_bg_id, class_id)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("bgtool_change_chiprun_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_bg_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool tocleanedchipimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned"); +} + +static bool updatechipimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundImfile.chip_bg_id", "=="); + PXOPT_COPY_S64(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + + // optional + + PXOPT_LOOKUP_STR(data_state, config->args, "-set_data_state", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + if (!data_state && !fault) { + psError(PS_ERR_UNKNOWN, true, "must supply either -set_fault or -set_data_state"); + return false; + } + char *sep = ""; + psString query = psStringCopy("UPDATE chipBackgroundImfile SET "); + if (data_state) { + if (!validDataState(data_state)) { + return false; + } + psStringAppend(&query, "%s data_state = '%s'", sep, data_state); + sep = ", "; + } + if (fault) { + psStringAppend(&query, "%s fault = %d", sep, fault); + sep = ", "; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); return false; } @@ -1449,5 +1591,5 @@ return true; } - if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -1814,5 +1956,5 @@ } -static bool tocleanwarpMode(pxConfig *config) +static bool pendingcleanupwarprunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); @@ -1824,5 +1966,5 @@ pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); - psString query = pxDataGet("bgtool_tocleanwarp.sql"); + psString query = pxDataGet("bgtool_pendingcleanupwarprun.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1871,24 +2013,17 @@ return true; } - -static bool cleanedwarpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); +static bool pendingcleanupwarpskyfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - if (!pxIsValidCleanedState(state)) { - psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); - return false; - } - - if (!psListLength(where->list)) { - psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); - return false; - } - - psString query = pxDataGet("bgtool_cleanedwarp.sql"); + psString query = pxDataGet("bgtool_pendingcleanupwarpskyfile.sql"); if (!query) { psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); @@ -1896,5 +2031,5 @@ } - { + if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); @@ -1903,11 +2038,93 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psError(psErrorCodeLast(), false, "database error"); - return false; - } - psFree(query); - - return true; + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool change_skyfile_data_state(pxConfig *config, psString data_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // warp_id, skycell_id are required + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psString query = pxDataGet("bgtool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_bg_id, skycell_id)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("bgtool_change_warprun_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_bg_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool tocleanedwarpskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "cleaned"); } @@ -1935,2 +2152,19 @@ return importTables(config, infile, warpTables); } + +static bool validDataState(psString data_state) +{ + // NOTE: can't use pxIsValidState because + // update, scrubbed and purged are not supported for the backround stages (yet?) + if (!strcmp(data_state, "new") || + !strcmp(data_state, "full") || + !strcmp(data_state, "drop") || + !strcmp(data_state, "cleaned") || + !strcmp(data_state, "goto_cleaned") || + !strcmp(data_state, "error_cleaned")) { + return true; + } else { + psError(PS_ERR_UNKNOWN, true, "%s is not a valid value data_state", data_state); + return false; + } +} Index: /trunk/ippTools/src/bgtool.h =================================================================== --- /trunk/ippTools/src/bgtool.h (revision 29670) +++ /trunk/ippTools/src/bgtool.h (revision 29671) @@ -35,4 +35,5 @@ BGTOOL_MODE_REVERTCHIP, BGTOOL_MODE_LISTCHIP, + BGTOOL_MODE_UPDATECHIPIMFILE, // Warp stage BGTOOL_MODE_DEFINEWARP, @@ -45,9 +46,12 @@ BGTOOL_MODE_REVERTWARP, BGTOOL_MODE_LISTWARP, + BGTOOL_MODE_UPDATEWARPSKYFILE, // Cleanups - BGTOOL_MODE_TOCLEANCHIP, - BGTOOL_MODE_CLEANEDCHIP, - BGTOOL_MODE_TOCLEANWARP, - BGTOOL_MODE_CLEANEDWARP, + BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, + BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, + BGTOOL_MODE_TOCLEANEDCHIPIMFILE, + BGTOOL_MODE_PENDINGCLEANUPWARPRUN, + BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, + BGTOOL_MODE_TOCLEANEDWARPSKYFILE, // Exporting BGTOOL_MODE_EXPORTCHIP, Index: /trunk/ippTools/src/bgtoolConfig.c =================================================================== --- /trunk/ippTools/src/bgtoolConfig.c (revision 29670) +++ /trunk/ippTools/src/bgtoolConfig.c (revision 29671) @@ -192,14 +192,30 @@ psMetadataAddU64(listchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -tocleanchip - psMetadata *tocleanchipArgs = psMetadataAlloc(); - psMetadataAddStr(tocleanchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddBool(tocleanchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(tocleanchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -cleanedchip - psMetadata *cleanedchipArgs = psMetadataAlloc(); - psMetadataAddS64(cleanedchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); - psMetadataAddStr(cleanedchipArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + // -pendingcleanupchiprun + psMetadata *pendingcleanupchiprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list runs for specified label", NULL); + psMetadataAddStr(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "list runs for specified data_group", NULL); + psMetadataAddBool(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupchiprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupchipimfile + psMetadata *pendingcleanupchipimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupchipimfileArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id (required)", 0); + psMetadataAddBool(pendingcleanupchipimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupchipimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanedchipimfile + psMetadata *tocleanedchipimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedchipimfileArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(tocleanedchipimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddBool(tocleanedchipimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(tocleanedchipimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -updatechipimfile + psMetadata *updatechipimfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatechipimfileArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(updatechipimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id (required)", NULL); + psMetadataAddStr(updatechipimfileArgs, PS_LIST_TAIL, "-set_data_state", 0, "new value for data_state", NULL); + psMetadataAddS16(updatechipimfileArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); // -exportchip @@ -361,14 +377,29 @@ psMetadataAddU64(listwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -tocleanwarp - psMetadata *tocleanwarpArgs = psMetadataAlloc(); - psMetadataAddStr(tocleanwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddBool(tocleanwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(tocleanwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -cleanedwarp - psMetadata *cleanedwarpArgs = psMetadataAlloc(); - psMetadataAddS64(cleanedwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); - psMetadataAddStr(cleanedwarpArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + // -pendingcleanupwarprun + psMetadata *pendingcleanupwarprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddStr(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "list blocks for specified data_group", NULL); + psMetadataAddBool(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupwarprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupwarpskyfile + psMetadata *pendingcleanupwarpskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanupwarpskyfileArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id (required)", 0); + psMetadataAddBool(pendingcleanupwarpskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupwarpskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanwarpskyfile + psMetadata *tocleanedwarpskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedwarpskyfileArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id (required)", 0); + psMetadataAddStr(tocleanedwarpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id (required)", 0); + psMetadataAddStr(tocleanedwarpskyfileArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + + // -updatewarpskyfile + psMetadata *updatewarpskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(updatewarpskyfileArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(updatewarpskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id (required)", NULL); + psMetadataAddStr(updatewarpskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "new value for data_state", NULL); + psMetadataAddS16(updatewarpskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); // -exportwarp @@ -397,6 +428,8 @@ PXOPT_ADD_MODE("-revertchip", "", BGTOOL_MODE_REVERTCHIP, revertchipArgs); PXOPT_ADD_MODE("-listchip", "", BGTOOL_MODE_LISTCHIP, listchipArgs); - PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs); - PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs); + PXOPT_ADD_MODE("-pendingcleanupchiprun", "", BGTOOL_MODE_PENDINGCLEANUPCHIPRUN, pendingcleanupchiprunArgs); + PXOPT_ADD_MODE("-pendingcleanupchipimfile", "", BGTOOL_MODE_PENDINGCLEANUPCHIPIMFILE, pendingcleanupchipimfileArgs); + PXOPT_ADD_MODE("-tocleanedchipimfile", "", BGTOOL_MODE_TOCLEANEDCHIPIMFILE, tocleanedchipimfileArgs); + PXOPT_ADD_MODE("-updatechipimfile", "", BGTOOL_MODE_UPDATECHIPIMFILE, updatechipimfileArgs); PXOPT_ADD_MODE("-exportchip", "", BGTOOL_MODE_EXPORTCHIP, exportchipArgs); PXOPT_ADD_MODE("-importchip", "", BGTOOL_MODE_IMPORTCHIP, importchipArgs); @@ -411,6 +444,8 @@ PXOPT_ADD_MODE("-revertwarp", "", BGTOOL_MODE_REVERTWARP, revertwarpArgs); PXOPT_ADD_MODE("-listwarp", "", BGTOOL_MODE_LISTWARP, listwarpArgs); - PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs); - PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs); + PXOPT_ADD_MODE("-pendingcleanupwarprun", "", BGTOOL_MODE_PENDINGCLEANUPWARPRUN, pendingcleanupwarprunArgs); + PXOPT_ADD_MODE("-pendingcleanupwarpskyfile", "", BGTOOL_MODE_PENDINGCLEANUPWARPSKYFILE, pendingcleanupwarpskyfileArgs); + PXOPT_ADD_MODE("-tocleanedwarpskyfile", "", BGTOOL_MODE_TOCLEANEDWARPSKYFILE, tocleanedwarpskyfileArgs); + PXOPT_ADD_MODE("-updatewarpskyfile", "", BGTOOL_MODE_UPDATEWARPSKYFILE, updatewarpskyfileArgs); PXOPT_ADD_MODE("-exportwarp", "", BGTOOL_MODE_EXPORTWARP, exportwarpArgs); PXOPT_ADD_MODE("-importwarp", "", BGTOOL_MODE_IMPORTWARP, importwarpArgs);