Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 35307) +++ /trunk/dbconfig/changes.txt (revision 35308) @@ -2388,6 +2388,9 @@ CREATE TABLE lapGroup ( seq_id BIGINT, + tess_id VARCHAR(64), projection_cell VARCHAR(64), state VARCHAR(16), + label VARCHAR(654), + registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP fault SMALLINT, PRIMARY KEY(seq_id, projection_cell), Index: /trunk/dbconfig/lap.md =================================================================== --- /trunk/dbconfig/lap.md (revision 35307) +++ /trunk/dbconfig/lap.md (revision 35308) @@ -33,6 +33,9 @@ lapGroup METADATA seq_id S64 0 + tess_id STR 64 projection_cell STR 64 state STR 64 + label STR 64 + registered TAI NULL fault S16 0 end Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 35307) +++ /trunk/ippScripts/Build.PL (revision 35308) @@ -130,4 +130,5 @@ scripts/regenerate_background.pl scripts/relgroup_exp_list.pl + scripts/queuestaticsky.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /trunk/ippScripts/scripts/queuestaticsky.pl =================================================================== --- /trunk/ippScripts/scripts/queuestaticsky.pl (revision 35308) +++ /trunk/ippScripts/scripts/queuestaticsky.pl (revision 35308) @@ -0,0 +1,272 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use POSIX qw(strftime); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my ($seq_id, $tess_id, $projection_cell, $label, $dist_group, $dbname, $pretend, $simple, $verbose, $no_update); + +GetOptions( + 'seq_id=s' => \$seq_id, + 'tess_id=s' => \$tess_id, + 'projection_cell=s' => \$projection_cell, + 'label=s' => \$label, + 'dist_group=s' => \$dist_group, + 'dbname=s' => \$dbname, + 'pretend' => \$pretend, + 'simple' => \$simple, + 'no-update' => \$no_update, + 'verbose|v' => \$verbose, +) or pod2usage(2); + +unless (defined $seq_id and defined $tess_id and defined $projection_cell and defined $label and defined $dist_group) { + warn ("label, seq_id, tess_id, and projection_cell are required\n"); + pod2usage(2); +}; + + +my $missing_tools; +my $laptool = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1); +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $PS_EXIT_CONFIG_ERROR ); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# XXX get from site.config +my $workdirBase = "neb://\@HOST\@.0/gpc1"; + +my @filters; +{ + my $command = "$laptool -filtersforgroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => 0); + unless ($success) { + $error_code = $error_code >> 8; + my_die("failed to run $command $error_code\n", $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); + my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + + foreach my $entry (@$list) { + push @filters, $entry->{filter}; + } +} + +my $nFilters = scalar @filters; +print STDERR "lapGroup has $nFilters filters\n"; +&my_die("No filters found for $seq_id $tess_id $projection_cell", $PS_EXIT_PROG_ERROR) unless $nFilters; +&my_die("Unexpected number of filters found for $seq_id $tess_id $projection_cell", $PS_EXIT_PROG_ERROR) unless $nFilters <= 5; + +my $datestr = strftime "%Y/%m/%d", gmtime; + +my $staticsky_command="staticskytool -definebyquery" + . " -select_label $label" + . " -select_tess_id $tess_id" + . " -select_skycell_id $projection_cell%" + . " -set_workdir $workdirBase/$label/$datestr" + . " -set_label $label" + . " -set_data_group $label" + . " -set_dist_group $dist_group"; + +$staticsky_command .= " -pretend" if $pretend; +$staticsky_command .= " -simple" if $simple; +$staticsky_command .= " -dbname $dbname" if $dbname; + +for (my $num = $nFilters; $num > 0; $num--) { + + # set up the possible combinations of $num filters from @filters + + my $filter_combos = setup_filter_combos($num); + + # printcombos($filter_combos) if $verbose; + + foreach my $filter_combo (@$filter_combos) { + my $command = $staticsky_command; + foreach my $f (@$filter_combo) { + $command .= " -select_filter $f"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + my_die("failed to run $command $error_code\n", $error_code); + } + } +} +{ + my $command = "$laptool -updategroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -set_state full"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + carp("failed to run $command $error_code\n", $error_code); + exit $error_code; + } + } else { + print STDERR "Skipping $command\n"; + } +} + +exit 0; + +sub my_die +{ + my $msg = shift; + my $exit_code = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code; + + carp($msg); + + my $command = "$laptool -updategroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -set_fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + carp("failed to run $command $error_code\n", $error_code); + exit $error_code; + } + } else { + print STDERR "Skipping $command\n"; + } + exit $exit_code; +} + + +# The rest of this file contains some subroutines for setting up the filter combinations + + +sub setup_filter_combos { + my $combos = shift; + + my_die( "invalid combos arg: $combos", $PS_EXIT_PROG_ERROR) + unless defined $combos and ($combos eq "5" or $combos eq "4" or $combos eq "3" or $combos eq "2" or $combos eq "1"); + + if ($combos eq 5) { + my @ary; + push @ary, \@filters; + return \@ary; + } elsif ($combos eq 4) { + return makecombo4(); + } elsif ($combos eq 3) { + return makecombo3(); + } elsif ($combos eq 2) { + return makecombo2(); + } elsif ($combos eq 1) { + my @ary; + foreach my $f (@filters) { + my @ary2 = ($f); + push @ary, \@ary2 + } + return \@ary; + } else { + die "how did we get here?"; + } +} + + + +sub printcombos { + my $combos = shift; + foreach my $c (@$combos) { + my $str; + foreach my $f (@$c) { + $str .= " -select_filter $f"; + } + print "$str\n"; + } +} + + +sub makecombo4 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + for (my $k = $j + 1; $k < $numFilters; $k++) { + for (my $l = $k + 1; $l < $numFilters; $l++) { + my @combo = ($filters[$i], $filters[$j], $filters[$k], $filters[$l]); + push @combos, \@combo; + } + } + } + } + return \@combos; +} + +sub makecombo3 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + for (my $k = $j + 1; $k < $numFilters; $k++) { + my @combo = ($filters[$i], $filters[$j], $filters[$k]); + push @combos, \@combo; + } + } + } + return \@combos; +} + +sub makecombo2 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + my @combo = ($filters[$i], $filters[$j]); + push @combos, \@combo; + } + } + return \@combos; +} + + +__END__ + +=pod + +=head1 NAME + +queuesskylap - queue LAP staticsky runs + +=head1 SYNOPSIS + + XXX: pod TODO + + queuesskylap --ra_min --ra_max --dec_min --dec_max [--go] + + Index: /trunk/ippTasks/lapgroup.pro =================================================================== --- /trunk/ippTasks/lapgroup.pro (revision 35308) +++ /trunk/ippTasks/lapgroup.pro (revision 35308) @@ -0,0 +1,161 @@ +## lapgroup.pro : tasks for lap group management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/lapgroup +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendinglapGroup + +### Database lists +$lapgroup_DB = 0 + +### Check status of lapgroup tasks +macro lapgroup.status + book listbook pendinglapGroup +end + +### Reset lapgroup tasks +macro lapgroup.reset + book init pendinglapGroup +end + +### Turn lapgroup tasks on +macro lapgroup.on + task lapgroup.load + active true + end + task lapgroup.run + active true + end +end + +### Turn lapgroup tasks off +macro lapgroup.off + task lapgroup.load + active false + end + task lapgroup.run + active false + end +end + +### Load jobs for lapGroup +### Tasks are loaded into pendinglapGroup. +task lapgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/lapgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = laptool -pendinggroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$lapgroup_DB + $run = $run -dbname $DB:$lapgroup_DB + $lapgroup_DB ++ + if ($lapgroup_DB >= $DB:n) set lapgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendinglapGroup -key seq_id:tess_id:projection_cell -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendinglapGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendinglapGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task lapgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendinglapGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendinglapGroup (pantaskState == INIT) + book getpage pendinglapGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendinglapGroup $pageName pantaskState RUN + book getword pendinglapGroup $pageName seq_id -var SEQ_ID + book getword pendinglapGroup $pageName tess_id -var TESS_ID + book getword pendinglapGroup $pageName projection_cell -var PROJECTION_CELL + book getword pendinglapGroup $pageName label -var LABEL + book getword pendinglapGroup $pageName dist_group -var DIST_GROUP + book getword pendinglapGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/lapgroup.log + stderr $LOGSUBDIR/lapgroup.log + + host anyhost + + $run = queuestaticsky.pl --seq_id $SEQ_ID --tess_id $TESS_ID --projection_cell $PROJECTION_CELL --label $LABEL --dist_group $DIST_GROUP + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendinglapGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendinglapGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendinglapGroup $options:0 pantaskState TIMEOUT + end +end + Index: /trunk/ippTasks/release.pro =================================================================== --- /trunk/ippTasks/release.pro (revision 35308) +++ /trunk/ippTasks/release.pro (revision 35308) @@ -0,0 +1,167 @@ +## release.pro : tasks for release management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/release +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendingrelGroup + +### Database lists +$relgroup_DB = 0 + +### Check status of release tasks +macro release.status + book listbook pendingrelGroup +end + +### Reset release tasks +macro release.reset + book init pendingrelgroup +end + +### Turn release tasks on +macro release.on + task relgroup.load + active true + end + task relgroup.run + active true + end +end + +### Turn release tasks off +macro release.off + task relgroup.load + active false + end + task relgroup.run + active false + end +end + +### Load jobs for relGroup +### Tasks are loaded into pendingrelGroup. +task relgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/relgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = releasetool -pendingrelgroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$relgroup_DB + $run = $run -dbname $DB:$relgroup_DB + $relgroup_DB ++ + if ($relgroup_DB >= $DB:n) set relgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendingrelGroup -key group_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendingrelGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendingrelGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task relgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendingrelGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendingrelGroup (pantaskState == INIT) + book getpage pendingrelGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendingrelGroup $pageName pantaskState RUN + book getword pendingrelGroup $pageName group_id -var GROUP_ID + book getword pendingrelGroup $pageName rel_id -var REL_ID + book getword pendingrelGroup $pageName group_type -var GROUP_TYPE + book getword pendingrelGroup $pageName lap_id -var LAP_ID + book getword pendingrelGroup $pageName group_name -var GROUP_NAME + book getword pendingrelGroup $pageName release_name -var RELEASE_NAME + book getword pendingrelGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/relgroup.log + stderr $LOGSUBDIR/relgroup.log + + host anyhost + + $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME + if ("$GROUP_TYPE" == "lap") + $run = $run --lap_id $LAP_ID + else + $run = $run --group_name $GROUP_NAME + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendingrelGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendingrelGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendingrelGroup $options:0 pantaskState TIMEOUT + end +end + Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 35307) +++ /trunk/ippTools/share/Makefile.am (revision 35308) @@ -457,4 +457,7 @@ laptool_WSdiff_check.sql \ laptool_stacks.sql \ + laptool_pendinggroup.sql \ + laptool_revertgroup.sql \ + laptool_filtersforgroup.sql \ vptool_find_rawexp.sql \ vptool_pendingimfile.sql \ Index: /trunk/ippTools/share/laptool_filtersforgroup.sql =================================================================== --- /trunk/ippTools/share/laptool_filtersforgroup.sql (revision 35308) +++ /trunk/ippTools/share/laptool_filtersforgroup.sql (revision 35308) @@ -0,0 +1,2 @@ +SELECT filter +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) Index: /trunk/ippTools/share/laptool_pendinggroup.sql =================================================================== --- /trunk/ippTools/share/laptool_pendinggroup.sql (revision 35308) +++ /trunk/ippTools/share/laptool_pendinggroup.sql (revision 35308) @@ -0,0 +1,4 @@ +SELECT DISTINCT lapGroup.*, + lapRun.dist_group +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) +WHERE lapGroup.state ='new' AND lapGroup.fault = 0 Index: /trunk/ippTools/share/laptool_revertgroup.sql =================================================================== --- /trunk/ippTools/share/laptool_revertgroup.sql (revision 35308) +++ /trunk/ippTools/share/laptool_revertgroup.sql (revision 35308) @@ -0,0 +1,3 @@ +UPDATE lapGroup +SET fault = 0 +WHERE fault != 0 Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 35307) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 35308) @@ -2258,8 +2258,11 @@ CREATE TABLE lapGroup ( seq_id BIGINT, + tess_id VARCHAR(64), projection_cell VARCHAR(64), state VARCHAR(16), + label VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time group was registered fault SMALLINT, - PRIMARY KEY(seq_id, projection_cell), + PRIMARY KEY(seq_id, tess_id, projection_cell), KEY(state), KEY(fault), Index: /trunk/ippTools/src/laptool.c =================================================================== --- /trunk/ippTools/src/laptool.c (revision 35307) +++ /trunk/ippTools/src/laptool.c (revision 35308) @@ -34,4 +34,12 @@ static bool inactiveexpMode(pxConfig *config); +// Groups +static bool definegroupMode(pxConfig *config); +static bool pendinggroupMode(pxConfig *config); +static bool filtersforgroupMode(pxConfig *config); +static bool updategroupMode(pxConfig *config); +static bool revertgroupMode(pxConfig *config); +static bool listgroupMode(pxConfig *config); + # define MODECASE(caseName, func) \ case caseName: \ @@ -67,4 +75,12 @@ MODECASE(LAPTOOL_MODE_INACTIVEEXP, inactiveexpMode); + + MODECASE(LAPTOOL_MODE_DEFINEGROUP, definegroupMode); + MODECASE(LAPTOOL_MODE_PENDINGGROUP, pendinggroupMode); + MODECASE(LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupMode); + MODECASE(LAPTOOL_MODE_UPDATEGROUP, updategroupMode); + MODECASE(LAPTOOL_MODE_REVERTGROUP, revertgroupMode); + MODECASE(LAPTOOL_MODE_LISTGROUP, listgroupMode); + default: psAbort("invalid option (this should not happen)"); @@ -900,3 +916,443 @@ return(true); } - +// --------------------------- +// Group level (a collection of completed lapRuns for a given lapSequence projection cell and collection of filters + +static bool definegroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // the following insures that the -filter argument has been set up properly (adapted from pxAddLabelSearchArgs) + psMetadataItem *item = psMetadataLookup(config->args, "-filter"); + psAssert (item, "-filter argument not found in config->args"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", "filter"); + psAssert (item->data.list->n, "%s should at least have a place-holder", "filter"); + psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data; + psAssert (entry, "%s should at least have a place-holder", "filter"); + // end of checking + + // Now if the ony entry is the place-holder then the user supplied no -filter arguments + // which are required + if (!entry->data.str) { + psError(PXTOOLS_ERR_ARGUMENTS, true, "at least one -filter is required"); + return false; + } + + int nFilters = item->data.list->n; + + psString query = psStringCopy("SELECT seq_id, tess_id, projection_cell,\n"); + + // We construct a query joining completed lap runs with the same lapSequence.seq_id, + // tess_id, and projection_cell + + // select the lap_ids for the various filters. These are only used for debugging. + psStringAppend(&query, "lap_id_%d", 0); + for (int i = 1; i < nFilters; i++) { + psStringAppend(&query, ", lap_id_%d", i); + } + + + // sub query for each supplied filter + char * lapRunForFilter = "(\nSELECT seq_id, tess_id, projection_cell, lap_id as 'lap_id_%d'\n" + "FROM lapRun\n" + "WHERE filter LIKE '%s'\n" + " AND lapRun.seq_id = %"PRId64"\n" + " AND (lapRun.state = 'done' or lapRun.state = 'full')\n" + " ) as lap_%d\n"; + + // loop over supplied filters and flesh out the query using the format above + psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); + psMetadataItem *filterItem = NULL; + int i = -1; + while ((filterItem = psListGetAndIncrement(iter))) { + ++i; + if (i == 0) { + psStringAppend(&query, "\nFROM\n"); + } else { + psStringAppend(&query, "\nJOIN\n"); + } + + psString filter = filterItem->data.str; + psStringAppend(&query, lapRunForFilter, i, filter, seq_id, i); + + if (i != 0) { + psStringAppend(&query, "USING (seq_id, tess_id, projection_cell)\n"); + } + } + // now join to lapGroup + psStringAppend(&query, "\nLEFT JOIN lapGroup USING(seq_id, tess_id, projection_cell)\n"); + + // we only want projection cells which do not already of an entry + psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n"); + psStringAppend(&query, "\nORDER by projection_cell\n"); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + if (!ippdbPrintMetadatas(stdout, output, "new_lapGroups", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + psString last_projection_cell = NULL; + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psString tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString projection_cell = psMetadataLookupStr(NULL, row, "projection_cell"); + if (last_projection_cell && !strcmp(last_projection_cell, projection_cell)) { + // duplicate lap runs for a filter will generate multiple rows + // Since we care about projection_cells not lapRuns per se this is not problem. + // Skip any duplicates. + continue; + } + last_projection_cell = projection_cell; + + lapGroupRow *group = lapGroupRowAlloc( + seq_id, + tess_id, + projection_cell, + "new", // state + label, + now, // registered + 0 // fault + ); + if (!group) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapGroup object"); + psFree(output); + psFree(now); + return(false); + } + + + if (!lapGroupInsertObject(config->dbh, group)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return(true); + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(now); + psFree(output); + + return(true); +} + +static bool pendinggroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "=="); + + psString query = pxDataGet("laptool_pendinggroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapGroup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool filtersforgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + psString query = pxDataGet("laptool_filtersforgroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "filters", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool updategroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // check that all of the keys that define a lapGroup are supplied + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + psMetadata *values = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); + PXOPT_COPY_S16(config->args, values, "-set_fault", "fault", "=="); + + if (!psListLength(values->list)) { + psFree(values); + psFree(where); + psError(PXTOOLS_ERR_ARGUMENTS, true, "must set at least one column"); + return false; + } + + long rows = psDBUpdateRows(config->dbh, "lapGroup", where, values); + if (rows < 1) { + psFree(values); + psError(PXTOOLS_ERR_SYS, true, "failed to update lapGroup for %" PRId64 " %s %s", seq_id, tess_id, projection_cell); + return false; + } + psFree(values); + + return(true); +} + +static bool revertgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required"); + return false; + } + + psString query = pxDataGet("laptool_revertgroup.sql"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("laptool", PS_LOG_INFO, "Updated %" PRIu64 " lapGroups", affected); + + return true; +} + +static bool listgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psError(PXTOOLS_ERR_SYS, true, "not yet implemented"); + return false; + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-seq_name", "name", "LIKE"); + + psString query = pxDataGet("laptool_listsequence.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /trunk/ippTools/src/laptool.h =================================================================== --- /trunk/ippTools/src/laptool.h (revision 35307) +++ /trunk/ippTools/src/laptool.h (revision 35308) @@ -20,5 +20,11 @@ LAPTOOL_MODE_UPDATEEXP, LAPTOOL_MODE_DIFFCHECK, - LAPTOOL_MODE_INACTIVEEXP + LAPTOOL_MODE_INACTIVEEXP, + LAPTOOL_MODE_DEFINEGROUP, + LAPTOOL_MODE_PENDINGGROUP, + LAPTOOL_MODE_FILTERSFORGROUP, + LAPTOOL_MODE_UPDATEGROUP, + LAPTOOL_MODE_REVERTGROUP, + LAPTOOL_MODE_LISTGROUP, } laptoolMode; Index: /trunk/ippTools/src/laptoolConfig.c =================================================================== --- /trunk/ippTools/src/laptoolConfig.c (revision 35307) +++ /trunk/ippTools/src/laptoolConfig.c (revision 35308) @@ -97,5 +97,4 @@ ADD_OPT(S64, updaterunArgs, "-set_quick_sass_id", "set quick stack sass_id", 0); ADD_OPT(S64, updaterunArgs, "-set_final_sass_id", "set final stack sass_id", 0); - // -pendingexp @@ -150,4 +149,52 @@ ADD_OPT(U64, inactiveexpArgs, "-limit", "limit result set to N items", 0); + // -definegroup + psMetadata *definegroupArgs = psMetadataAlloc(); + ADD_OPT(S64, definegroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, definegroupArgs, "-set_label", "set label (required)", 0); + + psMetadataAddStr(definegroupArgs, PS_LIST_TAIL, + "-filter", PS_META_DUPLICATE_OK, + "search by filter (LIKE comparison, multiple OK)", NULL); + ADD_OPT(Bool,definegroupArgs, "-pretend", "do not actuallym modify the database", false); + ADD_OPT(Bool,definegroupArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, definegroupArgs, "-limit", "limit result set to N items", 0); + + // -pendinggroup + psMetadata *pendinggroupArgs = psMetadataAlloc(); + psMetadataAddStr(pendinggroupArgs, PS_LIST_TAIL, + "-label", PS_META_DUPLICATE_OK, + "search by label (multiple OK)", NULL); + ADD_OPT(S64, pendinggroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + + ADD_OPT(Bool,pendinggroupArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, pendinggroupArgs, "-limit", "limit result set to N items", 0); + + // -updategroup + psMetadata *updategroupArgs = psMetadataAlloc(); + ADD_OPT(S64, updategroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, updategroupArgs, "-tess_id", "search by tess_id (required)", 0); + ADD_OPT(Str, updategroupArgs, "-projection_cell", "search by projection_cell (required)", 0); + ADD_OPT(Str, updategroupArgs, "-set_state", "set state", 0); + ADD_OPT(S16, updategroupArgs, "-set_fault", "set fault code", INT16_MAX); + ADD_OPT(Str, updategroupArgs, "-set_label", "set label", 0); + + // -revertgroup + psMetadata *revertgroupArgs = psMetadataAlloc(); + psMetadataAddStr(revertgroupArgs, PS_LIST_TAIL, + "-label", PS_META_DUPLICATE_OK, + "search by label (multiple OK)", NULL); + ADD_OPT(S64, revertgroupArgs, "-seq_id", "search by lap sequence ID", 0); + ADD_OPT(Str, revertgroupArgs, "-tess_id", "search by tess_id", 0); + ADD_OPT(Str, revertgroupArgs, "-projection_cell", "search by projection_cell", 0); + ADD_OPT(S16, revertgroupArgs, "-fault", "fault code", INT16_MAX); + + // -filtersforgroup + psMetadata *filtersforgroupArgs = psMetadataAlloc(); + ADD_OPT(S64, filtersforgroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, filtersforgroupArgs, "-tess_id", "search by tess_id (required)", 0); + ADD_OPT(Str, filtersforgroupArgs, "-projection_cell", "search by projection_cell (required)", 0); + ADD_OPT(Bool,filtersforgroupArgs, "-simple", "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); @@ -166,4 +213,9 @@ PXOPT_ADD_MODE("-diffcheck", "", LAPTOOL_MODE_DIFFCHECK, diffcheckArgs); PXOPT_ADD_MODE("-inactiveexp", "", LAPTOOL_MODE_INACTIVEEXP, inactiveexpArgs); + PXOPT_ADD_MODE("-definegroup", "", LAPTOOL_MODE_DEFINEGROUP, definegroupArgs); + PXOPT_ADD_MODE("-pendinggroup", "", LAPTOOL_MODE_PENDINGGROUP, pendinggroupArgs); + PXOPT_ADD_MODE("-filtersforgroup", "", LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupArgs); + PXOPT_ADD_MODE("-updategroup", "", LAPTOOL_MODE_UPDATEGROUP, updategroupArgs); + PXOPT_ADD_MODE("-revertgroup", "", LAPTOOL_MODE_REVERTGROUP, revertgroupArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {