Index: trunk/dbconfig/add.md =================================================================== --- trunk/dbconfig/add.md (revision 28178) +++ trunk/dbconfig/add.md (revision 28181) @@ -12,4 +12,7 @@ note STR 255 image_only BOOL f + minidvodb BOOL f + minidvodb_group STR 64 + minidvodb_name STR 64 END @@ -18,4 +21,5 @@ dtime_addstar F32 0.0 path_base STR 255 + dvodb_path STR 255 fault S16 0 # Key NOT NULL END @@ -24,3 +28,5 @@ label STR 64 # Primary Key END - + + + Index: trunk/dbconfig/ipp.m4 =================================================================== --- trunk/dbconfig/ipp.m4 (revision 28178) +++ trunk/dbconfig/ipp.m4 (revision 28181) @@ -24,4 +24,5 @@ include(label.md) include(magic.md) +include(minidvodb.md) include(calibration.md) include(flatcorr.md) Index: trunk/dbconfig/minidvodb.md =================================================================== --- trunk/dbconfig/minidvodb.md (revision 28181) +++ trunk/dbconfig/minidvodb.md (revision 28181) @@ -0,0 +1,20 @@ +minidvodbRun METADATA + minidvodb_id S64 0 + minidvodb_name STR 64 + minidvodb_group STR 64 + minidvodb_path STR 255 + mergedvodb_path STR 255 + state STR 64 + creation_date UTC 0001-01-01T00:00:00Z +END + +minidvodbProcessed METADATA + minidvodb_id S64 0 + merge_order S64 0 + dtime_resort F32 0.0 + dtime_relphot F32 0.0 + dtime_merge F32 0.0 + epoch UTC 0001-01-01T00:00:00Z + mergedvodb_path STR 255 + fault S16 0 +END Index: trunk/ippScripts/Build.PL =================================================================== --- trunk/ippScripts/Build.PL (revision 28178) +++ trunk/ippScripts/Build.PL (revision 28181) @@ -62,4 +62,6 @@ scripts/magic_destreak_cleanup.pl scripts/magic_destreak_defineruns.pl + scripts/minidvodb_createdb.pl + scripts/minidvodb_load.pl scripts/ippdb.pl scripts/ipp_cleanup.pl Index: trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- trunk/ippScripts/scripts/addstar_run.pl (revision 28178) +++ trunk/ippScripts/scripts/addstar_run.pl (revision 28181) @@ -37,5 +37,5 @@ } -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $image_only, $verbose, $no_update, +my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -48,4 +48,7 @@ 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database + 'minidvodb' => \$minidvodb, # use minidvodb? + 'minidvodb_name|w=s'=> \$minidvodb_name, # miniDVO database name + 'minidvodb_group|w=s' => \$minidvodb_group, # miniDVO database group 'image-only' => \$image_only, # Print to stdout 'verbose' => \$verbose, # Print to stdout @@ -58,5 +61,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --exp_tag --add_id --camera --outroot", + -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot", -exitval => 3, ) unless @@ -65,6 +68,9 @@ defined $outroot and defined $camroot and + defined $dvodb and defined $camera; - +if ($minidvodb && !defined($minidvodb_group)) { + my_die( "missing minidvodb_group", $add_id, 3 ); + } my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $add_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -103,6 +109,37 @@ my $dtime_addstar = 0; +if (defined $dvodbReal) { + if ($minidvodb) { + + if (!defined($minidvodb_name)) { + #take the active one, if it's not defined on the command line + #reverts would have this already set, for example. + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + + } + # tack on the minidvodb part to the db. + $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; + + } unless ($no_op) { - if (defined $dvodbReal) { + print $dvodbReal; ## addstar can either save the full set of detections, or just @@ -142,6 +179,7 @@ $fpaCommand .= " -path_base $outroot"; $fpaCommand .= " -dtime_addstar $dtime_addstar"; +$fpaCommand .= " -dvodb_path $dvodbReal"; $fpaCommand .= " -dbname $dbname" if defined $dbname; - +print $fpaCommand; # Add the result into the database unless ($no_update) { @@ -171,4 +209,5 @@ $command .= " -addprocessedexp"; $command .= " -fault $exit_code"; + $command .= " -dvodb_path $dvodbReal" if defined $dvodbReal; $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); Index: trunk/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 28181) +++ trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 28181) @@ -0,0 +1,255 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $dvodb,$minidvodb_interval, $minidvodb_group, $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'dvodb|w=s' => \$dvodb, # output DVO database + 'interval|w=s' => \$minidvodb_interval, #interval between creation of minidvodbs (default = 1day) + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --camera --dvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + defined $camera and + defined $outroot and + defined $dvodb; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_group , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +# XXX This is now not used: do we still need it? + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $dvodbReal; +if (defined $dvodb) { + $dvodbReal = $ipprc->dvo_catdir( $dvodb ); # catdir for DVO + $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ); +} else { + warn("dvodb undefined:\n"); + exit(4); +} + +if (!defined $minidvodb_interval) { + $minidvodb_interval = 1; +} + + +unless ($no_op) { + + + +#see if there is already one in new state + my $fpaCommand1 = "$addtool -listminidvodbrun"; + $fpaCommand1 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand1 .= " -state 'new'"; + $fpaCommand1 .= " -dbname $dbname" if defined $dbname; + + +unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand1, verbose => $verbose); + + + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to list minidvodb database: $error_code\n"); + exit($error_code); + } + + if (scalar(@{$stdout_buf})) { + $error_code = 3; + warn("Unwilling to create minidvodb, one already exists in new state: $error_code\n"); + exit($error_code); + } + + my $creation_date; + my $addRun_count; + my $minidvodb_name; +#find the active one's date, and find out if it has more than 1 addRun in it + + my $fpaCommand2 = "$addtool -listminidvodbrun"; + $fpaCommand2 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand2 .= " -state 'active'"; + $fpaCommand2 .= " -limit 1"; + $fpaCommand2 .= " -dbname $dbname" if defined $dbname; + +#print $fpaCommand2; + + + my ( $success2, $error_code2, $full_buf2, $stdout_buf2, $stderr_buf2 ) = + run(command => $fpaCommand2, verbose => $verbose); + &my_die( "Unable to get listminidvodbrun",$minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success2; + # if it didn't list something in active state (what?) then we definitely need to create a new one + if (defined(@$stdout_buf2)) { + my $metadata2 = $mdcParser->parse(join "", @$stdout_buf2) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components2 = parse_md_list($metadata2) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp2 = $$components2[0]; + $minidvodb_name = $comp2->{minidvodb_name}; + $creation_date = $comp2->{creation_date}; + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + if (!defined($creation_date)) { + &my_die("Unable to parse creation_date", $minidvodb_group, $PS_EXIT_PROG_ERROR); + } + } else { + $create_new = 1; #this is to force it to make a new one + } + + #find the number of add_ids that have been proccessed + my $fpaCommand3 = "$addtool -checkminidvodbrunaddrun"; + $fpaCommand3 .= " -minidvodb_group '$minidvodb_group'"; + $fpaCommand3 .= " -state 'active'"; + $fpaCommand3 .= " -minidvodb_name '$minidvodb_name'" if defined $minidvodb_name; + $fpaCommand3 .= " -limit 1"; + $fpaCommand3 .= " -dbname $dbname" if defined $dbname; + + +my ( $success3, $error_code3, $full_buf3, $stdout_buf3, $stderr_buf3 ) = + run(command => $fpaCommand3, verbose => $verbose); + &my_die( "Unable to get checkminidvodbunaddrun", $minidvodb_group, $PS_EXIT_SYS_ERROR) unless $success3; + + if (defined(@$stdout_buf3)) { #checkminidvodb returns nothing IF there have been no addruns added to the db yet + my $metadata3 = $mdcParser->parse(join "", @$stdout_buf3) or + &my_die("Unable to parse metadata config", $minidvodb_group, $PS_EXIT_PROG_ERROR); + + my $components3 = parse_md_list($metadata3) or + &my_die("Unable to parse metadata list", $minidvodb_group, $PS_EXIT_PROG_ERROR); + my $comp = $$components3[0]; + $addRun_count = $comp->{addRun_count}; + } + if (!defined($addRun_count)) { + ## there's nothing to parse if there's nothing + $addRun_count = 0; + } + + + if ($addRun_count > 30000) { + #it's too big, create_new + $create_new = 1; + } + if ($create_new == 0) { + my $parser = DateTime::Format::Strptime->new( pattern => '%Y-%m-%dT%H:%M:%S', time_zone => "HST" ); + my $creation_dt = $parser->parse_datetime( $creation_date )->mjd; + if ($mjd_start- $creation_dt > $minidvodb_interval && $addRun_count > 0 ) { + #db is old and has stuff in it, want to create_new + $create_new = 1; + } + } + + +} +#create the minidvodb entry (well, the command for it) + my $fpaCommand = "$addtool -addminidvodbrun"; + $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + unless ($no_update or !$create_new) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $minidvodb_group = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippScripts/scripts/minidvodb_merge.pl =================================================================== --- trunk/ippScripts/scripts/minidvodb_merge.pl (revision 28181) +++ trunk/ippScripts/scripts/minidvodb_merge.pl (revision 28181) @@ -0,0 +1,240 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_merge; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb, $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb|w=s' => \$mergedvodb, # output DVO database + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $mergedvodb and + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodb, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} + +my $dtime_addstar = 0; +#my $dtime_relphot = 0; + +unless ($no_op) { + if (defined $mergedvodbReal) { + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $mergedvodbReal"; + + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $mergedvodb, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + print "addstar -resort time $dtime_addstar\n"; + } + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $mergedvodbReal"; + + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $mergedvodb, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $command = "$addtool -listminidvodbrun -state mergedsdfsad -minidvodb_group " . $minidvodb_group; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of minidvodbruns", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $minidvodb_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $minidvodb_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $minidvodb_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + $this_is_the_first = 0; + } + } + + { + my $merge_command; + if ($this_is_the_first) { + $merge_command = "cp -rp $minidvodb $mergedvodb" + } else { + $merge_command = "$dvomerge $minidvodb into $mergedvodb" + } + + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $mergedvodb, $error_code); + } + + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; + } + + + { + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $mergedvodb, $error_code); + } + } + + } else { + &my_die("dvodb: $mergedvodb not found", $mergedvodb, $PS_EXIT_UNKNOWN_ERROR); + } +} else { + print "skipping processing for $mergedvodbReal\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -mergedvodb_path $mergedvodbReal" if defined $mergedvodbReal; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_relphot $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_relphot $dtime_merge" if defined $dtime_merge; + $command .= " -dbname $dbname" if defined $dbname; + + + + #print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 28178) +++ trunk/ippTasks/Makefile.am (revision 28181) @@ -32,4 +32,5 @@ dqstats.pro \ science.cleanup.pro \ + minidvodb.pro \ nightly_stacks.pro \ lossy_compress.pro Index: trunk/ippTasks/addstar.pro =================================================================== --- trunk/ippTasks/addstar.pro (revision 28178) +++ trunk/ippTasks/addstar.pro (revision 28181) @@ -45,7 +45,9 @@ end + # this variable will cycle through the known database names $addstar_DB = 0 $addstar_revert_DB = 0 + # select images ready for addstar analysis @@ -64,5 +66,5 @@ task.exec - if ($LABEL:n == 0) break + # if ($LABEL:n == 0) break $run = addtool -pendingexp if ($DB:n == 0) @@ -134,4 +136,7 @@ book getword addPendingExp $pageName reduction -var REDUCTION book getword addPendingExp $pageName dvodb -var DVODB + book getword addPendingExp $pageName minidvodb -var MINIDVODB + book getword addPendingExp $pageName minidvodb_name -var MINIDVODB_NAME + book getword addPendingExp $pageName minidvodb_group -var MINIDVODB_GROUP book getword addPendingExp $pageName image_only -var IMAGE_ONLY book getword addPendingExp $pageName dbname -var DBNAME @@ -164,4 +169,11 @@ $run = $run --image-only end + if ("$MINIDVODB" == "T") + $run = $run --minidvodb + $run = $run --minidvodb_group + if (("$MINIDVODB_NAME" != "NULL") && ("$MINIDVODB_NAME" != "(null)")) + $run = $run --minidvodb_name #have addstar_run.pl grab the 'active' name if it is NULL + end + end add_standard_args run Index: trunk/ippTasks/minidvodb.pro =================================================================== --- trunk/ippTasks/minidvodb.pro (revision 28181) +++ trunk/ippTasks/minidvodb.pro (revision 28181) @@ -0,0 +1,725 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?haveBooks)) + book create MINIDVODB + book create MINIDVODB_MERGE + book create MINIDVODB_CREATE + book create MINIDVODB_ACTIVE + $haveBooks = TRUE +end + +$MINIDVODB_DB = 0 + +$MINIDVODB_WAIT_DB = 0 +$MINIDVODB_MERGE_DB = 0 +$MINIDVODB_CREATE_DB = 0 +$MINIDVODB_ACTIVE_DB = 0 + +book init minidvodbWaitlist +book init minidvodbMergelist +book init minidvodbCreatelist +book init minidvodbActivelist + +macro minidvodb.create.status + book listbook minidvodbCreatelist +end + +macro minidvodb.create.reset + book init minidvodbCreatelist +end + +macro minidvodb.active.status + book listbook minidvodbActivelist +end + +macro minidvodb.active.reset + book init minidvodbActivelist +end + + +macro minidvodb.wait.status + book listbook minidvodbWaitlist +end + +macro minidvodb.wait.reset + book init minidvodbWaitlist +end + +macro minidvodb.merge.status + book listbook minidvodbMergelist +end + +macro minidvodb.merge.reset + book init minidvodbMergelist +end + +#this is the create task, there is no load, it just runs +macro minidvodb.create.on + task minidvodb.create + active true + end +end +macro minidvodb.create.off + task minidvodb.create + active false + end +end +##these are the tasks that flip from new -> active +macro minidvodb.active.on + task minidvodb.active.load + active true + end + task minidvodb.active.run + active true + end +end +macro minidvodb.active.off + task minidvodb.active.load + active false + end + task minidvodb.active.run + active false + end +end + +##these are the tasks that check to see if addRun processing is finished +##for a minidvodb in state wait +macro minidvodb.wait.on + task minidvodb.wait.load + active true + end + task minidvodb.wait.run + active true + end +end +macro minidvodb.wait.off + task minidvodb.wait.load + active false + end + task minidvodb.wait.run + active false + end +end + +##these merge the dbs + +macro minidvodb.merge.on + task minidvodb.merge.load + active true + end + task minidvodb.merge.run + active true + end +end +macro minidvodb.merge.off + task minidvodb.merge.load + active false + end + task minidvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.minidvodb + if ($0 != 5) + echo "USAGE: add.minidvodb (minidvodb_group) (dvodb) (interval days) (camera)" + break + end + + #wait - shoudl be renamed MINIDVODB_WAIT + book newpage MINIDVODB $1 + book setword MINIDVODB $1 MINIDVODB_GROUP $1 + book setword MINIDVODB $1 DVODB $2 + book setword MINIDVODB $1 STATE PENDING + #merge + book newpage MINIDVODB_MERGE $1 + book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_MERGE $1 DVODB $2 + book setword MINIDVODB_MERGE $1 STATE PENDING + #active + book newpage MINIDVODB_ACTIVE $1 + book setword MINIDVODB_ACTIVE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_ACTIVE $1 DVODB $2 + book setword MINIDVODB_ACTIVE $1 STATE PENDING + + #create note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out. + book newpage MINIDVODB_CREATE $1 + book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_CREATE $1 DVODB $2 + book setword MINIDVODB_CREATE $1 DVODB_DAYS $3 + book setword MINIDVODB_CREATE $1 CAMERA $4 + book setword MINIDVODB_CREATE $1 STATE PENDING +end + +macro del.minidvodb + if ($0 != 2) + echo "USAGE: del.minidvodb (minidvodb)" + break + end + book delpage MINIDVODB $1 + book delpage MINIDVODB_MERGE $1 + book delpage MINIDVODB_CREATE $1 + book delpage MINIDVODB_ACTIVE $1 +end + +macro show.minidvodb + if ($0 != 1) + echo "USAGE: show.minidvodb" + break + end + echo "minidvodb wait" + book listbook MINIDVODB + echo "minidvodb merge" + book listbook MINIDVODB_MERGE + echo "minidvodb create" + book listbook MINIDVODB_CREATE + echo "minidvodb_active" + book listbook MINIDVODB_ACTIVE +end + + + + + + + + +task minidvodb.wait.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.wait.load.log + + task.exec + book npages MINIDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB $i -var minidvodb_group + book setword MINIDVODB $minidvodb_group STATE NEW + end + book getpage MINIDVODB 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +#using check as opposed to list because it sees if it is done with the addRun state yet. + $run = addtool -checkminidvodbrunaddrun -state waiting + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbWaitlist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbWaitlist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbWaitlist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.wait.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbWaitlist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbWaitlist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbWaitlist $pageName pantaskState RUN + book getword minidvodbWaitlist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbWaitlist $pageName state -var STATE + stdout $LOGDIR/minidvodb.wait.run.log + stderr $LOGDIR/minidvodb.wait.run.log + + $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state to_be_merged + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbWaitlist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + echo "hostname: $JOB_HOSTNAME" + process_exit minidvodbWaitlist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbWaitlist $options:0 pantaskState TIMEOUT + end +end + + + +task minidvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.merge.load.log + + task.exec + book npages MINIDVODB_MERGE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_MERGE $i -var minidvodb_group + book setword MINIDVODB_MERGE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + #finds the minidvodbs in a state of 'to_be_merged' + $run = addtool -listminidvodbrun -state to_be_merged + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbMergelist -var N + if ($N == 0) break + + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbMergelist $pageName pantaskState RUN + book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP + book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH + book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.merge.run.log + stderr $LOGDIR/minidvodb.merge.run.log + + $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit minidvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbMergelist $options:0 pantaskState TIMEOUT + end +end + + +#this is the complicated script - it creates the minidvodbRuns +#since none may exist by default, I decided not to have a load.task +#it calls a perl script, which creates a new one if: +#there is none in an active state (true if there is none at all) +#the current one is older than the dvodb_age +#there are none in new (we have a task that moves it to 'active') +#the current active one has > 30000 add_ids in it + +#also confusing: it succeeds if it doesn't create it (if it doesn't want to create one) +#and it succeeds if it creates one. (otherwise there would be a lot of false failures, +# like in replication) +#it fails if it has problems with one of the addtool commands in the script. + +task minidvodb.create + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 240 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.create.log + + task.exec + + periods -poll 60 +#wait a bit before trying again + + book npages MINIDVODB_CREATE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_CREATE $i -var minidvodb_group + book setword MINIDVODB_CREATE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + + book setword MINIDVODB_CREATE $minidvodb_group STATE DONE + book getword MINIDVODB_CREATE $minidvodb_group DVODB -var dvodb + book getword MINIDVODB_CREATE $minidvodb_group DVODB_DAYS -var dvodb_days + book getword MINIDVODB_CREATE $minidvodb_group CAMERA -var camera + $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb_group $minidvodb_group --interval $dvodb_days + + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run --dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + command $run + end + + # success + task.exit 0 + if ($VERBOSE > 2) + showcommand + end + end + + # locked list + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + +task minidvodb.active.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/minidvodb.active.load.log + + task.exec + book npages MINIDVODB_ACTIVE -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + if ("$minidvodb_group" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage MINIDVODB_ACTIVE $i -var minidvodb_group + book setword MINIDVODB_ACTIVE $minidvodb_group STATE NEW + end + book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW + + # Select different database + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + + + $run = addtool -listminidvodbrun -state new + $run = $run -minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end + add_poll_args run + + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout minidvodbActivelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook minidvodbActivelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup minidvodbActivelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +task minidvodb.active.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + + book npages minidvodbActivelist -var N + if ($N == 0) break + + # look for new images in minidvodbActivelist (pantaskState == INIT) + book getpage minidvodbActivelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword minidvodbActivelist $pageName pantaskState RUN + book getword minidvodbActivelist $pageName minidvodb_group -var MINIDVODB_GROUP + stdout $LOGDIR/minidvodb.active.run.log + stderr $LOGDIR/minidvodb.active.run.log + + $run = addtool -flipminidvodbrun -minidvodb_group $MINIDVODB_GROUP + + +if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MINIDVODB_DB + $run = $run -dbname $DB:$MINIDVODB_DB + $MINIDVODB_DB ++ + if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 + end +# save the pageName for future reference below + options $pageName + + + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end +# default exit status + task.exit default + process_exit minidvodbActivelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + #showcommand crash + echo "hostname: $JOB_HOSTNAME" + # Set a fault code in the database + + process_exit minidvodbActivelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword minidvodbActivelist $options:0 pantaskState TIMEOUT + end +end + Index: trunk/ippTasks/survey.pro =================================================================== --- trunk/ippTasks/survey.pro (revision 28178) +++ trunk/ippTasks/survey.pro (revision 28181) @@ -207,10 +207,11 @@ macro survey.add.addstar - if ($0 != 3) - echo "USAGE: survey.add.addstar (label) (dvodb)" + if ($0 != 4) + echo "USAGE: survey.add.addstar (label) (dvodb) (minidvodb_group)" break end book newpage SURVEY_ADDSTAR $1 book setword SURVEY_ADDSTAR $1 DVODB $2 + book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $3 book setword SURVEY_ADDSTAR $1 STATE PENDING end @@ -638,4 +639,5 @@ book setword SURVEY_ADDSTAR $label STATE DONE book getword SURVEY_ADDSTAR $label DVODB -var dvodb + book getword SURVEY_ADDSTAR $label MINIDVODB_GROUP -var minidvodb_group # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb @@ -648,5 +650,5 @@ end - $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb + $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group echo $run command $run Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 28178) +++ trunk/ippTools/share/Makefile.am (revision 28181) @@ -4,12 +4,19 @@ dist_pkgdata_DATA = \ + addtool_checkminidvodbaddrun.sql \ addtool_donecleanup.sql \ addtool_find_cam_id.sql \ addtool_find_pendingexp.sql \ + addtool_find_pendingmergeprocess.sql \ addtool_find_processedexp.sql \ + addtool_find_minidvodbprocessed.sql \ + addtool_find_minidvodbrun.sql \ addtool_pendingcleanupexp.sql \ addtool_pendingcleanuprun.sql \ addtool_queue_cam_id.sql \ + addtool_queue_minidvodbrun.sql \ + addtool_reset_faulted_minidvodbruns.sql \ addtool_reset_faulted_runs.sql \ + addtool_revertminidvodbprocessed.sql \ addtool_revertprocessedexp.sql \ camtool_donecleanup.sql \ Index: trunk/ippTools/share/addtool_checkminidvodbaddrun.sql =================================================================== --- trunk/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 28181) +++ trunk/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 28181) @@ -0,0 +1,20 @@ +-- this selects the minidvodbs that have complete addRuns +SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count +FROM minidvodbRun +JOIN + (SELECT minidvodb_name, count(state) as cnt2, cnt, minidbg, minidbn + FROM addRun LEFT JOIN + -- this select grabs (and counts) all the addRuns with any state + -- and groups them by the name + (SELECT minidvodb_group as minidbg, minidvodb_name as minidbn, count(state) + AS cnt + FROM addRun + GROUP BY minidvodb_name ) + AS foo1 ON minidvodb_name = minidbn + -- the second select grabs and counts all the addRuns with state + -- of 'full' + WHERE addRun.state = 'full' + GROUP BY minidvodb_name ) +AS foo2 +USING(minidvodb_name) Index: trunk/ippTools/share/addtool_find_minidvodbprocessed.sql =================================================================== --- trunk/ippTools/share/addtool_find_minidvodbprocessed.sql (revision 28181) +++ trunk/ippTools/share/addtool_find_minidvodbprocessed.sql (revision 28181) @@ -0,0 +1,8 @@ +SELECT + minidvodbProcessed.*, + minidvodbRun.minidvodb_name + minidvodbRun.minidvodb_group +FROM minidvodbProcessed +JOIN minidvodbRun + USING(minidvodb_id) + Index: trunk/ippTools/share/addtool_find_minidvodbrun.sql =================================================================== --- trunk/ippTools/share/addtool_find_minidvodbrun.sql (revision 28181) +++ trunk/ippTools/share/addtool_find_minidvodbrun.sql (revision 28181) @@ -0,0 +1,5 @@ +SELECT + minidvodbRun.* +FROM minidvodbRun + + Index: trunk/ippTools/share/addtool_find_minidvodbrun2.sql =================================================================== --- trunk/ippTools/share/addtool_find_minidvodbrun2.sql (revision 28181) +++ trunk/ippTools/share/addtool_find_minidvodbrun2.sql (revision 28181) @@ -0,0 +1,23 @@ +-- this selects the minidvodbs that have complete addRuns +SELECT minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +AS minidvodb_group +FROM minidvodbRun +JOIN + (SELECT minidvodb_name, count(state) as cnt2, cnt, minidbg, minidbn + FROM addRun LEFT JOIN + -- this select grabs (and counts) all the addRuns with any state + -- and groups them by the name + (SELECT minidvodb_group as minidbg, minidvodb_name as minidbn, count(state) + AS cnt + FROM addRun + GROUP BY minidvodb_name ) + AS foo1 ON minidvodb_name = minidbn + -- the second select grabs and counts all the addRuns with state + -- of 'full' + WHERE addRun.state = 'full' + GROUP BY minidvodb_name ) +AS foo2 +USING(minidvodb_name) +WHERE cnt2 = cnt + -- the 3rd select only grabs the columns where the counts are the same + -- (same number of 'full' and any state) Index: trunk/ippTools/share/addtool_find_pendingmergeprocess.sql =================================================================== --- trunk/ippTools/share/addtool_find_pendingmergeprocess.sql (revision 28181) +++ trunk/ippTools/share/addtool_find_pendingmergeprocess.sql (revision 28181) @@ -0,0 +1,5 @@ +SELECT minidvodbRun.* +FROM minidvodbRun +LEFT JOIN minidvodbProcessed +USING(minidvodb_id) +WHERE (minidvodbRun.state = 'to_be_merged' AND minidvodbProcessed.minidvodb_id IS NULL) Index: trunk/ippTools/share/addtool_find_processedexp.sql =================================================================== --- trunk/ippTools/share/addtool_find_processedexp.sql (revision 28178) +++ trunk/ippTools/share/addtool_find_processedexp.sql (revision 28181) @@ -2,6 +2,8 @@ addProcessedExp.*, addRun.workdir -FROM addRun -JOIN camProcessedExp +FROM addProcessedExp +JOIN addRun + USING(add_id) +JOIN camRun USING(cam_id) JOIN chipRun Index: trunk/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- trunk/ippTools/share/addtool_queue_cam_id.sql (revision 28178) +++ trunk/ippTools/share/addtool_queue_cam_id.sql (revision 28181) @@ -11,5 +11,8 @@ '%s', -- dvodb '%s', -- note - %d -- image_only + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name FROM camRun WHERE Index: trunk/ippTools/share/addtool_queue_minidvodbrun.sql =================================================================== --- trunk/ippTools/share/addtool_queue_minidvodbrun.sql (revision 28181) +++ trunk/ippTools/share/addtool_queue_minidvodbrun.sql (revision 28181) @@ -0,0 +1,12 @@ +-- create rcRun's for all destinations that have filesets that they are interested in available +INSERT INTO minidvodbRun +SELECT -- rows in this select must match rcRun + minidvodb_name, + minidvodb_path, + mergedvodb_path, + 'new' + +FROM addRun +LEFT JOIN minidvodbRun USING(minidvodb_name) +-- WHERE minidvodbRun.minidvodb_name IS NULL + Index: trunk/ippTools/share/addtool_reset_faulted_minidvodbruns.sql =================================================================== --- trunk/ippTools/share/addtool_reset_faulted_minidvodbruns.sql (revision 28181) +++ trunk/ippTools/share/addtool_reset_faulted_minidvodbruns.sql (revision 28181) @@ -0,0 +1,6 @@ +UPDATE minidvodbProcessed, minidvodbRun, addRun +SET minidvodbRun.state = 'to_be_merged' +WHERE + minidvodbProcessed.minidvodb_id = minidvodbRun.minidvodb_id + AND addRun.minidvodb_id = minidvodbRun.minidvodb_id + AND minidvodbProcessed.fault != 0 Index: trunk/ippTools/share/addtool_revertminidvodbprocessed.sql =================================================================== --- trunk/ippTools/share/addtool_revertminidvodbprocessed.sql (revision 28181) +++ trunk/ippTools/share/addtool_revertminidvodbprocessed.sql (revision 28181) @@ -0,0 +1,6 @@ +DELETE FROM minidvodbProcessed +USING minidvodbProcessed, minidvodbRun, addRun +WHERE + minidvodbProcessed.minidvodb_id = minidvodbRun.minidvodb_id + AND addRun.minidvodb_name = minidvodbRun.minidvodb_name + AND minidvodbProcessed.fault != 0 Index: trunk/ippTools/src/addtool.c =================================================================== --- trunk/ippTools/src/addtool.c (revision 28178) +++ trunk/ippTools/src/addtool.c (revision 28181) @@ -40,4 +40,15 @@ static bool maskedMode(pxConfig *config); static bool unblockMode(pxConfig *config); +static bool addminidvodbrunMode(pxConfig *config); +static bool updateminidvodbrunMode(pxConfig *config); +static bool listminidvodbrunMode(pxConfig *config); +static bool flipminidvodbrunMode(pxConfig *config); +static bool checkminidvodbrunaddrunMode(pxConfig *config); +static bool addminidvodbprocessedMode(pxConfig *config); +static bool listminidvodbprocessedMode(pxConfig *config); +static bool revertminidvodbprocessedMode(pxConfig *config); +static bool updateminidvodbprocessedMode(pxConfig *config); + + # define MODECASE(caseName, func) \ @@ -69,4 +80,14 @@ MODECASE(ADDTOOL_MODE_MASKED, maskedMode); MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_ADDMINIDVODBRUN, addminidvodbrunMode); + MODECASE(ADDTOOL_MODE_UPDATEMINIDVODBRUN, updateminidvodbrunMode); + MODECASE(ADDTOOL_MODE_LISTMINIDVODBRUN, listminidvodbrunMode); + MODECASE(ADDTOOL_MODE_FLIPMINIDVODBRUN, flipminidvodbrunMode); + MODECASE(ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, checkminidvodbrunaddrunMode); + MODECASE(ADDTOOL_MODE_ADDMINIDVODBPROCESSED,addminidvodbprocessedMode); + MODECASE(ADDTOOL_MODE_LISTMINIDVODBPROCESSED,listminidvodbprocessedMode); + MODECASE(ADDTOOL_MODE_REVERTMINIDVODBPROCESSED,revertminidvodbprocessedMode); + MODECASE(ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED,updateminidvodbprocessedMode); + default: psAbort("invalid option (this should not happen)"); @@ -102,4 +123,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); + if (!psListLength(where->list)) { psFree(where); @@ -114,8 +136,13 @@ PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(minidvodb, config->args, "-set_minidvodb", false); PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the cam_id of all the exposures that we want to queue up. @@ -245,8 +272,11 @@ dvodb ? dvodb : row->dvodb, note ? note : NULL, - image_only + image_only, + minidvodb, + minidvodb_group, + minidvodb_name )) { if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error sfg"); } psError(PS_ERR_UNKNOWN, false, @@ -291,5 +321,5 @@ // pxUpdateRun gets parameters from config->args and runs the update query - bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", + bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", "addProcessedExp", false); @@ -374,4 +404,5 @@ PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + PXOPT_LOOKUP_STR(dvodb_path, config->args, "-dvodb_path", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -399,4 +430,5 @@ return false; } + //psError(PS_ERR_UNKNOWN, false, "database error %s", query); psFree(query); @@ -424,4 +456,5 @@ dtime_addstar, path_base, + dvodb_path, fault ); @@ -522,5 +555,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); psFree(query); return false; @@ -733,2 +766,841 @@ return true; } + +static bool addminidvodbrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + // required + + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); + + //optional + PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); + PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString minidvodbpath = "NULL"; + + // I don't know how to get around the complaints of minidvodb_path can't be null. this 'fixes' it, but someone smarter can fix it properly. + if (minidvodb_path) { + minidvodbpath = minidvodb_path; + } + + if (!minidvodbRunInsert(config->dbh, + 0, // job_id + minidvodb_name, + minidvodb_group, + minidvodbpath, + mergedvodb_path, + "new", + 0 + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 minidvodb_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", minidvodb_id); + + + if (!minidvodb_name) { + psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); + } + + if (!minidvodb_path) { + psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); + } + + psString query = NULL; + + psStringAppend(&query, "UPDATE minidvodbRun SET minidvodb_path = '%s', minidvodb_name = '%s' where minidvodb_id = %" PRIu64";", minidvodb_path, minidvodb_name, minidvodb_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updateminidvodbrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); + + PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); + PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE minidvodbRun SET "); + int cnt = 0; + psString comma = ","; + if (minidvodb_name) { + psStringAppend(&query, " minidvodb_name = '%s'", minidvodb_name); + cnt++; + } + + if (minidvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " minidvodb_path = '%s'", minidvodb_path); + cnt++; + } + + if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + if (minidvodb_group) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " minidvodb_group = '%s'", minidvodb_group); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; + +} + +static bool flipminidvodbrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group",true, false); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); + +//this flips the new - > active +// and the active - > waiting in one action + + +// the first query looks to find things that are new and where all the fields are filled (ie, ready to be flipped to active) + +// start a transaction eraly so it will contain any row level locks + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + + psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&firstquery, " AND %s", firstwhereClause); + + + + if (!p_psDBRunQuery(config->dbh, firstquery)) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(firstquery); + + //we don't care what the stuff is that is found, just that there is stuff. This is a check to see that there is something in the 'new' state, before flipping (so that if there is nothing in new, it won't flip the active to waiting. the flipminidvo is just to make it easy to flip from new -> active. + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + + //ok, there's something, so flip active -> waiting + + psString query = psStringCopy("UPDATE minidvodbRun SET state = 'waiting' WHERE state = 'active' "); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + + //now flip new -> active + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); + if (!p_psDBRunQuery(config->dbh, query2)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(query); + return false; + } + + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psFree(query); + return false; + } + psFree(query2); + psFree(where); + + return true; + +} + + +static bool checkminidvodbrunaddrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + //this checks to see if a minidvod_group/name is has completed addRun processing + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "=="); + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(all_addrun_states, config->args, "-all_addrun_states", false); + //this doesn't care what state the addRun is in (useful for counting addRuns) + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("addtool_checkminidvodbaddrun.sql"); + + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (!all_addrun_states) { + psStringAppend(&query, " AND (cnt2 = cnt) "); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "minidvodbRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + + +static bool listminidvodbrunMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "minidvodbRun.state", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("addtool_find_minidvodbrun.sql"); + + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "minidvodbRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool addminidvodbprocessedMode(pxConfig *config) { + + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); + PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); + + // optional + PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); + PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + //generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + psString query = pxDataGet("addtool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + minidvodbRunRow *pendingRow = minidvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( + pendingRow->minidvodb_id, + merge_order, + dtime_resort, + dtime_relphot, + dtime_merge, + epoch, + mergedvodb_path, + fault + ); + + if (!minidvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + + + + //this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); + printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + + + +// since there is only one exp per 'new' set mindvodbRun.state = 'merged' + + psString query2 = NULL ; + psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + return true; +} + + + +static bool listminidvodbprocessedMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbProcessed.minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbProcessed.minidvodb_group", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("addtool_find_minidvodbprocessed.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND minidvodbProcessed.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND minidvodbProcessed.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE minidvodbProcessed.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE minidvodbProcessed.fault = 0"); + } + psFree(where); + + // order by epoch + psStringAppend(&query, " ORDER BY minidvodb_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error "); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "minidvodbProcessed", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + + + +static bool revertminidvodbprocessedMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "addRun.minidvodb_group", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_minidvodbruns.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + { + psString query = pxDataGet("addtool_revertminidvodbprocessed.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + + +static bool updateminidvodbprocessedMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); + PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); + PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE minidvodbProcessed JOIN minidvodbRun USING (minidvodb_id) SET "); + int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + if (merge_order) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " merge_order = %" PRId64, merge_order); + cnt++; + } + + if (dtime_relphot) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_relphot = %f", dtime_relphot); + cnt++; + } + + if (dtime_resort) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_resort = %f", dtime_resort); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + Index: trunk/ippTools/src/addtool.h =================================================================== --- trunk/ippTools/src/addtool.h (revision 28178) +++ trunk/ippTools/src/addtool.h (revision 28181) @@ -39,5 +39,14 @@ ADDTOOL_MODE_DONECLEANUP, ADDTOOL_MODE_EXPORTRUN, - ADDTOOL_MODE_IMPORTRUN + ADDTOOL_MODE_IMPORTRUN, + ADDTOOL_MODE_ADDMINIDVODBRUN, + ADDTOOL_MODE_UPDATEMINIDVODBRUN, + ADDTOOL_MODE_LISTMINIDVODBRUN, + ADDTOOL_MODE_FLIPMINIDVODBRUN, + ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, + ADDTOOL_MODE_ADDMINIDVODBPROCESSED, + ADDTOOL_MODE_LISTMINIDVODBPROCESSED, + ADDTOOL_MODE_REVERTMINIDVODBPROCESSED, + ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED } addtoolMode; Index: trunk/ippTools/src/addtoolConfig.c =================================================================== --- trunk/ippTools/src/addtoolConfig.c (revision 28178) +++ trunk/ippTools/src/addtoolConfig.c (revision 28181) @@ -63,5 +63,7 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb", 0, "use minidvodb", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_bname", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); @@ -84,4 +86,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); @@ -98,4 +101,5 @@ psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-dvodb_path", 0, "define base output location", NULL); psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -142,4 +146,79 @@ psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + // -addminidvodbruns + psMetadata *addminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); + psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -updateminidvodbruns + psMetadata *updateminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddU64(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id ", 0); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by minidvodb_name (LIKE)", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); + psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + + // -listminidvodbrunArgs + psMetadata *listminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddU64(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); + psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by minidvodb.minidvodb_group", NULL); + psMetadataAddStr(listminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddU64(listminidvodbrunArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listminidvodbrunArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + //psMetadataAddBool(listminidvodbrunArgs, PS_LIST_TAIL, "-finished_addrun", 0, "limit to minidvodbs with completed addRuns (none in new state)", false); + psMetadata *flipminidvodbrunArgs = psMetadataAlloc(); + psMetadataAddStr(flipminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "for the supplied minidvodb_group (required): flip the current 'new' to 'active', the current 'active' to 'waiting'", NULL); + + psMetadata *checkminidvodbrunaddrunArgs = psMetadataAlloc(); + psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "for the supplied minidvodb_group (required): check if the addRun stage is complete (all in addRun.state = full) ", NULL); + psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "for the supplied minidvodb_name: check if the addRun stage is complete (all in addRun.state = full) ", NULL); + psMetadataAddStr(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-state", 0, "limit by minidvodbRun state ", NULL); + psMetadataAddU64(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(checkminidvodbrunaddrunArgs, PS_LIST_TAIL, "-all_addrun_states", 0, "list all minidvodbRun.minidvodb_names, not just ones that have complete addRuns", false); + + psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); + psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); + psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); + psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); + psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + psMetadata *listminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); + psMetadataAddStr(listminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by minidvodb.minidvodb_group", NULL); + psMetadataAddU64(listminidvodbprocessedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listminidvodbprocessedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listminidvodbprocessedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(revertminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(revertminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group", 0, "search by addRun.minidvodb_group", NULL); + + psMetadata *updateminidvodbprocessedArgs = psMetadataAlloc(); + psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); + psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); + psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -154,5 +233,16 @@ PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); - PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-addminidvodbrun", "create minidvodbs ", ADDTOOL_MODE_ADDMINIDVODBRUN, addminidvodbrunArgs); + PXOPT_ADD_MODE("-updateminidvodbrun", "change minidvodb properties", ADDTOOL_MODE_UPDATEMINIDVODBRUN, updateminidvodbrunArgs); + PXOPT_ADD_MODE("-listminidvodbrun", "list minidvodbs", ADDTOOL_MODE_LISTMINIDVODBRUN, listminidvodbrunArgs); + PXOPT_ADD_MODE("-flipminidvodbrun", "flip minidvodbs", ADDTOOL_MODE_FLIPMINIDVODBRUN, flipminidvodbrunArgs); + PXOPT_ADD_MODE("-checkminidvodbrunaddrun", "check minidvodbs to see if addRuns are completed", ADDTOOL_MODE_CHECKMINIDVODBRUNADDRUN, checkminidvodbrunaddrunArgs); + PXOPT_ADD_MODE("-addminidvodbprocessed","add a processed minidvodb", ADDTOOL_MODE_ADDMINIDVODBPROCESSED, addminidvodbprocessedArgs); + PXOPT_ADD_MODE("-listminidvodbprocessed","list processed minidvodbs", ADDTOOL_MODE_LISTMINIDVODBPROCESSED, listminidvodbprocessedArgs); + PXOPT_ADD_MODE("-revertminidvodbprocessed","revert processed minidvobs", ADDTOOL_MODE_REVERTMINIDVODBPROCESSED, revertminidvodbprocessedArgs); + PXOPT_ADD_MODE("-updateminidvodbprocessed","change processed minidvodb properties",ADDTOOL_MODE_UPDATEMINIDVODBPROCESSED, updateminidvodbprocessedArgs); + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: trunk/ippTools/src/flatcorr.c =================================================================== --- trunk/ippTools/src/flatcorr.c (revision 28178) +++ trunk/ippTools/src/flatcorr.c (revision 28181) @@ -671,5 +671,8 @@ row->dvodb, NULL, // note is not propagated - 0)) { + 0, + 0, //The minidvodb stuff is off + NULL, + NULL)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: trunk/ippTools/src/pstamptool.h =================================================================== --- trunk/ippTools/src/pstamptool.h (revision 28178) +++ trunk/ippTools/src/pstamptool.h (revision 28181) @@ -22,4 +22,6 @@ #include "pxtools.h" + +#define PSTAMP_FIRST_ERROR_CODE 0 // First error code value to consider typedef enum { Index: trunk/ippTools/src/pxadd.c =================================================================== --- trunk/ippTools/src/pxadd.c (revision 28178) +++ trunk/ippTools/src/pxadd.c (revision 28181) @@ -130,5 +130,8 @@ char *dvodb, char *note, - bool image_only) + bool image_only, + bool minidvodb, + char *minidvodb_group, + char *minidvodb_name) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -158,7 +161,10 @@ note ? note : "NULL", image_only, + minidvodb, + minidvodb_group, + minidvodb_name, (long long) cam_id )) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error %s", query); return false; } Index: trunk/ippTools/src/pxadd.h =================================================================== --- trunk/ippTools/src/pxadd.h (revision 28178) +++ trunk/ippTools/src/pxadd.h (revision 28181) @@ -38,5 +38,8 @@ char *dvodb, char *note, - bool image_only); + bool image_only, + bool minidvodb, + char *minidvodb_group, + char *minidvodb_name); #endif // PXADD_H