Index: /trunk/Ohana/src/libdvo/src/dvo_util.c =================================================================== --- /trunk/Ohana/src/libdvo/src/dvo_util.c (revision 33029) +++ /trunk/Ohana/src/libdvo/src/dvo_util.c (revision 33030) @@ -276,5 +276,5 @@ FREE(detections); - if (NInvalidDetID) fprintf (stderr, "ERROR: Encountered %ld invalid detection IDs\n", NInvalidDetID); + if (NInvalidDetID) fprintf (stderr, "ERROR: Encountered %zx invalid detection IDs\n", NInvalidDetID); return (Ndetect); Index: /trunk/dbconfig/mergedvodb.md =================================================================== --- /trunk/dbconfig/mergedvodb.md (revision 33030) +++ /trunk/dbconfig/mergedvodb.md (revision 33030) @@ -0,0 +1,29 @@ +mergedvodbRun METADATA + merge_id S64 0 + minidvodb_id S64 0 + mergedvodb STR 64 + mergedvodb_path STR 255 + state STR 64 + creation_date UTC 0001-01-01T00:00:00Z +END + +mergedvodbProcessed METADATA + merge_id S64 0 + merge_order S64 0 + dtime_verify F32 0.0 + dtime_merge F32 0.0 + dtime_script F32 0.0 + epoch UTC 0001-01-01T00:00:00Z + fault S16 0 +END + +mergedvodbCopy METADATA + mergevodbcopy_id S64 0 + merge_id S64 0 + mergedvodb_rsync_path STR 255 + destination_host STR 255 + fault S16 0 + state STR 64 + epoch UTC 0001-01-01T00:00:00Z + dtime F32 0.0 +END Index: /trunk/dbconfig/minidvodb.md =================================================================== --- /trunk/dbconfig/minidvodb.md (revision 33029) +++ /trunk/dbconfig/minidvodb.md (revision 33030) @@ -4,5 +4,4 @@ minidvodb_group STR 64 minidvodb_path STR 255 - mergedvodb_path STR 255 state STR 64 creation_date UTC 0001-01-01T00:00:00Z @@ -11,12 +10,8 @@ minidvodbProcessed METADATA minidvodb_id S64 0 - merge_order S64 0 dtime_resort F32 0.0 dtime_relphot F32 0.0 - dtime_verify F32 0.0 - dtime_merge F32 0.0 dtime_script F32 0.0 epoch UTC 0001-01-01T00:00:00Z - mergedvodb_path STR 255 fault S16 0 END Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 33029) +++ /trunk/ippScripts/Build.PL (revision 33030) @@ -65,5 +65,9 @@ scripts/minidvodb_createdb.pl scripts/minidvodb_merge.pl + scripts/minidvodb_premerge.pl scripts/minidvodb_copy.pl + scripts/mergedvodb_copy.pl + scripts/mergedvodb_merge.pl + scripts/mergedvodb_queue.pl scripts/ippdb.pl scripts/ipp_cleanup.pl Index: /trunk/ippScripts/scripts/addstar_run.pl =================================================================== --- /trunk/ippScripts/scripts/addstar_run.pl (revision 33029) +++ /trunk/ippScripts/scripts/addstar_run.pl (revision 33030) @@ -30,4 +30,5 @@ my $missing_tools; my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); @@ -37,5 +38,6 @@ } my $minidvodb_path; -my ( $add_id, $camera, $stage, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my $stage_extra1; +my ( $add_id, $camera, $stage, $stage_id, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -43,4 +45,5 @@ 'camera|c=s' => \$camera, # Camera 'stage|s=s' => \$stage, # Camera + 'stage_id|w=s' => \$stage_id, 'multi_num|w=s' => \$multi_num, # the number for a staticskymulti (for finding cmf) 'dbname|d=s' => \$dbname, # Database name @@ -71,4 +74,8 @@ defined $dvodb and defined $camera; +if ($stage =~ /cam/ && !defined $stage_id) { + my_die("cam stage requires -stage_id", $add_id, 3); + +} if ($minidvodb && !defined($minidvodb_group)) { my_die( "missing minidvodb_group", $add_id, 3 ); @@ -108,11 +115,52 @@ # it was PSASTRO.OUTPUT -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if ($stage =~ /cam/) { + # if it is cam stage we need to be careful when grabbing the filename. + # This breaks down into a few steps: + + #get info about the cam_id + my $magicked; + { + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$camtool -processedexp -cam_id " . $stage_id; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR); + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mparsed = $comp->{magicked}; + + if (!defined($mparsed)) { + &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found a value for magicked:$mparsed\n"; + $magicked = $mparsed; + } + + #is this cam_id magicked or not? + if ($magicked) { + $stage_extra1 = $magicked; + $fpaObjects = $ipprc->destreaked_filename("$fpaObjects") or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + print "cam_id is magicked, using $fpaObjects for the cam smf\n"; + } else { + print "cam_id is NOT magicked, using $fpaObjects for the cam smf\n"; + } + } +} if (($stage =~/staticsky/) || ($stage =~/stack/)) { $fpaObjects =~ s/smf$/cmf/; - -} - +} if ($stage =~/staticsky_multi/) { $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above @@ -120,9 +168,8 @@ my $nice_num = sprintf ("%03d", $multi_num); $fpaObjects =~ s/cmf$/$nice_num.cmf/; #this make it look for .001.cmf, etc - } my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + # convert supplied DVO database name to UNIX filename my $dvodbReal; @@ -131,84 +178,69 @@ $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); } - - - - my $dtime_addstar = 0; - if (defined $dvodbReal) { - if ($minidvodb) { - - #if (!defined($minidvodb_name)) { - #take the active one, if it's not defined on the command line - #reverts would have this already set, for example. - my $command = "addtool -listminidvodbrun "; - $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; - $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; - $command .= " -state 'active' -limit 1"; - $command .= " -dbname $dbname" if defined $dbname; - print $command; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); - - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - $minidvodb_path = $comp->{minidvodb_path}; - $minidvodb_name = $comp->{minidvodb_name}; - - if (!defined($minidvodb_path)) { - &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); - } - if (!defined($minidvodb_name)) { - &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); - } - #} - # tack on the minidvodb part to the db. -# $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; -#we don't need this now that I fixed the paths - - } - else { - - $minidvodb_path = $dvodbReal; - } - -unless ($no_op) { + if ($minidvodb) { + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_path = $comp->{minidvodb_path}; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_path)) { + &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); + } + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + } + else { + $minidvodb_path = $dvodbReal; + } + + unless ($no_op) { print $dvodbReal; - - ## addstar can either save the full set of detections, or just - ## the image metadata, in the dvodb. this is set in the - ## database table addRun - - # addstar requires the user to have a valid .ptolemyrc which - # in turn points at ippconfig/dvo.site - - # get the names for the camera and the real input file - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); - - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - my $command = "$addstar -update"; # XXX optionally set -update? - $command .= " -D CAMERA $camdir"; - $command .= " -D CATDIR $minidvodb_path"; - $command .= " $realFile"; - $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference + + ## addstar can either save the full set of detections, or just + ## the image metadata, in the dvodb. this is set in the + ## database table addRun + + # addstar requires the user to have a valid .ptolemyrc which + # in turn points at ippconfig/dvo.site + + # get the names for the camera and the real input file + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); + + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + my $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -D CAMERA $camdir"; + $command .= " -D CATDIR $minidvodb_path"; + $command .= " $realFile"; + $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference $command .= " -image" if $image_only; if ($stage =~ /staticsky/) { $command .= " -accept-astrom "; } #careful here - this matches staticsky and staticsky_multi - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script } } @@ -221,4 +253,5 @@ $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; +$fpaCommand .= " -stage_extra1" if defined $stage_extra1; $fpaCommand .= " -dbname $dbname" if defined $dbname; print $fpaCommand; @@ -253,6 +286,8 @@ $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + # $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure) + $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $command .= " -dbname $dbname" if defined $dbname; + print $command; system ($command); } Index: /trunk/ippScripts/scripts/mergedvodb_copy.pl =================================================================== --- /trunk/ippScripts/scripts/mergedvodb_copy.pl (revision 33030) +++ /trunk/ippScripts/scripts/mergedvodb_copy.pl (revision 33030) @@ -0,0 +1,189 @@ +#!/usr/bin/env perl + +#This is essentially the same as minidvodb_copy.pl - the tables are essentially identical, and the task (to rsync a copy) is essentially identical. A new script was created for rsyncing the merges in case they should diverge in the future. + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; + +my $dtime; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $rsync = can_run('rsync') or (warn "Can't find rsync" and $missing_tools = 1); +my $mergedvodbtool = can_run('mergedvodbtool') or (warn "Can't find mergedvodbtool" and $missing_tools = 1); +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $df = can_run('df') or (warn "Can't find df" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + + +my ( $mergedvodbcopy_id, $merge_id, $mergedvodbrun_path, $mergedvodb_rsync_path, $destination_host, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodbcopy_id|w=s' => \$mergedvodbcopy_id, #minidvodb database + 'merge_id|w=s' => \$mergedvodb_id, #minidvodb_id + 'mergedvodbrun_path|w=s' => \$mergedvodbrun_path, #minidvodb_id + 'mergedvodb_rsync_path|w=s' => \$mergedvodb_rsync_path, + 'destination_host|w=s' => \$destination_host, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodbcopy_id --mergedvodb_rsync_path --mergedvodbrun_path --destination_host", + -exitval => 3, + ) unless + defined $mergedvodbcopy_id and + defined $mergedvodb_rsync_path and + defined $mergedvodbrun_path and + defined $destination_host; + + + +my $ipprc = PS::IPP::Config->new(); + +my @df; + + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodbcopy_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +{ #can we rsync? + + my $sizes = `du -sk $mergedvodbrun_path`; + my $total = 0; + for(split /[\r\n]+/,$sizes) # split on one or more newline characters + { + my($number,$file) = split /\t/,$_,2; # split on tab ($file not used here) + $total += $number; + } +#print 'Total: '.$total; +{ + print "Checking available diskspace on $destination_host\n\n"; + + my $command = "$ssh $destination_host df $mergedvodb_rsync_path"; + print "$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform '$ssh $destination_host df $mergedvodb_rsync_path' $error_code", $minidvodbcopy_id, $error_code); + } + + my @fullbuf= split(/\n/, @{$stdout_buf}[0]); + # print $#{$stdout_buf}; +# print "xx $#df xx"; +# print $fullbuf[0]; + if ($#fullbuf ==1 ) { +# print $fullbuf[1]; + @df = split /\s+/, $fullbuf[1]; + if ($#df == 5) { +# + #print $df[3], $df[4]; + + $df[4]=~ s/\%//; + + if ($df[4] < 97 && $df[3]-$total) { + print "\nThere is enough available space ($total < $df[3]) for the rsync\n"; + } else { + &my_die("Not enough available disk space: $df[4]% available, $df[3] on destination, $total free required for copy", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +# + + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nCan't figure out %free or available", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nToo many lines ($#fullbuf) vs 1", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +} +} + +{#do the rsync + my $command = "$rsync -rvuaq"; + $command .= " $mergedvodbrun_path"; + $command .= " $destination_host"; + $command .= ":$mergedvodb_rsync_path"; + print "\nPerforming rsync:\n"; + print "$command\n\n"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform rsync: $error_code", $mergedvodbcopy_id, $error_code); + } + + print "Rsync complete..\n"; + +} + +{#update the database + $dtime = 86400.0*(DateTime->now->mjd - $mjd_start); + + my $command = "$mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_state full -set_dtime $dtime"; + $command .= " -dbname $dbname" if defined $dbname; + print "$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = 20; + &my_die("Unable to update database - rsync completed without error' $error_code", $mergedvodbcopy_id, $error_code); + } + +} +exit 0; + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $mergedbvodbcopy_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodbcopy_id\n"; + +if (defined $mergedvodb_id ) { + my $command = "mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_fault $exit_code"; + $command .= " -set_dtime $dtime" if defined $dtime; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: /trunk/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- /trunk/ippScripts/scripts/mergedvodb_merge.pl (revision 33030) +++ /trunk/ippScripts/scripts/mergedvodb_merge.pl (revision 33030) @@ -0,0 +1,237 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_merge; +my $dtime_verify; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb_path, $merge_id, $mergedvodb, $minidvodb_path, $dbname,$camera,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb_path|w=s' => \$mergedvodb_path, # output DVO database + 'minidvodb_path|w=s' => \$minidvodb_path, #minidvodb database + 'mergedvodb|w=s' => \$mergedvodb, #mergedvodb + 'merge_id|w=s' => \$merge_id, #minidvodb_id + 'dbname|d=s' => \$dbname, # Database name + 'camera|w=s' => \$camera, #camera name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --merge_id -- minidvodb_path --mergedvodb_path --mergedvodb --camera", + -exitval => 3, + ) unless + defined $mergedvodb_path and + defined $minidvodb_path and + defined $mergedvodb and + defined $merge_id and + defined $camera; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $merge_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $merge_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename + +unless ($no_op) { + + #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge + + #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR + { + my $nothing_faulted = 0; + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -faulted -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + print "looking for faulted:\n\n$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of faulted mergedvodbs", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted + $nothing_faulted =1; + print "previous merges are okay.\n"; + } + &my_die( "Previous merges faulted, do not proceed until they are investigated:", $merge_id, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; + } + { + my $command = "$dvoverify -s $minidvodb_path"; + print "$command\n"; + my $mjd_dvoverify_start = DateTime->now->mjd; # MJD of starting script + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoverify: $error_code", $merge_id, $error_code); + } + $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start); # MJD of starting script + print "dvoverify time $dtime_verify\n"; + } + #Merge + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of mergedvodbruns", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $merge_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $merge_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mergedvodb_path_tmp = $comp->{mergedvodb_path}; + + if (!defined($mergedvodb_path_tmp)) { + &my_die("Unable to parse mergedvodb_path", $merge_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 mergedvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + if (-e "$mergedvodb_path/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $merge_id, 4); + } + $this_is_the_first =0; + } + + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; + } else { + $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + } + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $merge_id, $error_code); + } + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; +# } + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; +# $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to mergedvodbprocessed: $error_code", $merge_id, $error_code); + } + } + +#} else { +# &my_die("dvodb: $mergedvodb_path not found", $merge_id, $PS_EXIT_UNKNOWN_ERROR); +} +else { + print "skipping processing for $mergedvodb_path\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $merge_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb_path\n"; + +if (defined $merge_id ) { + + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; + $command .= " -fault $exit_code"; + #$command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + + + + print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: /trunk/ippScripts/scripts/mergedvodb_queue.pl =================================================================== --- /trunk/ippScripts/scripts/mergedvodb_queue.pl (revision 33030) +++ /trunk/ippScripts/scripts/mergedvodb_queue.pl (revision 33030) @@ -0,0 +1,164 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +# Ideally, we want to queue new runs with mergetool -definebyquery. We can't for the survey task, because we use the value in 'site.config' to find the mergedvodb_path. I do not know how to tell pantasks how to do this, but I know how to do it in perl. + + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $minidvodb_group, $mergedvodb , $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); + +# we just need any old camera to be able to read site.config. I'm using SIMPLE, unless it is overridden on the commandline (it makes no difference for this case) + +$camera = 'SIMPLE'; + + +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'mergedvodb|w=s' => \$mergedvodb, # output miniDVODB + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + # defined $camera and + defined $outroot and + defined $mergedvodb; + +print "camera is $camera\n"; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); +print "log file is $logDest\n"; +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} else { + warn("mergedvodb undefined:\n"); + exit(4); +} +if (!defined $mergedvodbReal) { + warn("can't parse out mergedvodb_path\n"); + exit(4); +} + + + +print "mergedvodb_path = $mergedvodbReal\n"; + +unless ($no_op) { + + #create the mergedvodb entry (well, the command for it) + my $fpaCommand = "$mergetool -definebyquery"; + $fpaCommand .= " -mergedvodb $mergedvodb"; + $fpaCommand .= " -minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $mergedvodbReal" ; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + print "running the following: \n\n"; + print "$fpaCommand\n\n"; + + unless ($no_update) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $mergedvodb = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: /trunk/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- /trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 33029) +++ /trunk/ippScripts/scripts/minidvodb_createdb.pl (revision 33030) @@ -226,5 +226,4 @@ $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; $fpaCommand .= " -set_minidvodb_path $minidvodbReal" if defined $minidvodbReal; - $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; $fpaCommand .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- /trunk/ippScripts/scripts/minidvodb_premerge.pl (revision 33030) +++ /trunk/ippScripts/scripts/minidvodb_premerge.pl (revision 33030) @@ -0,0 +1,174 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $minidvodb_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +my $dtime_addstar = 0; + +unless ($no_op) { + + #this is chopped into several parts: addstar, relphot + #addstar + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + $dtime_resort = $dtime_addstar; + print "addstar -resort time $dtime_addstar\n"; + } + + #relphot + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $minidvodb"; + print "$command\n"; + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $minidvodb_id, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $minidvodb_id, $error_code); + } + } +#} else { +# &my_die("dvodb: minidvodb_id = $minidvodb_id not found", $minidvodb_id, $PS_EXIT_UNKNOWN_ERROR); +#} +} else { + print "skipping processing for minidvodb_id = $minidvodb_id\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + print STDERR "$msg $minidvodb_id\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: /trunk/ippTasks/Makefile.am =================================================================== --- /trunk/ippTasks/Makefile.am (revision 33029) +++ /trunk/ippTasks/Makefile.am (revision 33030) @@ -38,4 +38,5 @@ minidvodb.pro \ minidvodbcopy.pro \ + mergedvodb.pro \ nightly_science.pro \ burntool.pro \ Index: /trunk/ippTasks/addstar.pro =================================================================== --- /trunk/ippTasks/addstar.pro (revision 33029) +++ /trunk/ippTasks/addstar.pro (revision 33030) @@ -53,10 +53,28 @@ macro addstar.revert.off - task addstar.revert + task addstar.revert.cam + active false + end + task addstar.revert.stack + active false + end + task addstar.revert.staticsky + active false + end + task addstar.revert.staticsky_multi active false end end macro addstar.revert.on - task addstar.revert + task addstar.revert.cam + active true + end + task addstar.revert.stack + active true + end + task addstar.revert.staticsky_multi + active true + end + task addstar.revert.staticsky active true end @@ -65,6 +83,10 @@ # this variable will cycle through the known database names -$addstar_DB = 0 -$addstar_revert_DB = 0 +$addstar_DB = 0 +#this may not work for more databases (addstar) will do that later) +$addstar_revert_DB_C = 0 +$addstar_revert_DB_S = 0 +$addstar_revert_DB_SS = 0 +$addstar_revert_DB_SSM = 0 @@ -318,5 +340,6 @@ book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1 book getword addPendingExp $pageName stageroot -var STAGEROOT - book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage_id -var STAGE_ID book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE book getword addPendingExp $pageName reduction -var REDUCTION @@ -367,4 +390,7 @@ if ("$STAGE" == "staticsky_multi") $run = $run --multi_num $STAGE_EXTRA1 + end + if ("$STAGE" == "cam") + $run = $run --stage_id $STAGE_ID end if ("$IMAGE_ONLY" == "T") @@ -411,5 +437,5 @@ end -task addstar.revert +task addstar.revert.cam host local @@ -425,13 +451,13 @@ task.exec if ($LABEL:n == 0) break - $run = addtool -revertprocessedexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_revert_DB - $run = $run -dbname $DB:$addstar_revert_DB - $addstar_revert_DB ++ - if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 + $run = addtool -revertprocessedexp -stage cam + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_C + $run = $run -dbname $DB:$addstar_revert_DB_C + $addstar_revert_DB_C ++ + if ($addstar_revert_DB_C >= $DB:n) set addstar_revert_DB_C = 0 end add_poll_labels run @@ -457,2 +483,140 @@ end end +task addstar.revert.stack + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage stack + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_S + $run = $run -dbname $DB:$addstar_revert_DB_S + $addstar_revert_DB_S ++ + if ($addstar_revert_DB_S >= $DB:n) set addstar_revert_DB_S = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task addstar.revert.staticsky + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage staticsky + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SS + $run = $run -dbname $DB:$addstar_revert_DB_SS + $addstar_revert_DB_SS ++ + if ($addstar_revert_DB_SS >= $DB:n) set addstar_revert_DB_SS = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task addstar.revert.staticsky_multi + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage staticsky_multi + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SSM + $run = $run -dbname $DB:$addstar_revert_DB_SSM + $addstar_revert_DB_SSM ++ + if ($addstar_revert_DB_SSM >= $DB:n) set addstar_revert_DB_SSM = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /trunk/ippTasks/mergedvodb.pro =================================================================== --- /trunk/ippTasks/mergedvodb.pro (revision 33030) +++ /trunk/ippTasks/mergedvodb.pro (revision 33030) @@ -0,0 +1,226 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?havemergedvodbBooks)) + book create MERGEDVODB_MERGE + $havemergedvodbBooks = TRUE +end + +$MERGEDVODB_DB = 0 +$MERGEDVODB_MERGE_DB = 0 + +book init mergedvodbMergelist + +macro mergedvodb.merge.status + book listbook mergedvodbMergelist +end + +macro mergedvodb.merge.reset + book init mergedvodbMergelist +end + +macro mergedvodb.merge.on + task mergedvodb.merge.load + active true + end + task mergedvodb.merge.run + active true + end +end +macro mergedvodb.merge.off + task mergedvodb.merge.load + active false + end + task mergedvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.mergedvodb + if ($0 != 2) + echo "USAGE: add.mergedvodb (mergedvodb)" + break + end + #merge + book newpage MERGEDVODB_MERGE $1 + book setword MERGEDVODB_MERGE $1 MERGEDVODB $1 + book setword MERGEDVODB_MERGE $1 STATE PENDING +end + +macro del.mergedvodb + if ($0 != 2) + echo "USAGE: del.mergedvodb (mergedvodb)" + break + end + book delpage MERGEDVODB_MERGE $1 +end + +macro show.mergedvodb + if ($0 != 1) + echo "USAGE: show.mergedvodb" + break + end + echo "mergedvodb merge" + book listbook MERGEDVODB_MERGE + echo "n pages" + book npages MERGEDVODB_MERGE -var N + echo $N +end + + +task mergedvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/mergedvodb.merge.load.log + + task.exec + book npages MERGEDVODB_MERGE -var N + if ($N == 0) + echo "No labels for processing" + break + endif + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + if ("$mergedvodb" == "NULL") + # All labels have been done --- reset + for i 0 $N + book getpage MERGEDVODB_MERGE $i -var mergedvodb + book setword MERGEDVODB_MERGE $mergedvodb STATE NEW + end + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + + # Select different database + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + book setword MERGEDVODB_MERGE $mergedvodb STATE DONE + + $run = mergetool -pendingmerge -limit 1 + $run = $run -mergedvodb $mergedvodb + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run -dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + #add_poll_args run + #echo $run + command $run + end + + # success + task.exit 0 + #echo "success" + # convert 'stdout' to book format + ipptool2book stdout mergedvodbMergelist -key merge_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook mergedvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup mergedvodbMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task mergedvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60000 + + #we only want one running at a time + + host local + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + book npages mergedvodbMergelist -var N + if ($N == 0) break + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage mergedvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + book setword mergedvodbMergelist $pageName pantaskState RUN + book getword mergedvodbMergelist $pageName merge_id -var MERGE_ID + book getword mergedvodbMergelist $pageName mergedvodb -var MERGEDVODB + book getword mergedvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH + book getword mergedvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword mergedvodbMergelist $pageName state -var STATE + stdout $LOGDIR/mergedvodb.merge.run.log + stderr $LOGDIR/mergedvodb.merge.run.log + $run = mergedvodb_merge.pl --camera GPC1 + $run = $run --mergedvodb_path $MERGEDVODB_PATH + $run = $run --minidvodb_path $MINIDVODB_PATH + $run = $run --merge_id $MERGE_ID --mergedvodb $MERGEDVODB + echo $run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run --dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit mergedvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit mergedvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword mergedvodbMergelist $options:0 pantaskState TIMEOUT + end +end + Index: /trunk/ippTasks/minidvodb.pro =================================================================== --- /trunk/ippTasks/minidvodb.pro (revision 33029) +++ /trunk/ippTasks/minidvodb.pro (revision 33030) @@ -11,5 +11,5 @@ if (not($?haveminidvodbBooks)) book create MINIDVODB - book create MINIDVODB_MERGE + book create MINIDVODB_PREMERGE book create MINIDVODB_CREATE book create MINIDVODB_ACTIVE @@ -20,10 +20,10 @@ $MINIDVODB_WAIT_DB = 0 -$MINIDVODB_MERGE_DB = 0 +$MINIDVODB_PREMERGE_DB = 0 $MINIDVODB_CREATE_DB = 0 $MINIDVODB_ACTIVE_DB = 0 book init minidvodbWaitlist -book init minidvodbMergelist +book init minidvodbPreMergelist book init minidvodbCreatelist book init minidvodbActivelist @@ -54,10 +54,10 @@ end -macro minidvodb.merge.status - book listbook minidvodbMergelist -end - -macro minidvodb.merge.reset - book init minidvodbMergelist +macro minidvodb.premerge.status + book listbook minidvodbPreMergelist +end + +macro minidvodb.premerge.reset + book init minidvodbPreMergelist end @@ -112,17 +112,17 @@ ##these merge the dbs -macro minidvodb.merge.on - task minidvodb.merge.load +macro minidvodb.premerge.on + task minidvodb.premerge.load active true end - task minidvodb.merge.run + task minidvodb.premerge.run active true end end -macro minidvodb.merge.off - task minidvodb.merge.load +macro minidvodb.premerge.off + task minidvodb.premerge.load active false end - task minidvodb.merge.run + task minidvodb.premerge.run active false end @@ -142,8 +142,8 @@ book setword MINIDVODB $1 STATE PENDING #merge - book newpage MINIDVODB_MERGE $1 - book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 - book setword MINIDVODB_MERGE $1 DVODB $3 - book setword MINIDVODB_MERGE $1 STATE PENDING + book newpage MINIDVODB_PREMERGE $1 + book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_PREMERGE $1 DVODB $3 + book setword MINIDVODB_PREMERGE $1 STATE PENDING #active book newpage MINIDVODB_ACTIVE $1 @@ -168,5 +168,5 @@ end book delpage MINIDVODB $1 - book delpage MINIDVODB_MERGE $1 + book delpage MINIDVODB_PREMERGE $1 book delpage MINIDVODB_CREATE $1 book delpage MINIDVODB_ACTIVE $1 @@ -180,6 +180,6 @@ echo "minidvodb wait" book listbook MINIDVODB - echo "minidvodb merge" - book listbook MINIDVODB_MERGE + echo "minidvodb premerge" + book listbook MINIDVODB_PREMERGE echo "minidvodb create" book listbook MINIDVODB_CREATE @@ -339,5 +339,5 @@ -task minidvodb.merge.load +task minidvodb.premerge.load host local @@ -348,8 +348,8 @@ stdout NULL - stderr $LOGDIR/minidvodb.merge.load.log + stderr $LOGDIR/minidvodb.premerge.load.log task.exec - book npages MINIDVODB_MERGE -var N + book npages MINIDVODB_PREMERGE -var N if ($N == 0) # echo "No labels for processing" @@ -357,13 +357,13 @@ endif - book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW if ("$minidvodb_group" == "NULL") # All labels have been done --- reset # echo "Resetting labels" for i 0 $N - book getpage MINIDVODB_MERGE $i -var minidvodb_group - book setword MINIDVODB_MERGE $minidvodb_group STATE NEW + book getpage MINIDVODB_PREMERGE $i -var minidvodb_group + book setword MINIDVODB_PREMERGE $minidvodb_group STATE NEW end - book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW # Select different database @@ -372,5 +372,5 @@ end #finds the minidvodbs in a state of 'to_be_merged' - book setword MINIDVODB_MERGE $minidvodb_group STATE DONE + book setword MINIDVODB_PREMERGE $minidvodb_group STATE DONE $run = addtool -listminidvodbrun -state to_be_merged -limit 1 $run = $run -minidvodb_group $minidvodb_group @@ -392,10 +392,10 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout minidvodbPreMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) - book listbook minidvodbMergelist + book listbook minidvodbPreMergelist end # delete existing entries in the appropriate pantaskStates - process_cleanup minidvodbMergelist + process_cleanup minidvodbPreMergelist end @@ -418,5 +418,5 @@ -task minidvodb.merge.run +task minidvodb.premerge.run periods -poll $RUNPOLL periods -exec $RUNEXEC @@ -432,21 +432,20 @@ periods -exec $RUNEXEC - book npages minidvodbMergelist -var N + book npages minidvodbPreMergelist -var N if ($N == 0) break # look for new images in minidvodbWaitlist (pantaskState == INIT) - book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT + book getpage minidvodbPreMergelist 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break - book setword minidvodbMergelist $pageName pantaskState RUN - book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID - book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP - book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH - book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH - book getword minidvodbMergelist $pageName state -var STATE - stdout $LOGDIR/minidvodb.merge.run.log - stderr $LOGDIR/minidvodb.merge.run.log - - $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + book setword minidvodbPreMergelist $pageName pantaskState RUN + book getword minidvodbPreMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP + book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbPreMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.premerge.run.log + stderr $LOGDIR/minidvodb.premerge.run.log + + $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID if ($DB:n == 0) @@ -472,5 +471,5 @@ # default exit status task.exit default - process_exit minidvodbMergelist $options:0 $JOB_STATUS + process_exit minidvodbPreMergelist $options:0 $JOB_STATUS end @@ -482,5 +481,5 @@ # Set a fault code in the database - process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR + process_exit minidvodbPreMergelist $options:0 $EXIT_CRASH_ERR end @@ -488,5 +487,5 @@ task.exit timeout showcommand timeout - book setword minidvodbMergelist $options:0 pantaskState TIMEOUT + book setword minidvodbPreMergelist $options:0 pantaskState TIMEOUT end end Index: /trunk/ippTasks/survey.pro =================================================================== --- /trunk/ippTasks/survey.pro (revision 33029) +++ /trunk/ippTasks/survey.pro (revision 33030) @@ -3,22 +3,4 @@ # test for required global variables check.globals - -# there is a bug in 'getbook', use a different method below -# book getbook SURVEY_DIFF -var isBook -# if ("$isBook" != "SURVEY_DIFF") -# book create SURVEY_DIFF -# end -# book getbook SURVEY_MAGIC -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_MAGIC -# end -# book getbook SURVEY_DESTREAK -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_DESTREAK -# end -# book getbook SURVEY_DIST -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_DIST -# end if (not($?haveSurveyBooks)) @@ -30,4 +12,5 @@ book create SURVEY_DIST book create SURVEY_ADDSTAR + book create SURVEY_MERGEDVODB book create SURVEY_CHIP_BG book create SURVEY_WARP_BG @@ -44,4 +27,5 @@ $SURVEY_DIST_DB = 0 $SURVEY_ADDSTAR_DB = 0 +$SURVEY_MERGEDVODB_DB = 0 $SURVEY_CHIP_BG_DB = 0 $SURVEY_WARP_BG_DB = 0 @@ -68,4 +52,7 @@ active true end + task survey.mergedvodb + active true + end task survey.destreak active true @@ -104,4 +91,7 @@ active false end + task survey.mergedvodb + active false + end task survey.destreak active false @@ -270,4 +260,31 @@ book listbook SURVEY_ADDSTAR end + +macro survey.add.mergedvodb + if ($0 != 4) + echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)" + break + end + book newpage SURVEY_MERGEDVODB $1 + book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2 + book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3 +end + +macro survey.del.mergedvodb + if ($0 != 2) + echo "USAGE: survey.del.mergedvodb (tag)" + break + end + book delpage SURVEY_MERGEDVODB $1 +end + +macro survey.show.mergedvodb + if ($0 != 1) + echo "USAGE: survey.show.mergedvodb" + break + end + book listbook SURVEY_MERGEDVODB +end + # user functions to manipulate destreak labels @@ -826,11 +843,12 @@ if ("$stage" == "cam") #only queue destreaked cams. stacks and staticsky don't need this - if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") + # if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked + # now with no magic, all are uncensored $run = $run -uncensored - else +# else #if not isp, then run destreaked version only - $run = $run -destreaked - end +# $run = $run -destreaked +# end end if ("$stage" == "staticsky") @@ -875,4 +893,82 @@ end end + + +task survey.mergedvodb + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.mergedvodb.log + stderr $LOGDIR/survey.mergedvodb.log + + task.exec + book npages SURVEY_MERGEDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All labels have been done --- reset + # echo "Resetting labels" + for i 0 $N + book getpage SURVEY_MERGEDVODB $i -var tag + book setword SURVEY_MERGEDVODB $tag STATE NEW + end + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + + # Select different database + $SURVEY_MERGEDVODB_DB ++ + if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0 + end + book setword SURVEY_MERGEDVODB $tag STATE DONE + book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb + book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group + + # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb + #$run = mergetool + $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB + option $DB:$SURVEY_MERGEDVODB_DB + end + #this run doesn't work because there's no workdir + #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group + + + # echo $run + + echo $run + command $run + end + + # success + + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 33029) +++ /trunk/ippTools/share/Makefile.am (revision 33030) @@ -310,4 +310,14 @@ minidvodbtool_queue_minidvodb_id.sql \ minidvodbtool_revertminidvodbcopy.sql \ + mergetool_find_minidvodb_id.sql \ + mergetool_find_mergedvodbcopy.sql \ + mergetool_find_merge_id_dvo.sql \ + mergetool_find_pendingmergeprocess.sql \ + mergetool_find_pending_merge.sql \ + mergetool_find_processed.sql \ + mergetool_queue_mergecopy_id.sql \ + mergetool_queue_merge_id.sql \ + mergetool_revertmergedvodbcopy.sql \ + mergetool_revertmergedvodbprocessed.sql \ pstamptool_completedreq.sql \ pstamptool_datastore.sql \ Index: /trunk/ippTools/share/addtool_checkminidvodbaddrun.sql =================================================================== --- /trunk/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33029) +++ /trunk/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33030) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count FROM minidvodbRun Index: /trunk/ippTools/share/addtool_find_minidvodbrun2.sql =================================================================== --- /trunk/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33029) +++ /trunk/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33030) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group FROM minidvodbRun Index: /trunk/ippTools/share/addtool_queue_minidvodbrun.sql =================================================================== --- /trunk/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33029) +++ /trunk/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33030) @@ -4,5 +4,4 @@ minidvodb_name, minidvodb_path, - mergedvodb_path, 'new' Index: /trunk/ippTools/share/mergetool_find_merge_id_dvo.sql =================================================================== --- /trunk/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33030) @@ -0,0 +1,9 @@ +SELECT mergedvodbRun.* FROM mergedvodbRun +JOIN mergedvodbProcessed USING(merge_id) +WHERE ( mergedvodbRun.state = 'merged' OR mergedvodbRun.state = 'full') + AND mergedvodbProcessed.fault = 0 + AND merge_id NOT IN (SELECT merge_id + FROM mergedvodbCopy + JOIN mergedvodbRun USING(mergedvodb_id) + WHERE %s + ) Index: /trunk/ippTools/share/mergetool_find_mergedvodbcopy.sql =================================================================== --- /trunk/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33030) @@ -0,0 +1,7 @@ +SELECT + mergedvodbCopy.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbCopy + JOIN mergedvodbRun using (mergedvodb_id) + + Index: /trunk/ippTools/share/mergetool_find_minidvodb_id.sql =================================================================== --- /trunk/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33030) @@ -0,0 +1,8 @@ +SELECT minidvodbRun.* FROM minidvodbRun +JOIN minidvodbProcessed USING(minidvodb_id) +WHERE minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodb_id NOT IN (SELECT minidvodb_id + FROM mergedvodbRun + JOIN minidvodbRun USING(minidvodb_id) + WHERE %s + ) Index: /trunk/ippTools/share/mergetool_find_pending_merge.sql =================================================================== --- /trunk/ippTools/share/mergetool_find_pending_merge.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_find_pending_merge.sql (revision 33030) @@ -0,0 +1,14 @@ +SELECT + mergedvodbRun.*, + minidvodbRun.minidvodb_path +FROM minidvodbRun +JOIN mergedvodbRun + USING(minidvodb_id) +LEFT JOIN mergedvodbProcessed + USING(merge_id) +WHERE + (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) + AND mergedvodbRun.mergedvodb IS NOT NULL + AND mergedvodbRun.mergedvodb_path IS NOT NULL + + Index: /trunk/ippTools/share/mergetool_find_pendingmergeprocess.sql =================================================================== --- /trunk/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33030) @@ -0,0 +1,5 @@ +SELECT mergedvodbRun.* +FROM mergedvodbRun +LEFT JOIN mergedvodbProcessed +USING(merge_id) +WHERE (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) Index: /trunk/ippTools/share/mergetool_find_processed.sql =================================================================== --- /trunk/ippTools/share/mergetool_find_processed.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_find_processed.sql (revision 33030) @@ -0,0 +1,7 @@ +SELECT + mergedvodbProcessed.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbProcessed +JOIN mergedvodbRun + USING(merge_id) + Index: /trunk/ippTools/share/mergetool_queue_merge_id.sql =================================================================== --- /trunk/ippTools/share/mergetool_queue_merge_id.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_queue_merge_id.sql (revision 33030) @@ -0,0 +1,11 @@ +INSERT INTO mergedvodbRun + SELECT + 0, -- merge_id + minidvodb_id, -- minidvodb_id + '%s', -- mergedvodb + '%s', -- mergedvodb_path + '%s', -- state + FROM minidvodbRun join minidvodbProcessed using (minidvodb_id) + WHERE + minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodbRun.minidvodb_id = %lld Index: /trunk/ippTools/share/mergetool_queue_mergecopy_id.sql =================================================================== --- /trunk/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33030) @@ -0,0 +1,16 @@ +INSERT INTO mergedvodbCopy + SELECT + 0, -- mergedvodbcopy_id + merge_id, -- merge_id + '%s', -- mergedvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM mergedvodbRun + WHERE + (mergedvodbRun.state = 'merged' + OR mergedvodbRun.state = 'full') + AND mergedvodbRun.merge_id = %lld Index: /trunk/ippTools/share/mergetool_queue_minidvodb_id.sql =================================================================== --- /trunk/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33030) @@ -0,0 +1,15 @@ +INSERT INTO minidvodbCopy + SELECT + 0, -- minidvodbcopy_id + minidvodb_id, -- minidvodb_id + '%s', -- minidvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM minidvodbRun + WHERE + minidvodbRun.state = 'merged' + AND minidvodbRun.minidvodb_id = %lld Index: /trunk/ippTools/share/mergetool_revertmergedvodbcopy.sql =================================================================== --- /trunk/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33030) @@ -0,0 +1,7 @@ +UPDATE mergedvodbCopy + -- SET minidvodbCopy.state = 'new', + SET mergedvodbCopy.fault = 0 +WHERE + mergedvodbCopy.state = 'new' + AND mergedvodbCopy.fault >0 + Index: /trunk/ippTools/share/mergetool_revertmergedvodbprocessed.sql =================================================================== --- /trunk/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33030) +++ /trunk/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33030) @@ -0,0 +1,6 @@ +DELETE FROM mergedvodbProcessed +USING mergedvodbProcessed, mergedvodbRun +WHERE + mergedvodbProcessed.merge_id = mergedvodbRun.merge_id + AND mergedvodbProcessed.fault != 0 + AND mergedvodbRun.state = 'new' Index: /trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql =================================================================== --- /trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33029) +++ /trunk/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33030) @@ -1,5 +1,6 @@ SELECT minidvodbRun.* FROM minidvodbRun JOIN minidvodbProcessed USING(minidvodb_id) -WHERE minidvodbRun.state = 'merged' +WHERE minidvodbRun.state = 'merged' + AND minidvodbProcessed.fault = 0 AND minidvodb_id NOT IN (SELECT minidvodb_id FROM minidvodbCopy Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 33029) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 33030) @@ -1964,5 +1964,4 @@ minidvodb_group VARCHAR(64) NOT NULL, minidvodb_path VARCHAR(255) NOT NULL, - mergedvodb_path VARCHAR(255) NOT NULL, state VARCHAR(64) NOT NULL, creation_date TIMESTAMP, @@ -1972,12 +1971,8 @@ CREATE TABLE minidvodbProcessed ( minidvodb_id BIGINT, - merge_order BIGINT, dtime_resort FLOAT, dtime_relphot FLOAT, - dtime_verify FLOAT, - dtime_merge FLOAT, dtime_script FLOAT, epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - mergedvodb_path VARCHAR(255) NOT NULL, fault SMALLINT NOT NULL, UNIQUE KEY (minidvodb_id), @@ -2000,4 +1995,41 @@ FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE vpRun ( vp_id BIGINT AUTO_INCREMENT, Index: /trunk/ippTools/src/addtool.c =================================================================== --- /trunk/ippTools/src/addtool.c (revision 33029) +++ /trunk/ippTools/src/addtool.c (revision 33030) @@ -292,7 +292,8 @@ } else { //This picks only the unmagicked/uncensored ones - if (strcmp(stage,"cam") == 0) { - psStringAppend(&query, " AND (camRun.magicked = 0)"); - } + //if (strcmp(stage,"cam") == 0) { + //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored. + //psStringAppend(&query, " AND (camRun.magicked = 0)"); + //} if (strcmp(stage,"stack") == 0) { psStringAppend(&query, " AND (stackRun.magicked = 0)"); @@ -824,5 +825,5 @@ PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - + PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false); // generate restrictions psMetadata *where = psMetadataAlloc(); @@ -902,5 +903,19 @@ } } - + //if there is a stage_extra1, set it in addRun (it's not known until it is processed) + if (stage_extra1) { + psString setExtra = NULL; + psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1); + psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id); + + if (!p_psDBRunQuery(config->dbh, setExtra)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + + return false; + } + } @@ -1049,5 +1064,5 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -1221,7 +1236,5 @@ PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); - - //optional + //optional PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); @@ -1238,4 +1251,7 @@ if (minidvodb_path) { minidvodbpath = minidvodb_path; + } else { + psError(PS_ERR_UNKNOWN, false, "require minidvodb_path"); + return false; } @@ -1245,6 +1261,5 @@ minidvodb_group, minidvodbpath, - mergedvodb_path, - "new", + "new", 0 )) { @@ -1268,9 +1283,4 @@ psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); } - - if (!minidvodb_path) { - psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); - } - if (minidvodb_path) { psStringAppend(&minidvodb_path,"/%s",minidvodb_name); @@ -1304,5 +1314,4 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); - PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); @@ -1310,5 +1319,4 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); @@ -1345,12 +1353,4 @@ } - if (mergedvodb_path) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); - cnt++; - } - if (minidvodb_group) { if (cnt) { @@ -1396,5 +1396,5 @@ } - psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL"); psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -1444,5 +1444,5 @@ //now flip new -> active - psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL "); psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); if (!p_psDBRunQuery(config->dbh, query2)) { @@ -1621,13 +1621,9 @@ // required PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); // optional - PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); - PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); @@ -1680,12 +1676,8 @@ minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( pendingRow->minidvodb_id, - merge_order, dtime_resort, dtime_relphot, - dtime_verify, - dtime_merge, dtime_script, epoch, - mergedvodb_path, fault ); @@ -1702,67 +1694,4 @@ } - - - - //this finds the # of merged things (for the merge order) - psString query3 = NULL; - psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); - - if (!p_psDBRunQuery(config->dbh, query3)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query3); - return false; - } - psArray *output2 = p_psDBFetchResult(config->dbh); - if (!output2) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output2)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - bool status; - psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); - if (!status) { - - psAbort("failed to lookup value for count column"); - return false; - } - psString final = NULL; - psStringAppend(&final, "%" PRIu64, m_order); - //return false; - psFree(query3); - psFree(output2); - - - - //update the merge_order - - psString query4 = NULL; - psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); - psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); - //printf("%s", query4); - if (!p_psDBRunQuery(config->dbh, query4)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query4); - return false; - } - // -psFree(query4); - - - -// since there is only one exp per 'new' set mindvodbRun.state = 'merged' - psString query2 = NULL ; psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); @@ -1777,12 +1706,4 @@ return false; } - - - - - - - - psFree(row); psFree(pendingRow); @@ -1797,10 +1718,4 @@ return false; } - - - //print the merge_order (why not!) - printf("%s", final); - psFree(final); - return true; } @@ -1961,10 +1876,7 @@ PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); -PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); @@ -1986,13 +1898,4 @@ } - if (merge_order) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - - psStringAppend(&query, " merge_order = %" PRId64, merge_order); - cnt++; - } - if (dtime_relphot) { if (cnt) { @@ -2011,19 +1914,5 @@ } - if (dtime_merge) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_merge = %f", dtime_merge); - cnt++; - } -if (dtime_verify) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_verify = %f", dtime_verify); - cnt++; - } -if (dtime_script) { + if (dtime_script) { if (cnt) { psStringAppend(&query, "%s", comma); Index: /trunk/ippTools/src/addtoolConfig.c =================================================================== --- /trunk/ippTools/src/addtoolConfig.c (revision 33029) +++ /trunk/ippTools/src/addtoolConfig.c (revision 33030) @@ -115,4 +115,6 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); +psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1", 0, "set stage_extra1", 0); + // -processedexp @@ -165,5 +167,4 @@ psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -174,9 +175,7 @@ psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -205,12 +204,8 @@ psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); - psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); - psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -232,9 +227,6 @@ psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); - psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); psMetadata *argSets = psMetadataAlloc(); Index: /trunk/ippTools/src/mergetool.c =================================================================== --- /trunk/ippTools/src/mergetool.c (revision 33030) +++ /trunk/ippTools/src/mergetool.c (revision 33030) @@ -0,0 +1,1227 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingmergeMode(pxConfig *config); +static bool addmergedMode(pxConfig *config); +static bool listmergedMode(pxConfig *config); +static bool revertmergedMode(pxConfig *config); +static bool updatemergedMode(pxConfig *config); +static bool definebyquerymergecopyMode(pxConfig *config); +static bool listmergedvodbcopyMode(pxConfig *config); +static bool revertmergedvodbcopyMode(pxConfig *config); +static bool updatemergedvodbcopyMode(pxConfig *config); + + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = mergetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MERGETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MERGETOOL_MODE_PENDINGMERGE, pendingmergeMode); + MODECASE(MERGETOOL_MODE_ADDMERGED, addmergedMode); + MODECASE(MERGETOOL_MODE_LISTMERGED, listmergedMode); + MODECASE(MERGETOOL_MODE_REVERTMERGED, revertmergedMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGED, updatemergedMode); + MODECASE(MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyMode); + MODECASE(MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyMode); + + + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + // PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + //find the items to queue (uniq minidvodb_id) + psString dvodb_string = NULL; + psString bare_query = NULL; + +bare_query = pxDataGet("mergetool_find_minidvodb_id.sql"); + psStringAppend(&dvodb_string, "mergedvodbRun.mergedvodb = '%s'", mergedvodb); + +if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + psFree(where); + + + psStringAppend(&query, " GROUP BY minidvodb_id"); + + + psTrace("mergetool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); + +if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + if (!mergedvodb_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb_path: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + + psFree(row); + +} + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 minidvodb_id =0; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + minidvodb_id = row->minidvodb_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxmergeQueueByMinidvodbID(config, + minidvodb_id, + mergedvodb, + mergedvodb_path + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue mergedvodb %" PRId64, minidvodb_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + + } + +psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + psString query = NULL; + + query = psStringCopy("UPDATE mergedvodbRun SET"); + + int cnt = 0; + psString comma = ","; + if (state) { + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + + return true; +} + +static bool pendingmergeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_pending_merge.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbPendingMerge", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + // optional + +//PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psString query = pxDataGet("mergetool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + psTrace("mergetool.c", PS_LOG_INFO, "query is:\n%s", query); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + +psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + mergedvodbRunRow *pendingRow = mergedvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + mergedvodbProcessedRow *row = mergedvodbProcessedRowAlloc( + pendingRow->merge_id, + 0, + dtime_verify ? dtime_verify : 0, + dtime_merge ? dtime_merge : 0, + dtime_script ? dtime_script : 0, + epoch, + fault + ); + + if (!mergedvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + +//this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from mergedvodbRun join mergedvodbProcessed using (merge_id) where state = 'full' and mergedvodb = '%s';", mergedvodb); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update mergedvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where merge_id = %" PRIu64, merge_id); + //printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + +// since there is only one exp per 'new' set mergedvodbRun.state = 'full' + + psString query2 = NULL ; + + if (!fault) { + psStringAppend(&query2, "UPDATE mergedvodbRun SET state = 'full' WHERE merge_id = %'" PRIu64, row->merge_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + } + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + + + + + + + + + + + return true; +} + + +static bool listmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_processed.sql"); + +if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + + + // we either add AND (condition) or WHERE (condition): + if (psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND fault != 0"); + } + if (psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND fault = 0"); + } + if (!psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE fault != 0"); + } + if (!psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE fault = 0"); + } + +psFree(where); + + // order by merge_id + psStringAppend(&query, " ORDER BY merge_id"); + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + + return true; +} + + +static bool revertmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbRun.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = NULL; + query = pxDataGet("mergetool_revertmergedvodbprocessed.sql"); + +if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + return true; +} + +static bool updatemergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-merge_id", true, false); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbProcessed.merge_id", "=="); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE mergedvodbProcessed JOIN mergedvodbRun USING (merge_id) SET "); +int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + + + if (dtime_verify) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_verify = %f", dtime_verify); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + +if (dtime_script) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_script = %f", dtime_script); + cnt++; + } +if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error \n%s\n",query ); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + +static bool definebyquerymergecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + pxAddLabelSearchArgs (config, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); // define using camRun label + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(destination_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(mergedvodb_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(last_merged, config->args, "-last_merged", false); + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + psString bare_query = NULL; + if (destination_host) { + psTrace("mergedvodbtool.c", PS_LOG_INFO, "destination_host argument found (%s) using mergedvodbtool_find_merge_id_dvo.sql\n", destination_host); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("mergedvodbtool_find_merge_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "mergedvodbCopy.destination_host = '%s'", destination_host); + } else { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host"); + + return false; + } + + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + if (last_merged) { + psStringAppend(&query, " ORDER BY merge_order DESC LIMIT 1"); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into mergedvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb_rsync_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined mergedvodb_rsync_path: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + if (!destination_host) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + + psFree(row); + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByMergeID(config, + row->merge_id, + destination_host ? destination_host : "NULL", + mergedvodb_rsync_path ?mergedvodb_rsync_path : "NULL" + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue merge_id: %" PRId64, row->merge_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool listmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pending, config->args, "-pending", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("mergedvodbtool_find_mergedvodbcopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault = 0"); + } + psFree(where); + + if (pending) { + //add the cuts for pending (state new, no faults) + psStringAppend(&query, " %s", " AND mergedvodbCopy.state = 'new' AND mergedvodbCopy.fault = 0 AND mergedvodbCopy.destination_host IS NOT NULL AND mergedvodbCopy.mergedvodb_rsync_path IS NOT NULL"); + } + + + // order by epoch + psStringAppend(&query, " ORDER BY mergedvodbcopy_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool revertmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "destination_host", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbCopy.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("mergedvodbtool_revertmergedvodbcopy.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + } + psError(PS_ERR_UNKNOWN, false, "database error %s", query ); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updatemergedvodbcopyMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(mergedvodbcopy_id, config->args, "-mergedvodbcopy_id", false, false); + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(host, config->args, "-host", false, false); + PXOPT_LOOKUP_STR(rsync_path, config->args, "-mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_S16(set_fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(set_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_STR(set_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_F32(dtime, config->args, "-set_dtime", false, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "mergedvodbCopy.state", "=="); + PXOPT_COPY_STR(config->args, where, "-host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_rsync_path", "mergedvodbCopy.mergedvodb_rsync_path", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE mergedvodbCopy JOIN mergedvodbRun USING (merge_id) SET "); + int cnt = 0; + psString comma = ","; + if (set_fault) { + psStringAppend(&query, " mergedvodbCopy.fault = %d", set_fault); + cnt++; + } + if (set_rsync_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.mergedvodb_rsync_path = '%s'" , set_rsync_path); + cnt++; + } + + if (set_host) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.destination_host = '%s'" , set_host); + cnt++; + } + + + if (set_state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.state = '%s'" , set_state); + cnt++; + } + + if (dtime) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodbCopy.dtime = %f", dtime); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + + Index: /trunk/ippTools/src/mergetool.h =================================================================== --- /trunk/ippTools/src/mergetool.h (revision 33030) +++ /trunk/ippTools/src/mergetool.h (revision 33030) @@ -0,0 +1,41 @@ +/* + * mergetool.h + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MERGETOOL_H +#define MERGETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MERGETOOL_MODE_NONE = 0x0, + MERGETOOL_MODE_DEFINEBYQUERY, + MERGETOOL_MODE_UPDATERUN, + MERGETOOL_MODE_PENDINGMERGE, + MERGETOOL_MODE_ADDMERGED, + MERGETOOL_MODE_LISTMERGED, + MERGETOOL_MODE_REVERTMERGED, + MERGETOOL_MODE_UPDATEMERGED, + MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, + MERGETOOL_MODE_LISTMERGEDVODBCOPY, + MERGETOOL_MODE_REVERTMERGEDVODBCOPY, + MERGETOOL_MODE_UPDATEMERGEDVODBCOPY +} mergetoolMode; + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MERGETOOL_H Index: /trunk/ippTools/src/mergetoolConfig.c =================================================================== --- /trunk/ippTools/src/mergetoolConfig.c (revision 33030) +++ /trunk/ippTools/src/mergetoolConfig.c (revision 33030) @@ -0,0 +1,196 @@ +/* + * addtoolConfig.c + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + // pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodbRun.mergedvodb and use as mergedvodb", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_group", PS_META_DUPLICATE_OK, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + // psMetadataAddStr(definebyqueryArge, PS_LIST_TAIL, "-set_mergedvodb", 0, "define mergedvodb"); + + // psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *updaterunArgs = psMetadataAlloc(); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", 0); + psMetadataAddU64(updaterunArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + + + psMetadata *pendingmergeArgs = psMetadataAlloc(); + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(pendingmergeArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + + psMetadata *addmergedArgs = psMetadataAlloc(); + psMetadataAddU64(addmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id (required)", 0); + psMetadataAddStr(addmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb (required)", 0); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); + psMetadataAddTime(addmergedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddS16(addmergedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + + + + psMetadata *listmergedArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + + psMetadata *revertmergedArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(revertmergedArgs, PS_LIST_TAIL, "-fault", 0, "search by minidvodbCopyfault", 0); + psMetadataAddStr(revertmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadata *updatemergedArgs = psMetadataAlloc(); + + psMetadataAddU64(updatemergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(updatemergedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updatemergedArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + // -definebyquerymergecopy + psMetadata *definebyquerymergecopyArgs = psMetadataAlloc(); + psMetadataAddS64(definebyquerymergecopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodb", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "define workdir", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "set destination host", NULL); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-last_merged", 0, "if multiple minidvodbs have been dvomerged between rsyncs, then only queue the most recent merged dvodb for copying", false); + + psMetadata *listmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by mergedvodb_id", 0); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodbRun.mergedvodb", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by mergedvodbCopy.destination_host", NULL); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-pending", 0, "limit to pending items", false); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by destination_host", NULL); + psMetadataAddS16(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by mergedvodbCopyfault", 0); + psMetadata *updatemergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-host", 0, "search by host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb_rsync_path", 0, "search by rsync", NULL); + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + psMetadataAddF32(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_dtime", 0, "set elapsed time for transfer", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "change the mergedvodb_rsync_path", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "change the host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + + + + + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from minindvodbRun", MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "update existing mergedvodbRuns", MERGETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingmerge", "list pending mergedvodbRuns", MERGETOOL_MODE_PENDINGMERGE, pendingmergeArgs); + PXOPT_ADD_MODE("-addmerged", "add processed mergedvodbRun", MERGETOOL_MODE_ADDMERGED, addmergedArgs); + PXOPT_ADD_MODE("-listmerged","list merged dvodbs", MERGETOOL_MODE_LISTMERGED, listmergedArgs); + PXOPT_ADD_MODE("-revertmerged","revert merged dvodbs", MERGETOOL_MODE_REVERTMERGED, revertmergedArgs); + PXOPT_ADD_MODE("-updatemerged","change merged dvodb properties",MERGETOOL_MODE_UPDATEMERGED, updatemergedArgs); + PXOPT_ADD_MODE("-definebyquerymergecopy", "create runs from mergedvodbRun", MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyArgs); + PXOPT_ADD_MODE("-listmergedvodbcopy","list copy mergedvodbs", MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyArgs); + PXOPT_ADD_MODE("-revertmergedvodbcopy","revert copy mergedvobs",MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyArgs); + PXOPT_ADD_MODE("-updatemergedvodbcopy","change mergedvodb copy properties",MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyArgs); + + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: /trunk/ippTools/src/minidvodbtool.c =================================================================== --- /trunk/ippTools/src/minidvodbtool.c (revision 33029) +++ /trunk/ippTools/src/minidvodbtool.c (revision 33030) @@ -89,5 +89,5 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodbRun_id", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); pxAddLabelSearchArgs (config, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); // define using camRun label Index: /trunk/ippTools/src/pxmerge.c =================================================================== --- /trunk/ippTools/src/pxmerge.c (revision 33030) +++ /trunk/ippTools/src/pxmerge.c (revision 33030) @@ -0,0 +1,97 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodb, + char *mergedvodb_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_merge_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbRun SELECT 0, minidvodb_id, '%s', '%s', 'new', NULL FROM minidvodbRun WHERE minidvodbRun.minidvodb_id = %lld and minidvodbRun.state = 'merged'", mergedvodb, mergedvodb_path, (long long) minidvodb_id)) + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + return true; +} + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_mergecopy_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbCopy SELECT 0, merge_id, '%s', '%s', 0, 'new', NULL, 0 FROM mergedvodbRun WHERE mergedvodbRun.merge_id = %lld and (mergedvodbRun.state = 'merged' or mergedvodbRun.state = 'full')", mergedvodb_rsync_path, destination_host, (long long) merge_id)) + + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + + return true; +} Index: /trunk/ippTools/src/pxmerge.h =================================================================== --- /trunk/ippTools/src/pxmerge.h (revision 33030) +++ /trunk/ippTools/src/pxmerge.h (revision 33030) @@ -0,0 +1,41 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMERGE_H +#define PXMERGE_H 1 + +#include + +#include "pxtools.h" + + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodbb, + char *mergedvodb_path + ); + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ); + + +#endif // PXMERGE_H Index: /trunk/psLib/src/fits/psFitsTableNew.c =================================================================== --- /trunk/psLib/src/fits/psFitsTableNew.c (revision 33029) +++ /trunk/psLib/src/fits/psFitsTableNew.c (revision 33030) @@ -130,5 +130,5 @@ if (numCols > 0) { table->columns = psAlloc(sizeof(psFitsTableColumn) * numCols); - memset(table->columns, sizeof(psFitsTableColumn) * numCols, 0); + memset(table->columns, 0, sizeof(psFitsTableColumn) * numCols); } else { table->columns = NULL;