Index: /trunk/dbconfig/changes.txt =================================================================== --- /trunk/dbconfig/changes.txt (revision 24173) +++ /trunk/dbconfig/changes.txt (revision 24174) @@ -1105,2 +1105,23 @@ ALTER TABLE receiveFileset ADD COLUMN dirinfo VARCHAR(255) AFTER state; +-- Playing with the diffs for warp-warp +ALTER TABLE diffRun ADD COLUMN bothways TINYINT DEFAULT 0 AFTER tess_id; +UPDATE diffRun SET bothways = 1 WHERE reduction = 'WARPWARP'; +ALTER TABLE diffRun DROP FOREIGN KEY diffRun_ibfk_1; +ALTER TABLE diffRun DROP COLUMN exp_id; +ALTER TABLE diffRun ADD KEY(label); +ALTER TABLE diffRun ADD COLUMN exposure TINYINT DEFAULT 0 AFTER bothways; +UPDATE diffRun, diffInputSkyfile SET diffRun.exposure = 1 WHERE diffRun.diff_id = diffInputSkyfile.diff_id and diffInputSkyfile.warp1 IS NOT NULL; +ALTER TABLE stackRun ADD KEY(label); + +ALTER TABLE magicRun ADD COLUMN inverse TINYINT NOT NULL DEFAULT 0 AFTER diff_id; +ALTER TABLE diffSkyfile DROP COLUMN uri; + +DROP TABLE IF EXISTS magicSkyfileMask; +ALTER TABLE magicInputSkyfile DROP FOREIGN KEY magicInputSkyfile_ibfk_2; +ALTER TABLE magicInputSkyfile DROP KEY diff_id; +ALTER TABLE magicInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(magic_id,node); +ALTER TABLE magicInputSkyfile DROP COLUMN diff_id; +ALTER TABLE magicNodeResult CHANGE COLUMN uri path_base VARCHAR(255); + +-- Version 1.1.53 Index: /trunk/dbconfig/config.md =================================================================== --- /trunk/dbconfig/config.md (revision 24173) +++ /trunk/dbconfig/config.md (revision 24174) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.52 + pkg_version STR 1.1.53 END Index: /trunk/dbconfig/diff.md =================================================================== --- /trunk/dbconfig/diff.md (revision 24173) +++ /trunk/dbconfig/diff.md (revision 24174) @@ -10,5 +10,6 @@ registered TAI NULL tess_id STR 64 # Key - exp_id S64 0 # fkey(exp_id) ref rawExp(exp_id) + bothways BOOL f + exposure BOOL f magicked BOOL f END @@ -32,5 +33,4 @@ diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) skycell_id STR 64 # - uri STR 255 path_base STR 255 data_state STR 64 Index: /trunk/dbconfig/magic.md =================================================================== --- /trunk/dbconfig/magic.md (revision 24173) +++ /trunk/dbconfig/magic.md (revision 24174) @@ -6,4 +6,5 @@ exp_id S64 0 # Key diff_id S64 0 # Key + inverse BOOL FALSE state STR 64 # Key workdir STR 255 @@ -15,7 +16,9 @@ END +### This is left over from when diffs were composed of a single skycell +### When we're not too busy, it should be deleted in favour of: +### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0 magicInputSkyfile METADATA magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) - diff_id S64 0 # Primary Key fkey(diff_id) ref diffRun(diff_id) node STR 64 # END @@ -30,5 +33,5 @@ magic_id S64 0 # Primary Key fkey(magic_id) ref magicRun(magic_id) node STR 64 # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node) - uri STR 255 + path_base STR 255 fault S16 0 # Key END Index: /trunk/ippScripts/Build.PL =================================================================== --- /trunk/ippScripts/Build.PL (revision 24173) +++ /trunk/ippScripts/Build.PL (revision 24174) @@ -56,5 +56,4 @@ scripts/magic_tree.pl scripts/magic_process.pl - scripts/magic_mask.pl scripts/magic_destreak.pl scripts/ippdb.pl @@ -73,4 +72,5 @@ scripts/ipp_serial_stack.pl scripts/ipp_serial_mops.pl + scripts/ipp_serial_magic.pl scripts/ipp_mops_translate.pl scripts/ipp_simulation_data.pl Index: /trunk/ippScripts/scripts/diff_skycell.pl =================================================================== --- /trunk/ippScripts/scripts/diff_skycell.pl (revision 24173) +++ /trunk/ippScripts/scripts/diff_skycell.pl (revision 24174) @@ -35,5 +35,5 @@ } -my ($diff_id, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect); +my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect); my ($skycell_id, $diff_skyfile_id); GetOptions( @@ -44,4 +44,5 @@ 'threads=s' => \$threads, # Number of threads to use 'outroot=s' => \$outroot, # Output root name + 'inverse' => \$inverse, # Make inverse subtraction? 'reduction=s' => \$reduction, # Reduction class 'verbose' => \$verbose, # Print to stdout @@ -96,5 +97,4 @@ my $camera; # Camera my ($inputMagic, $templateMagic); # Are the inputs been magicked? -my $inverse = 1; # Generate inverse subtraction if warp-warp foreach my $file (@$files) { if (defined $file->{template} and $file->{template}) { @@ -107,5 +107,4 @@ # template is a stack so it doesn't need to be magicked $templateMagic = 1; - $inverse = 0; ## use an explicit stack name for psphot output objects } else { @@ -123,5 +122,4 @@ $inputVariance = "PPSTACK.OUTPUT.VARIANCE"; $inputSources = "PSPHOT.OUT.CMF.MEF"; ## this must be consistent with the value in stack_skycell.pl:161 - $inverse = 0; } else { $inputMask = "PSWARP.OUTPUT.MASK"; Index: /trunk/ippScripts/scripts/ipp_serial_magic.pl =================================================================== --- /trunk/ippScripts/scripts/ipp_serial_magic.pl (revision 24174) +++ /trunk/ippScripts/scripts/ipp_serial_magic.pl (revision 24174) @@ -0,0 +1,157 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Data::Dumper; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ($dbname, # Database name to use + $identifier, # Identifier to process + $workdir_default, # Default working directory + $verbose, # Verbose operations? + $no_op, # No operations? + $no_update, # No updating? + ); +GetOptions( 'dbname=s' => \$dbname, + 'magic_id=s' => \$identifier, + 'workdir=s' => \$workdir_default, + 'verbose' => \$verbose, + 'no-op' => \$no_op, + 'no-update' => \$no_update, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Required options: --dbname --workdir", + -exitval => 3, + ) unless + defined $dbname; + +$workdir_default = `pwd` unless defined $workdir_default; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +# Look for programs we need +my $missing_tools; +my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); +my $magic_tree = can_run('magic_tree.pl') or (warn "Can't find magic_tree.pl" and $missing_tools = 1); +my $magic_process = can_run('magic_process.pl') or (warn "Can't find magic_process.pl" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +# Tree +{ + my @whole; # The whole list for processing + { + my $command = "$magictool -totree -dbname $dbname"; # Command to run + $command .= " -magic_id $identifier" if defined $identifier; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => $verbose ); + die "Unable to get magic tree list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); + } + + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from magictool.\n"; + + foreach my $item (@$list) { + my $magic_id = $item->{magic_id}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $tess_id = $item->{tess_id}; + my $ra = $item->{ra}; + my $dec = $item->{decl}; + my $workdir = $item->{workdir}; + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + + my $outroot = caturi( $workdir, $exp_id, "$exp_id.mgc.$magic_id" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$magic_tree --magic_id $magic_id --camera $camera --tess_id $tess_id --outroot $outroot --ra $ra --dec $dec --dbname $dbname"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => $verbose ); + die "Unable to do magic tree on $magic_id: $error_code\n" if not $success; + } + + @single = (); + } + } +} + +# Process leaf +{ + my @whole; # The whole list for processing + { + my $command = "$magictool -toprocess -dbname $dbname"; # Command to run + $command .= " -magic_id $identifier" if defined $identifier; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => $verbose ); + die "Unable to get magic tree list: $error_code\n" if not $success; + @whole = split /\n/, join( '', @$stdout_buf ); + } + + my @single = (); + + while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or + die "Unable to parse output from magictool.\n"; + + foreach my $item (@$list) { + my $magic_id = $item->{magic_id}; + my $exp_id = $item->{exp_id}; + my $camera = $item->{camera}; + my $node = $item->{node}; + my $workdir = $item->{workdir}; + $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL"); + + my $outroot = caturi( $workdir, $exp_id, "$exp_id.mgc.$magic_id.$node" ); + $ipprc->outroot_prepare( $outroot ); + + my $command = "$magic_process --magic_id $magic_id --camera $camera --node $node --outroot $outroot --dbname $dbname"; + $command .= " --verbose" if defined $verbose; + $command .= " --no-op" if defined $no_op; + $command .= " --no-update" if defined $no_update; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run( command => $command, verbose => $verbose ); + die "Unable to do magic process on $magic_id $node: $error_code\n" if not $success; + } + + @single = (); + } + } +} + + + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + + +__END__ + + Index: /trunk/ippScripts/scripts/magic_definerun.pl =================================================================== --- /trunk/ippScripts/scripts/magic_definerun.pl (revision 24173) +++ /trunk/ippScripts/scripts/magic_definerun.pl (revision 24174) @@ -164,4 +164,7 @@ &my_die("failed to retrieve new magic run information", $PS_EXIT_CONFIG_ERROR) if !$magic_id; +### This is left over from when diffs were composed of a single skycell +### When we're not too busy, it should be deleted in favour of: +### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0 foreach my $diff_skyfile (@$inputs) { my $skycell_id = $diff_skyfile->{skycell_id}; @@ -171,6 +174,7 @@ } my $diff_id = $diff_skyfile->{diff_id}; - my $command = "$magictool -addinputskyfile -magic_id $magic_id -diff_id $diff_id"; - $command .= " -node $skycell_id"; + my $command = "$magictool -addinputskyfile"; + $command .= " -magic_id $magic_id"; + $command .= " -node $skycell_id"; $command .= " -dbname $dbname" if $dbname; Index: /trunk/ippScripts/scripts/magic_destreak.pl =================================================================== --- /trunk/ippScripts/scripts/magic_destreak.pl (revision 24173) +++ /trunk/ippScripts/scripts/magic_destreak.pl (revision 24174) @@ -35,5 +35,5 @@ # Parse the command-line arguments -my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base); +my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base); my ($outroot, $recoveryroot); my ($replace, $remove, $release); @@ -49,4 +49,5 @@ 'uri=s' => \$uri, # uri of the input image 'path_base=s' => \$path_base, # path_base of the input + 'inverse' => \$inverse, # Inverse subtraction? 'cam_path_base=s'=> \$cam_path_base, # path_base of the associated camera run 'outroot=s' => \$outroot, # "directory" for temporary images (may be nebulous) @@ -183,7 +184,8 @@ my $image = $uri; -my ($mask, $ch_mask, $weight, $astrom); +my ($image, $mask, $ch_mask, $weight, $astrom); if ($stage eq "raw") { + $image = $uri; $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ; @@ -191,4 +193,5 @@ # we use the mask output from the camera stage for input and replace # the output of the chip stage with that mask as well. + $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id); $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id); @@ -196,9 +199,12 @@ $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); } elsif ($stage eq "warp") { + $image = $ipprc->filename("PSWARP.OUTPUT", $path_base); $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base); $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base); } elsif ($stage eq "diff") { - $mask = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base); - $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base); + my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images + $image = $ipprc->filename($name, $path_base); + $mask = $ipprc->filename("$name.MASK", $path_base); + $weight = $ipprc->filename("$name.VARIANCE", $path_base); } Index: unk/ippScripts/scripts/magic_mask.pl =================================================================== --- /trunk/ippScripts/scripts/magic_mask.pl (revision 24173) +++ (revision ) @@ -1,168 +1,0 @@ -#!/usr/bin/env perl - -use Carp; -use warnings; -use strict; - -## report the program and machine -use Sys::Hostname; -my $host = hostname(); -print "\n\n"; -print "Starting script $0 on $host\n\n"; - -use vars qw( $VERSION ); -$VERSION = '0.01'; - -use IPC::Cmd 0.36 qw( can_run run ); -use PS::IPP::Metadata::Config; -use PS::IPP::Metadata::List qw( parse_md_list ); - -use Astro::FITS::CFITSIO qw( :constants ); -Astro::FITS::CFITSIO::PerlyUnpacking(1); - -use PS::IPP::Config 1.01 qw( :standard ); - -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -# Look for programs we need -my $missing_tools; -my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1); -my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1); -if ($missing_tools) { - warn("Can't find required tools."); - exit($PS_EXIT_CONFIG_ERROR); -} - -# Parse the command-line arguments -my ($magic_id, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile); -GetOptions( - 'magic_id=s' => \$magic_id, # Magic identifier - 'camera=s' => \$camera, # Camera name - 'dbname=s' => \$dbname, # Database name - 'outroot=s' => \$outroot, # Output root name - 'save-temps' => \$save_temps, # Save temporary files? - 'verbose' => \$verbose, # Print stuff? - 'no-update' => \$no_update, # Don't update the database? - 'no-op' => \$no_op, # Don't do any operations? - 'logfile=s' => \$logfile, # Redirect output? - ) or pod2usage( 2 ); - -pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --magic_id --camera --outroot", - -exitval => 3) unless - defined $magic_id and - defined $camera and - defined $outroot; - -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $PS_EXIT_SYS_ERROR ) if $logfile; - -my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -### Get a list of inputs -my $inputs; # List of inputs -{ - my $command = "$magictool -inputs -magic_id $magic_id -node root"; # Command to run - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magictool -inputs: $error_code", $magic_id, $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); - - $inputs = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); -} - -### Do the heavy lifting -{ - my $command = "$streaksremove --streaks"; # Command to execute - my @basenames; # List of base names - - # Concatenate the names - foreach my $node (@$inputs) { - push @basenames, $ipprc->file_resolve( $node->{path_base} ); - } - $command .= ' ' . join(' ', @basenames); - - unless ($no_op) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform RemoveStreaks: $error_code", $magic_id, $error_code); - } - - foreach my $basename (@basenames) { - file_check( $basename . '.mask' ); - } - } else { - print "Skipping command: $command\n"; - } -} - - - -### Input mask into database -{ - my $command = "$magictool -addmask"; - $command .= " -magic_id $magic_id"; - $command .= " -uri $outroot"; - $command .= " -dbname $dbname" if defined $dbname; - - # Add the processed file to the database - unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $error_code); - } - } else { - print "Skipping command: $command\n"; - } -} - - - -### Pau. - -sub file_check -{ - my $file = shift; # Name of file - &my_die("Unable to find output file: $file", $magic_id, $PS_EXIT_SYS_ERROR) unless - $ipprc->file_exists($file); -} - -sub my_die -{ - my $msg = shift; # Warning message on die - my $magic_id = shift; # Magic identifier - my $exit_code = shift; # Exit code to add - - $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; - - carp($msg); - if (defined $magic_id and not $no_update) { - my $command = "$magictool -addmask"; - $command .= " -magic_id $magic_id"; - $command .= " -fault $exit_code"; - $command .= " -dbname $dbname" if defined $dbname; - system($command); - } - exit $exit_code; -} - -END { - my $status = $?; - system("sync") == 0 - or die "failed to execute sync: $!" ; - $? = $status; -} - -__END__ Index: /trunk/ippScripts/scripts/magic_process.pl =================================================================== --- /trunk/ippScripts/scripts/magic_process.pl (revision 24173) +++ /trunk/ippScripts/scripts/magic_process.pl (revision 24174) @@ -37,5 +37,5 @@ # Parse the command-line arguments -my ($magic_id, $node, $camera, $dbname, $outroot, $template_uri, $save_temps, $verbose, $no_update, $no_op, $logfile); +my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile); GetOptions( @@ -45,5 +45,4 @@ 'dbname=s' => \$dbname, # Database name 'outroot=s' => \$outroot, # Output root name - 'template_uri=s' => \$template_uri,# uri of diff template 'save-temps' => \$save_temps, # Save temporary files? 'verbose' => \$verbose, # Print stuff? @@ -59,6 +58,5 @@ defined $node and defined $camera and - defined $outroot and - defined $template_uri; + defined $outroot; my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $node, $PS_EXIT_CONFIG_ERROR ); # IPP configuration @@ -138,7 +136,7 @@ } - my $template = $ipprc->file_resolve($template_uri); - &my_die("failed to resolve template_uri: $template_uri", $magic_id, $node, $PS_EXIT_DATA_ERROR) - if !($template); + my $template = resolve_template($innode); + &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR) + unless defined $template; $command .= " --detect --image $image --mask $mask --weight $weight -k $template"; @@ -189,13 +187,13 @@ foreach my $innode (@$inputs) { # root for inputs from previous stage - my $in_uri = $innode->{uri}; - print $sfh "$in_uri\n"; - - cat_list_to_list($in_fh, $in_uri, "input.list"); + my $in_path_base = $innode->{magic_path_base}; + print $sfh "$in_path_base\n"; + + cat_list_to_list($in_fh, $in_path_base, "input.list"); # build input lists by combining the lists from # previous stages - cat_list_to_list($ifh, $in_uri, "image.list"); - cat_list_to_list($mfh, $in_uri, "mask.list"); - cat_list_to_list($wfh, $in_uri, "weight.list"); + cat_list_to_list($ifh, $in_path_base, "image.list"); + cat_list_to_list($mfh, $in_path_base, "mask.list"); + cat_list_to_list($wfh, $in_path_base, "weight.list"); } close $in_fh; @@ -243,5 +241,5 @@ $command .= " -magic_id $magic_id"; $command .= " -node $node"; - $command .= " -uri $outroot"; + $command .= " -path_base $outroot"; $command .= " -dbname $dbname" if defined $dbname; @@ -262,12 +260,15 @@ my $resolved = $ipprc->file_resolve($streaks_file); - my $fh; - open $fh, "<$resolved" or - &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); - # the first line in the streaks file contains the number of streaks found - my $num_streaks = <$fh>; - chomp $num_streaks; - close $fh; - print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose; + my $num_streaks = -1; + unless ($no_op) { + my $fh; + open $fh, "<$resolved" or + &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR); + # the first line in the streaks file contains the number of streaks found + $num_streaks = <$fh>; + chomp $num_streaks; + close $fh; + print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose; + } my $command = "$magictool -addmask"; @@ -309,8 +310,8 @@ sub cat_list_to_list { my $out = shift; # output file handle - my $uri = shift; # uri to append ... + my $path_base = shift; # path_base to append ... my $extension = shift; # ... the extension to - my $filename = "$uri.$extension"; + my $filename = "$path_base.$extension"; my $in; @@ -321,14 +322,46 @@ } -sub resolve_inputs { - +sub resolve_inputs +{ my $node = shift; - my $input_base = $node->{path_base}; - - my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base)); - my $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base)); - my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.VARIANCE", $input_base)); - - return ($image, $mask, $weight); + my $input_base = $node->{diff_path_base}; + + my ($image, $mask, $variance); # Names to return + if ($node->{inverse}) { + $image = "PPSUB.INVERSE"; + $mask = "PPSUB.INVERSE.MASK"; + $variance = "PPSUB.INVERSE.VARIANCE"; + } else { + $image = "PPSUB.OUTPUT"; + $mask = "PPSUB.OUTPUT.MASK"; + $variance = "PPSUB.OUTPUT.VARIANCE"; + } + + $image = $ipprc->file_resolve($ipprc->filename($image, $input_base)); + $mask = $ipprc->file_resolve($ipprc->filename($mask, $input_base)); + $variance= $ipprc->file_resolve($ipprc->filename($variance, $input_base)); + + return ($image, $mask, $variance); +} + +sub resolve_template +{ + my $node = shift; + + my $image; # Name of template to return + my $path_base; # Base name for name + if (defined $node->{warp_path_base} and $node->{warp_path_base} ne "NULL") { + $path_base = $node->{warp_path_base}; + $image = "PSWARP.OUTPUT"; + } elsif (defined $node->{stack_path_base} and $node->{stack_path_base} ne "NULL") { + $path_base = $node->{stack_path_base}; + $image = "PPSTACK.OUTPUT"; + } else { + return undef; + } + + $image = $ipprc->file_resolve($ipprc->filename($image, $path_base)); + + return $image } Index: /trunk/ippScripts/scripts/magic_tree.pl =================================================================== --- /trunk/ippScripts/scripts/magic_tree.pl (revision 24173) +++ /trunk/ippScripts/scripts/magic_tree.pl (revision 24174) @@ -73,31 +73,4 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files -### Get a list of warpSkyfiles -### This is a workaround to allow processing with older diffSkyfiles that don't have -### a wcs. If warp_id is provided, we get the wcs from there -my %warpSkyfiles; # hash of warps -if ($warp_id) { - my $command = "$warptool -warped -warp_id $warp_id"; # Command to run - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform warptool -warped: $error_code", $magic_id, $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR); - - my $warps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR); - - # make a hash indexed by skycell_id - foreach my $warp ( @$warps ) { - my $skycell_id = $warp->{skycell_id}; - die "failed to lookup skycell_id from warp" if !$skycell_id; - $warpSkyfiles{$skycell_id} = $warp; - } -} ### Get a list of skycells my @skycells; # List of skycells @@ -126,29 +99,9 @@ my @fields; foreach my $input ( @skycells ) { - # the filename method doesn't add the $skycell_id - - # my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); - #my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); - -# my $skycellBase = "$outroot/$skycell_id"; -# my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skycellBase); -# $ipprc->skycell_file($tess_id, $skycell_id, $skyfile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR); - - my $skyfile; - my $skycell_id = $input->{node}; - if ($warp_id) { - # Applying the workaround - my $warp = $warpSkyfiles{$skycell_id}; - die "warpSkyfile for $skycell_id not found" if !$warp; - my $skyfileBase = $warp->{path_base}; - $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skyfileBase); - } else { - # using the diffSkyfile - $skyfile = $input->{uri}; - } - my $skyfileResolved = $ipprc->file_resolve( $skyfile ); - -# my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved ); -# &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status; + # We use the WCS in the diff image + my $name = "PPSUB.OUTPUT"; # Name of file + my $skycell_id = $input->{node}; # Name of skycell + my $skyfile = $ipprc->filename($name, $input->{path_base}, $skycell_id); # Filename for diff + my $skyfileResolved = $ipprc->file_resolve( $skyfile ); # Resolved filename my ($header, $status) = (undef, 0); @@ -162,5 +115,4 @@ # Get the useful header keywords -# my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR); my $naxis1 = $$header{'NAXIS1'}; my $naxis2; @@ -181,16 +133,16 @@ $naxis2 = $$header{'ZNAXIS2'} or &my_die("Can't find ZNAXIS2", $magic_id, $PS_EXIT_SYS_ERROR); } - my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR); - my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR); - my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR); - my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR); - my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR); - my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR); - my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR); - my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR); - my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR); - my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR); - my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR); - my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); + my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR); my $crota1 = $$header{'CROTA1'}; my $crota2 = $$header{'CROTA2'}; Index: /trunk/ippTasks/diff.pro =================================================================== --- /trunk/ippTasks/diff.pro (revision 24173) +++ /trunk/ippTasks/diff.pro (revision 24174) @@ -146,4 +146,5 @@ book getword diffSkyfile $pageName skycell_id -var SKYCELL_ID book getword diffSkyfile $pageName camera -var CAMERA + book getword diffSkyfile $pageName bothways -var BOTHWAYS book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE book getword diffSkyfile $pageName dbname -var DBNAME @@ -164,4 +165,7 @@ $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output + if ($BOTHWAYS > 1) + $run = $run --inverse + end add_standard_args run Index: /trunk/ippTasks/magic.pro =================================================================== --- /trunk/ippTasks/magic.pro (revision 24173) +++ /trunk/ippTasks/magic.pro (revision 24174) @@ -280,5 +280,4 @@ book getword magicToProcess $pageName camera -var CAMERA book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE - book getword magicToProcess $pageName template_uri -var TEMPLATE_URI book getword magicToProcess $pageName dbname -var DBNAME @@ -302,5 +301,5 @@ mkdir $outpath - $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --template_uri $TEMPLATE_URI --logfile $logfile + $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile add_standard_args run @@ -354,6 +353,6 @@ if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0 end -# XXX magicDSRun doesn't have label so can't use add_poll_args -# add_poll_args run + add_poll_args run + add_poll_labels run command $run end @@ -410,4 +409,5 @@ book getword magicToDS $pageName uri -var URI book getword magicToDS $pageName path_base -var PATH_BASE + book getword magicToDS $pageName inverse -var INVERSE book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE book getword magicToDS $pageName outroot -var OUTROOT @@ -424,4 +424,8 @@ $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --remove $REMOVE + if ($INVERSE > 0) + # Inverse subtraction + $run = $run --inverse + end add_standard_args run Index: /trunk/ippTasks/simtest.basic.auto =================================================================== --- /trunk/ippTasks/simtest.basic.auto (revision 24173) +++ /trunk/ippTasks/simtest.basic.auto (revision 24174) @@ -58,5 +58,5 @@ END -### Stack automation??? +### Stack automation automate METADATA name STR STACK @@ -64,8 +64,8 @@ END -### Diff automation??? +### Diff automation automate METADATA name STR DIFF - regular STR "difftool -definebyquery -label proc -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" + regular STR "difftool -definewarpstack -label proc -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" END Index: /trunk/ippTools/configure.ac =================================================================== --- /trunk/ippTools/configure.ac (revision 24173) +++ /trunk/ippTools/configure.ac (revision 24174) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.52]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.53]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: /trunk/ippTools/share/Makefile.am =================================================================== --- /trunk/ippTools/share/Makefile.am (revision 24173) +++ /trunk/ippTools/share/Makefile.am (revision 24174) @@ -81,7 +81,9 @@ dettool_tostacked.sql \ difftool_completed_runs.sql \ - difftool_definebyquery_part1.sql \ - difftool_definebyquery_part2.sql \ - difftool_definebyquery_temp_create.sql \ + difftool_definewarpstack_part1.sql \ + difftool_definewarpstack_part2.sql \ + difftool_definewarpstack_temp_create.sql \ + difftool_definewarpwarp_temp_create.sql \ + difftool_definewarpwarp_temp_insert.sql \ difftool_definewarpwarp_select.sql \ difftool_definewarpwarp_insert.sql \ Index: unk/ippTools/share/difftool_definebyquery.sql =================================================================== --- /trunk/ippTools/share/difftool_definebyquery.sql (revision 24173) +++ (revision ) @@ -1,58 +1,0 @@ --- Get list of warps that can be diffed, with any associated diff, --- and the best stack to use as a template --- Warps without an existing diff can be identified by a NULL diff_id -SELECT - warpsToDiff.warp_id, - warpsToDiff.skycell_id, - warpsToDiff.tess_id, - warpsToDiff.filter, - warpsToDiff.good_frac, - warpsToDiff.diff_id, - current_stack_id, - best_stack_id, - exp_id -FROM ( - -- Get list of warps that can be diffed, with any associated diff - SELECT - warpSkyfile.warp_id, - warpSkyfile.skycell_id, - warpSkyfile.tess_id, - warpSkyfile.good_frac, - warpRun.label, - filter, - warpRun.label as warp_label, - diffInputs.diff_id, - diffInputs.stack2 AS current_stack_id, - rawExp.exp_id - FROM warpSkyfile - JOIN warpRun USING(warp_id) - JOIN fakeRun USING(fake_id) - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - -- Check if it has an associated diff - LEFT JOIN diffInputSkyfile AS diffInputs - ON diffInputs.warp1 = warpSkyfile.warp_id - AND diffInputs.skycell_id = warpSkyfile.skycell_id - AND diffInputs.stack2 IS NOT NULL - WHERE - warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - -- warpsToDiff WHERE hook %s - ) AS warpsToDiff --- Get best stack as a function of skycell_id, filter -JOIN ( - SELECT - MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment - skycell_id, - filter - FROM stackRun - JOIN stackSumSkyfile USING(stack_id) - WHERE stackRun.state = 'full' - AND stackSumSkyfile.fault = 0 - AND stackSumSkyfile.quality = 0 - -- stacksForDiff WHERE hook %s - GROUP BY - skycell_id, - filter - ) AS stacksForDiff USING(skycell_id, filter) Index: unk/ippTools/share/difftool_definebyquery_part1.sql =================================================================== --- /trunk/ippTools/share/difftool_definebyquery_part1.sql (revision 24173) +++ (revision ) @@ -1,21 +1,0 @@ -SELECT - exp_id, - warp_id, - rawExp.filter, - warpRun.label, - warpRun.tess_id, - COUNT(skycell_id) as skycell_count -FROM warpRun -JOIN warpSkyfile USING(warp_id) -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN rawExp USING(exp_id) -LEFT JOIN diffRun USING(exp_id) -WHERE - warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - -- warp where hook %s - -- exp where hook %s - -- diff where hook %s -GROUP BY exp_id, warp_id DESC Index: unk/ippTools/share/difftool_definebyquery_part2.sql =================================================================== --- /trunk/ippTools/share/difftool_definebyquery_part2.sql (revision 24173) +++ (revision ) @@ -1,36 +1,0 @@ --- insert skycells to diff -INSERT INTO skycellsToDiff -SELECT - 0, -- diff_id - warpSkyfile.skycell_id, - warpSkyfile.warp_id, -- warp1 - NULL, -- stack1 - NULL, -- warp2 - max_stack_id, -- stack2 - warpSkyfile.tess_id, - 0 -FROM warpSkyfile -JOIN warpRun USING(warp_id) -LEFT JOIN ( - SELECT - MAX(stack_id) AS max_stack_id, -- most recent stack, by virtue of auto-increment - stackRun.skycell_id, - stackRun.tess_id, - filter - FROM stackRun - JOIN stackSumSkyfile USING(stack_id) - WHERE stackSumSkyfile.fault = 0 - AND stackSumSkyfile.quality = 0 - AND stackRun.state = 'full' - -- stacks where hook %s - GROUP BY - skycell_id, - filter - ) as bestStacks USING(skycell_id) -WHERE - warpSkyfile.warp_id = %lld - AND warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - AND filter = '%s' - AND warpSkyfile.tess_id = bestStacks.tess_id --- warp where hook %s Index: unk/ippTools/share/difftool_definebyquery_temp_create.sql =================================================================== --- /trunk/ippTools/share/difftool_definebyquery_temp_create.sql (revision 24173) +++ (revision ) @@ -1,10 +1,0 @@ -CREATE TEMPORARY TABLE skycellsToDiff ( -diff_id BIGINT, -skycell_id VARCHAR(64), -warp1 BIGINT, -stack1 BIGINT, -warp2 BIGINT, -stack2 BIGINT, -tess_id VARCHAR(64), -diff_skycell_id BIGINT -) ENGINE=MEMORY; Index: /trunk/ippTools/share/difftool_definewarpstack.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpstack.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpstack.sql (revision 24174) @@ -0,0 +1,58 @@ +-- Get list of warps that can be diffed, with any associated diff, +-- and the best stack to use as a template +-- Warps without an existing diff can be identified by a NULL diff_id +SELECT + warpsToDiff.warp_id, + warpsToDiff.skycell_id, + warpsToDiff.tess_id, + warpsToDiff.filter, + warpsToDiff.good_frac, + warpsToDiff.diff_id, + current_stack_id, + best_stack_id, + exp_id +FROM ( + -- Get list of warps that can be diffed, with any associated diff + SELECT + warpSkyfile.warp_id, + warpSkyfile.skycell_id, + warpSkyfile.tess_id, + warpSkyfile.good_frac, + warpRun.label, + filter, + warpRun.label as warp_label, + diffInputs.diff_id, + diffInputs.stack2 AS current_stack_id, + rawExp.exp_id + FROM warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + -- Check if it has an associated diff + LEFT JOIN diffInputSkyfile AS diffInputs + ON diffInputs.warp1 = warpSkyfile.warp_id + AND diffInputs.skycell_id = warpSkyfile.skycell_id + AND diffInputs.stack2 IS NOT NULL + WHERE + warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + -- warpsToDiff WHERE hook %s + ) AS warpsToDiff +-- Get best stack as a function of skycell_id, filter +JOIN ( + SELECT + MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment + skycell_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackRun.state = 'full' + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + -- stacksForDiff WHERE hook %s + GROUP BY + skycell_id, + filter + ) AS stacksForDiff USING(skycell_id, filter) Index: /trunk/ippTools/share/difftool_definewarpstack_part1.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpstack_part1.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpstack_part1.sql (revision 24174) @@ -0,0 +1,33 @@ +SELECT + exp_id, + warp_id, + rawExp.filter, + warpRun.label, + warpRun.tess_id, + COUNT(skycell_id) as skycell_count +FROM warpRun +JOIN warpSkyfile USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN ( + SELECT DISTINCT + diffRun.*, + exp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun + ON warpRun.warp_id = diffInputSkyfile.warp1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + WHERE warp1 IS NOT NULL +) AS diffExp USING(exp_id) +WHERE + warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + -- warp where hook %s + -- exp where hook %s + -- diff where hook %s +GROUP BY exp_id, warp_id DESC Index: /trunk/ippTools/share/difftool_definewarpstack_part2.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpstack_part2.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpstack_part2.sql (revision 24174) @@ -0,0 +1,36 @@ +-- insert skycells to diff +INSERT INTO skycellsToDiff +SELECT + 0, -- diff_id + warpSkyfile.skycell_id, + warpSkyfile.warp_id, -- warp1 + NULL, -- stack1 + NULL, -- warp2 + max_stack_id, -- stack2 + warpSkyfile.tess_id, + 0 +FROM warpSkyfile +JOIN warpRun USING(warp_id) +LEFT JOIN ( + SELECT + MAX(stack_id) AS max_stack_id, -- most recent stack, by virtue of auto-increment + stackRun.skycell_id, + stackRun.tess_id, + filter + FROM stackRun + JOIN stackSumSkyfile USING(stack_id) + WHERE stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND stackRun.state = 'full' + -- stacks where hook %s + GROUP BY + skycell_id, + filter + ) as bestStacks USING(skycell_id) +WHERE + warpSkyfile.warp_id = %lld + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND filter = '%s' + AND warpSkyfile.tess_id = bestStacks.tess_id +-- warp where hook %s Index: /trunk/ippTools/share/difftool_definewarpstack_temp_create.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpstack_temp_create.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpstack_temp_create.sql (revision 24174) @@ -0,0 +1,10 @@ +CREATE TEMPORARY TABLE skycellsToDiff ( +diff_id BIGINT, +skycell_id VARCHAR(64), +warp1 BIGINT, +stack1 BIGINT, +warp2 BIGINT, +stack2 BIGINT, +tess_id VARCHAR(64), +diff_skycell_id BIGINT +) ENGINE=MEMORY; Index: /trunk/ippTools/share/difftool_definewarpwarp_select.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpwarp_select.sql (revision 24173) +++ /trunk/ippTools/share/difftool_definewarpwarp_select.sql (revision 24174) @@ -1,6 +1,7 @@ SELECT inputWarpRun.warp_id AS input_warp_id, + inputWarpRun.tess_id AS tess_id, inputRawExp.exp_id AS input_exp_id, - inputWarpRun.tess_id AS tess_id, + -- The following trick pulls out the warp_id that has the smallest distance SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id FROM warpRun AS inputWarpRun @@ -9,5 +10,5 @@ JOIN chipRun AS inputChipRun USING(chip_id) JOIN rawExp AS inputRawExp USING(exp_id) --- To find exposures that haven't been diffed:%s LEFT JOIN diffRun ON diffRun.exp_id = inputRawExp.exp_id +-- To find exposures that haven't been diffed, insert newline here:%s LEFT JOIN diffs USING(exp_id) JOIN warpRun AS templateWarpRun ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template! Index: /trunk/ippTools/share/difftool_definewarpwarp_temp_create.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpwarp_temp_create.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpwarp_temp_create.sql (revision 24174) @@ -0,0 +1,8 @@ +-- Exposures that have diffs +-- Temporary table has useful indices that make future query faster! +CREATE TEMPORARY TABLE diffs ( + diff_id BIGINT, + exp_id BIGINT, + PRIMARY KEY(diff_id, exp_id), + KEY(exp_id) +) ENGINE=MEMORY; Index: /trunk/ippTools/share/difftool_definewarpwarp_temp_insert.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpwarp_temp_insert.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpwarp_temp_insert.sql (revision 24174) @@ -0,0 +1,32 @@ +-- Get list of exposures that have diffs +-- Only interested in whole exposures (diffRun.exposure = 1) +INSERT INTO diffs +SELECT DISTINCT + diffWarps.diff_id, + exp_id +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE diffInputSkyfile.warp1 IS NOT NULL + AND diffRun.exposure = 1 + -- WHERE hook %s + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE diffInputSkyfile.warp2 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.bothways = 1 + -- WHERE hook %s + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) Index: /trunk/ippTools/share/difftool_definewarpwarp_test.sql =================================================================== --- /trunk/ippTools/share/difftool_definewarpwarp_test.sql (revision 24174) +++ /trunk/ippTools/share/difftool_definewarpwarp_test.sql (revision 24174) @@ -0,0 +1,82 @@ +DROP TABLE IF EXISTS diffs; + +-- Create a temporary table with useful indices +CREATE TEMPORARY TABLE diffs ( + diff_id BIGINT, + exp_id BIGINT, + PRIMARY KEY(diff_id, exp_id), + KEY(exp_id) +) ENGINE=MEMORY; + + +-- Get list of exposures that have diffs +INSERT INTO diffs +SELECT DISTINCT + diffWarps.diff_id, + exp_id +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE warp1 IS NOT NULL + -- WHERE hook %s + AND diffRun.label = 'ThreePi.Run2.r.v0' + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id + FROM diffRun + JOIN diffInputSkyfile + ON diffInputSkyfile.diff_id = diffRun.diff_id + AND diffRun.bothways = 1 + WHERE warp2 IS NOT NULL + -- WHERE hook %s + AND diffRun.label = 'ThreePi.Run2.r.v0' + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +; + + + +-- EXPLAIN +SELECT + inputWarpRun.warp_id AS input_warp_id, + inputWarpRun.tess_id AS tess_id, + inputRawExp.exp_id AS input_exp_id, + SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id +FROM warpRun AS inputWarpRun +JOIN fakeRun AS inputFakeRun USING(fake_id) +JOIN camRun AS inputCamRun USING(cam_id) +JOIN chipRun AS inputChipRun USING(chip_id) +JOIN rawExp AS inputRawExp USING(exp_id) +-- To find exposures that haven't been diffed, insert newline here:%s + LEFT JOIN diffs USING(exp_id) +JOIN warpRun AS templateWarpRun + ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template! + AND templateWarpRun.tess_id = inputWarpRun.tess_id -- Ensure using same tessellation +JOIN fakeRun AS templateFakeRun + ON templateFakeRun.fake_id = templateWarpRun.fake_id +JOIN camRun AS templateCamRun + ON templateCamRun.cam_id = templateFakeRun.cam_id +JOIN chipRun AS templateChipRun + ON templateChipRun.chip_id = templateCamRun.chip_id +JOIN rawExp AS templateRawExp + ON templateRawExp.exp_id = templateChipRun.exp_id + AND templateRawExp.filter = inputRawExp.filter +-- WHERE hook %s +WHERE (inputWarpRun.label = 'ThreePi.Run2.r.v0') + AND ((DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2)))) < 0.10000000 + 0.00000119)) + AND ((TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)) > 0.00000000 - 0.00000119)) + AND (templateWarpRun.label = 'ThreePi.Run2.r.v0') +-- AND (diffs.diff_id IS NULL) + AND inputWarpRun.state = 'full' +-- WHERE done +GROUP BY input_warp_id +; Index: /trunk/ippTools/share/difftool_skyfile.sql =================================================================== --- /trunk/ippTools/share/difftool_skyfile.sql (revision 24173) +++ /trunk/ippTools/share/difftool_skyfile.sql (revision 24174) @@ -8,13 +8,12 @@ stack1, warp2, - stack2, - exp_id + stack2 FROM diffRun -JOIN diffSkyfile - USING(diff_id) -JOIN diffInputSkyfile - USING (diff_id, skycell_id) +JOIN diffSkyfile USING(diff_id) +JOIN diffInputSkyfile USING(diff_id, skycell_id) JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -JOIN rawExp - USING(exp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: /trunk/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- /trunk/ippTools/share/difftool_todiffskyfile.sql (revision 24173) +++ /trunk/ippTools/share/difftool_todiffskyfile.sql (revision 24174) @@ -9,5 +9,6 @@ diffRun.tess_id, diffRun.label, - diffRun.state + diffRun.state, + diffRun.bothways FROM diffRun JOIN diffInputSkyfile USING(diff_id) Index: /trunk/ippTools/share/magicdstool_todestreak.sql =================================================================== --- /trunk/ippTools/share/magicdstool_todestreak.sql (revision 24173) +++ /trunk/ippTools/share/magicdstool_todestreak.sql (revision 24174) @@ -13,4 +13,5 @@ rawImfile.uri as uri, NULL as path_base, + magicRun.inverse, camProcessedExp.path_base as cam_path_base, outroot, @@ -44,4 +45,5 @@ chipProcessedImfile.uri, chipProcessedImfile.path_base, + magicRun.inverse, camProcessedExp.path_base as cam_path_base, outroot, @@ -51,4 +53,5 @@ FROM magicDSRun JOIN magicMask USING (magic_id) +JOIN magicRun USING(magic_id) JOIN camProcessedExp USING(cam_id) JOIN chipRun ON chip_id = stage_id @@ -79,4 +82,5 @@ warpSkyfile.uri, warpSkyfile.path_base, + magicRun.inverse, NULL as cam_path_base, outroot, @@ -113,6 +117,7 @@ magicRun.diff_id as stage_id, diffSkyfile.skycell_id as component, - diffSkyfile.uri, + NULL AS uri, diffSkyfile.path_base, + magicRun.inverse, NULL as cam_path_base, outroot, @@ -126,5 +131,5 @@ JOIN magicInputSkyfile USING(magic_id) JOIN diffSkyfile - ON magicInputSkyfile.diff_id = diffSkyfile.diff_id + ON magicRun.diff_id = diffSkyfile.diff_id AND magicInputSkyfile.node = diffSkyfile.skycell_id LEFT JOIN magicDSFile Index: /trunk/ippTools/share/magictool_definebyquery_insert.sql =================================================================== --- /trunk/ippTools/share/magictool_definebyquery_insert.sql (revision 24173) +++ /trunk/ippTools/share/magictool_definebyquery_insert.sql (revision 24174) @@ -3,5 +3,4 @@ SELECT @MAGIC_ID@, -- Update this with the appropriate magic_id - diff_id, skycell_id FROM diffSkyfile Index: /trunk/ippTools/share/magictool_definebyquery_select.sql =================================================================== --- /trunk/ippTools/share/magictool_definebyquery_select.sql (revision 24173) +++ /trunk/ippTools/share/magictool_definebyquery_select.sql (revision 24174) @@ -1,8 +1,36 @@ -- Get a list of exposures on which magic may be performed -SELECT +SELECT exp_id, - MAX(diffRun.diff_id) AS diff_id -FROM diffRun -JOIN rawExp USING(exp_id) + MAX(diffWarps.diff_id) AS diff_id, + -- The following trick pulls out the 'inverse' value for the maximum diff_id + CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id, + 0 AS inverse + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE diffInputSkyfile.warp1 IS NOT NULL + AND diffRun.exposure = 1 + -- WHERE hook %s + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id, + 1 AS inverse + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + WHERE diffInputSkyfile.warp2 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.bothways = 1 + -- WHERE hook %s + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) LEFT JOIN magicRun USING(exp_id) -- WHERE hook %s Index: /trunk/ippTools/share/magictool_inputs.sql =================================================================== --- /trunk/ippTools/share/magictool_inputs.sql (revision 24173) +++ /trunk/ippTools/share/magictool_inputs.sql (revision 24174) @@ -1,39 +1,73 @@ -SELECT * -FROM ( --- Single skycells +-- Get input details for a magic node SELECT magicRun.magic_id, - magicRun.state, - magicInputSkyfile.node, - diffSkyfile.diff_id, - diffSkyfile.uri, - diffSkyfile.path_base, - diffSkyfile.fault, - diffSkyfile.quality + magicRun.inverse, -- Using the inverse subtraction? + magicRun.diff_id, + magicInputs.node, + -- Only one of diff_path_base and magic_path_base should be non-NULL + -- If diff_path_base is non-NULL, then only one of warp_path_base and stack_path_base should be non-NULL + magicInputs.diff_path_base, -- path_base for the diff (if any) + magicInputs.warp_path_base, -- path_base for the template warp (if any) + magicInputs.stack_path_base, -- path_base for the template stack (if any) + magicInputs.magic_path_base -- path_base for child nodes (if any) FROM magicRun -JOIN magicInputSkyfile -USING(magic_id) -JOIN diffSkyfile - ON magicInputSkyfile.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -UNION --- Merged skycells -SELECT - magicRun.magic_id, - magicRun.state, - magicTree.node, - 0, -- no diff_id - magicNodeResult.uri, - NULL, -- magicNodeResult doesn't have a path_base - magicNodeResult.fault, - 0 as quality -FROM magicTree -JOIN magicRun - USING(magic_id) -JOIN magicNodeResult - ON magicTree.magic_id = magicNodeResult.magic_id - AND magicTree.dep = magicNodeResult.node -) as Foo -WHERE - fault = 0 - AND quality = 0 +JOIN ( + -- Single skycells: have uri=NULL + SELECT + magic_id, + magicRun.diff_id, + skycell_id AS node, + diffSkyfile.path_base AS diff_path_base, + warpSkyfile.path_base AS warp_path_base, + stackSumSkyfile.path_base AS stack_path_base, + NULL AS magic_path_base + FROM magicRun + JOIN magicTree USING(magic_id) + JOIN magicInputSkyfile USING(magic_id, node) + JOIN diffRun USING(diff_id) + JOIN diffSkyfile + ON diffSkyfile.diff_id = magicRun.diff_id + AND diffSkyfile.skycell_id = magicInputSkyfile.node + JOIN ( + -- Template for non-inverse + SELECT + magic_id, + skycell_id, + warp1 AS warp_id, + stack1 AS stack_id + FROM magicRun + JOIN diffInputSkyfile USING(diff_id) + WHERE magicRun.inverse = 0 + -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s + UNION + -- Template for inverse + SELECT + magic_id, + skycell_id, + warp2 AS warp_id, + stack2 AS stack_id + FROM magicRun + JOIN diffInputSkyfile USING(diff_id) + WHERE magicRun.inverse = 1 + -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s + ) AS diffTemplates USING(magic_id, skycell_id) + LEFT JOIN warpSkyfile USING(warp_id, skycell_id) + LEFT JOIN stackSumSkyfile USING(stack_id) + -- WHERE hook (magicRun.magic_id, magicTree.node) %s + UNION + -- Merged skycells: have diff_id=0, various_path_base=NULL + SELECT + magicTree.magic_id, + 0 AS diff_id, + magicTree.dep, + NULL AS diff_path_base, + NULL AS warp_path_base, + NULL AS stack_path_base, + magicNodeResult.path_base AS magic_path_base + FROM magicTree + JOIN magicRun USING(magic_id) + JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.dep = magicNodeResult.node + -- WHERE hook (magicRun.magic_id, magicTree.node) %s + ) AS magicInputs USING(magic_id) Index: /trunk/ippTools/share/magictool_inputskyfile.sql =================================================================== --- /trunk/ippTools/share/magictool_inputskyfile.sql (revision 24173) +++ /trunk/ippTools/share/magictool_inputskyfile.sql (revision 24174) @@ -1,11 +1,11 @@ SELECT magicInputSkyfile.*, - diffSkyfile.uri + magicRun.inverse, + diffSkyfile.path_base FROM magicRun -JOIN magicInputSkyfile - USING(magic_id) +JOIN magicInputSkyfile USING(magic_id) JOIN diffSkyfile - ON magicInputSkyfile.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id + ON diffSkyfile.diff_id = magicRun.diff_id + AND diffSkyfile.skycell_id = magicInputSkyfile.node WHERE magicRun.state = 'new' Index: /trunk/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /trunk/ippTools/share/magictool_toprocess_inputs.sql (revision 24173) +++ /trunk/ippTools/share/magictool_toprocess_inputs.sql (revision 24174) @@ -1,66 +1,22 @@ -SELECT * FROM -(SELECT +SELECT magicTree.*, + magicRun.workdir, rawExp.exp_id, rawExp.camera, --- diffSkyfile.path_base, - magicRun.workdir, -- convert magic_id into a boolean value (1 or 0) -- note that the type stays a 64 bit int - magicNodeResult.magic_id IS TRUE as done, - magicNodeResult.fault IS TRUE as bad, - warpSkyfile.uri as template_uri -FROM magicTree -JOIN magicRun USING(magic_id) + magicNodeResult.magic_id IS TRUE AS done, + magicNodeResult.fault IS TRUE AS bad +FROM magicRun +JOIN magicTree USING(magic_id) JOIN magicInputSkyfile USING(magic_id, node) -JOIN diffSkyfile - ON magicInputSkyfile.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -JOIN diffInputSkyfile - ON diffSkyfile.diff_id = diffInputSkyfile.diff_id - AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id -JOIN warpSkyfile - ON diffInputSkyfile.warp2 = warpSkyfile.warp_id - AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id JOIN rawExp USING(exp_id) -LEFT JOIN magicNodeResult - ON magicTree.magic_id = magicNodeResult.magic_id - AND magicTree.node = magicNodeResult.node +JOIN diffSkyfile -- only get nodes that match a skycell + ON diffSkyfile.diff_id = magicRun.diff_id + AND diffSkyfile.skycell_id = magicInputSkyfile.node +LEFT JOIN magicNodeResult USING(magic_id, node) WHERE magicRun.state = 'new' AND magicNodeResult.magic_id IS NULL AND magicNodeResult.node IS NULL - -- where hook 1 %s -UNION -SELECT - magicTree.*, - rawExp.exp_id, - rawExp.camera, --- diffSkyfile.path_base, - magicRun.workdir, - -- convert magic_id into a boolean value (1 or 0) - -- note that the type stays a 64 bit int - magicNodeResult.magic_id IS TRUE as done, - magicNodeResult.fault IS TRUE as bad, - stackSumSkyfile.uri as template_uri -FROM magicTree -JOIN magicRun USING(magic_id) -JOIN magicInputSkyfile USING(magic_id, node) -JOIN diffSkyfile - ON magicInputSkyfile.diff_id = diffSkyfile.diff_id - AND magicInputSkyfile.node = diffSkyfile.skycell_id -JOIN diffInputSkyfile - ON diffSkyfile.diff_id = diffInputSkyfile.diff_id - AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id -JOIN stackSumSkyfile - ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id -JOIN rawExp USING(exp_id) -LEFT JOIN magicNodeResult - ON magicTree.magic_id = magicNodeResult.magic_id - AND magicTree.node = magicNodeResult.node -WHERE - magicRun.state = 'new' - AND magicNodeResult.magic_id IS NULL - AND magicNodeResult.node IS NULL - -- where hook 2 %s -) as foo +-- WHERE hook %s Index: /trunk/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /trunk/ippTools/share/magictool_toprocess_tree.sql (revision 24173) +++ /trunk/ippTools/share/magictool_toprocess_tree.sql (revision 24174) @@ -1,21 +1,18 @@ SELECT magicTree.*, - exp_id, - camera, magicRun.workdir, + rawExp.exp_id, + rawExp.camera, -- convert magic_id into a boolean value (1 or 0) -- note that the type stays a 64 bit int - magicNodeResult.magic_id IS TRUE as done, - magicNodeResult.fault IS TRUE as bad, - 'notused' as template_uri + magicNodeResult.magic_id IS TRUE AS done, + magicNodeResult.fault IS TRUE AS bad FROM magicTree JOIN magicRun USING(magic_id) JOIN rawExp USING(exp_id) -LEFT JOIN magicNodeResult - ON magicTree.magic_id = magicNodeResult.magic_id - AND magicTree.node = magicNodeResult.node +LEFT JOIN magicNodeResult USING(magic_id, node) WHERE magicRun.state = 'new' - -- where hook %s +-- WHERE hook %s ORDER BY magicRun.magic_id Index: /trunk/ippTools/share/magictool_totree.sql =================================================================== --- /trunk/ippTools/share/magictool_totree.sql (revision 24173) +++ /trunk/ippTools/share/magictool_totree.sql (revision 24174) @@ -6,6 +6,7 @@ ra, decl, - tess_id + diffRun.tess_id FROM magicRun +JOIN diffRun USING(diff_id) JOIN rawExp USING(exp_id) LEFT JOIN magicTree Index: /trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_create_tables.sql (revision 24173) +++ /trunk/ippTools/share/pxadmin_create_tables.sql (revision 24174) @@ -877,5 +877,6 @@ KEY(state), KEY(skycell_id), - KEY(tess_id) + KEY(tess_id), + KEY(label) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -933,5 +934,6 @@ registered DATETIME, tess_id VARCHAR(64), - exp_id BIGINT, + bothways TINYINT DEFAULT 0, + exposure TINYINT DEFAULT 0, magicked TINYINT, PRIMARY KEY(diff_id), @@ -939,4 +941,5 @@ KEY(state), KEY(tess_id), + KEY(label), FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -969,5 +972,4 @@ diff_id BIGINT, skycell_id VARCHAR(64), - uri VARCHAR(255), path_base VARCHAR(255), data_state VARCHAR(64), @@ -1006,4 +1008,5 @@ exp_id BIGINT, diff_id BIGINT, + inverse TINYINT NOT NULL DEFAULT 0, state VARCHAR(64), workdir VARCHAR(255), @@ -1046,5 +1049,5 @@ magic_id BIGINT, node VARCHAR(64), - uri VARCHAR(255), + path_base VARCHAR(255), fault SMALLINT, PRIMARY KEY(magic_id, node), Index: /trunk/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /trunk/ippTools/share/pxadmin_drop_tables.sql (revision 24173) +++ /trunk/ippTools/share/pxadmin_drop_tables.sql (revision 24174) @@ -48,5 +48,4 @@ DROP TABLE IF EXISTS magicNodeResult; DROP TABLE IF EXISTS magicMask; -DROP TABLE IF EXISTS magicSkyfileMask; DROP TABLE IF EXISTS magicDSRun; DROP TABLE IF EXISTS magicDSFile; Index: /trunk/ippTools/src/difftool.c =================================================================== --- /trunk/ippTools/src/difftool.c (revision 24173) +++ /trunk/ippTools/src/difftool.c (revision 24174) @@ -40,5 +40,5 @@ static bool revertdiffskyfileMode(pxConfig *config); static bool definepoprunMode(pxConfig *config); -static bool definebyqueryMode(pxConfig *config); +static bool definewarpstackMode(pxConfig *config); static bool definewarpwarpMode(pxConfig *config); static bool pendingcleanuprunMode(pxConfig *config); @@ -79,5 +79,5 @@ MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -116,7 +116,8 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); + PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); // default @@ -133,5 +134,6 @@ registered, tess_id, - exp_id, + bothways, + exposure, false ); @@ -487,5 +489,4 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false); @@ -521,5 +522,4 @@ diff_id, skycell_id, - uri, path_base, "full", @@ -771,5 +771,4 @@ psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none - psS64 exp_id, // exposure id for input_warp_id (if defined) pxConfig *config // Configuration ) @@ -802,5 +801,6 @@ registered, tess_id, - exp_id, + false, + false, false // magicked ); @@ -882,5 +882,4 @@ PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false); PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); - PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -902,9 +901,4 @@ psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No input has been defined (-input_stack_id or -input_warp_id)"); - return false; - } - if (input_warp_id && !exp_id) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, - "-exp_id is required with -input_warp_id."); return false; } @@ -917,5 +911,4 @@ template_warp_id ? template_warp_id : PS_MAX_S64, template_stack_id ? template_stack_id : PS_MAX_S64, - exp_id ? exp_id : PS_MAX_S64, config)) { psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun"); @@ -935,5 +928,5 @@ -static bool definebyqueryMode(pxConfig *config) +static bool definewarpstackMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -960,7 +953,8 @@ PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); // find all things to queue - psString query = pxDataGet("difftool_definebyquery_part1.sql"); + psString query = pxDataGet("difftool_definewarpstack_part1.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -1057,6 +1051,17 @@ } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + // create temporary table - query = pxDataGet("difftool_definebyquery_temp_create.sql"); + query = pxDataGet("difftool_definewarpstack_temp_create.sql"); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1070,5 +1075,5 @@ query = NULL; - psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql"); + psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql"); psArray *list = psArrayAllocEmpty(16); // List of runs, to print @@ -1183,5 +1188,6 @@ registered, tess_id, - exp_id, + true, + false, false // magicked ); @@ -1284,5 +1290,7 @@ // Haversine formula for great circle distance PXOPT_COPY_F32(config->args, selectWhere, "-distance", - "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<="); + "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + " + "COS(inputRawExp.decl)*COS(templateRawExp.decl)*" + "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<="); PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false); @@ -1308,7 +1316,76 @@ PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(selectWhere); + psFree(insertWhere); + return false; + } + + if (!rerun) { + // Need to build table of exposures with diffs + psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL + if (!tempCreate) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (!p_psDBRunQuery(config->dbh, tempCreate)) { + psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate); + psFree(tempCreate); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(tempCreate); + + psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table + if (!tempInsert) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + psString where = psStringCopy(""); // WHERE for insertion + if (label) { + psStringAppend(&where, "\nAND diffRun.label = '%s'", label); + } + + if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) { + psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert); + psFree(tempInsert); + psFree(where); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(where); + psFree(tempInsert); + } + + // Get list of warps to diff psString select = pxDataGet("difftool_definewarpwarp_select.sql"); if (!select) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(selectWhere); + psFree(insertWhere); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -1323,26 +1400,14 @@ psFree(selectWhere); - if (!rerun) { - psStringAppend(&whereClause, "\n%s (diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE"); - if (label) { - psStringAppend(&whereClause, " OR diffRun.label != '%s'", label); - } - psStringAppend(&whereClause, ")"); - } - if (!available) { psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE"); } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(select); - psFree(whereClause); - psFree(insertWhere); - return false; + if (!rerun) { + psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE"); } if (!p_psDBRunQueryF(config->dbh, select, - !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun? + !rerun ? "\n" : "", // Activate LEFT JOIN against diffs? whereClause)) { psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause); @@ -1414,9 +1479,8 @@ psMetadata *row = results->data[i]; // Result row from query - psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id"); psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id"); const char *template = psMetadataLookupStr(NULL, row, "template_warp_id"); const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); - if (!exp_id || !input_id || !template || !tess_id) { + if (!input_id || !template || !tess_id) { psError(PXTOOLS_ERR_PROG, false, "Identifiers not found"); psFree(list); @@ -1430,5 +1494,5 @@ diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, - tess_id, exp_id, false); // Run to insert + tess_id, true, true, false); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: /trunk/ippTools/src/difftool.h =================================================================== --- /trunk/ippTools/src/difftool.h (revision 24173) +++ /trunk/ippTools/src/difftool.h (revision 24174) @@ -34,5 +34,5 @@ DIFFTOOL_MODE_REVERTDIFFSKYFILE, DIFFTOOL_MODE_DEFINEPOPRUN, - DIFFTOOL_MODE_DEFINEBYQUERY, + DIFFTOOL_MODE_DEFINEWARPSTACK, DIFFTOOL_MODE_DEFINEWARPWARP, DIFFTOOL_MODE_PENDINGCLEANUPRUN, Index: /trunk/ippTools/src/difftoolConfig.c =================================================================== --- /trunk/ippTools/src/difftoolConfig.c (revision 24173) +++ /trunk/ippTools/src/difftoolConfig.c (revision 24174) @@ -49,4 +49,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); @@ -90,5 +92,4 @@ psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); - psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0, "define URI of file", 0); psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); @@ -142,5 +143,4 @@ psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID for template", 0); psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0, "define warp ID for template", 0); psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0, "define stack ID for template", 0); @@ -148,22 +148,23 @@ psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0, "define stack ID for input", 0); - // -definebyquery - psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); - psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); - psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -definewarpstack + psMetadata *definewarpstackArgs = psMetadataAlloc(); + psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); + psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); + psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false); // -definewarpwarp @@ -239,5 +240,5 @@ PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); PXOPT_ADD_MODE("-definepoprun", "", DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunArgs); - PXOPT_ADD_MODE("-definebyquery", "", DIFFTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definewarpstack", "", DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackArgs); PXOPT_ADD_MODE("-definewarpwarp", "", DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpArgs); PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); Index: /trunk/ippTools/src/magictool.c =================================================================== --- /trunk/ippTools/src/magictool.c (revision 24173) +++ /trunk/ippTools/src/magictool.c (revision 24174) @@ -127,9 +127,12 @@ PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "=="); - + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns + PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "=="); + PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "=="); + + psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else + PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "=="); // Get list of exposures ready to magic @@ -146,36 +149,46 @@ PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); - psString queryWhere = NULL; // WHERE conditions for entire query + psString diffWhereStr = NULL; // WHERE conditions for diffRuns + psString queryWhereStr = NULL; // WHERE conditions for entire query if (!available) { - psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'"); - } else { - // what if no skycells for the diff run completed? - } + psStringAppend(&diffWhereStr, "\nAND diffRun.state = 'full'"); + } + // what if no skycells for the diff run completed? + if (!rerun) { - const char *newWhere = " magic_id IS NULL"; // String to add - if (queryWhere) { - psStringAppend(&queryWhere, " AND %s", newWhere); - } else { - psStringAppend(&queryWhere, "\nWHERE %s", newWhere); - } - } + psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE"); + } + // now add the user specified qualifiers - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause); + if (psListLength(diffWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(diffWhere, NULL); + psStringAppend(&diffWhereStr, "\nAND %s", whereClause); psFree(whereClause); } - psFree(where); - if (!queryWhere) { - psStringAppend(&queryWhere, " "); - } - - if (!p_psDBRunQueryF(config->dbh, query, queryWhere)) { + if (psListLength(queryWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(queryWhere, NULL); + psStringAppend(&queryWhereStr, "\n%s %s", queryWhereStr ? "AND" : "WHERE", whereClause); + psFree(whereClause); + } + psFree(diffWhere); + psFree(queryWhere); + + // Ensure the WHERE strings have something + if (!diffWhereStr) { + diffWhereStr = psStringCopy(""); + } + if (!queryWhereStr) { + queryWhereStr = psStringCopy(""); + } + + if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(queryWhere); + psFree(diffWhereStr); + psFree(queryWhereStr); psFree(query); return false; } - psFree(queryWhere); + psFree(diffWhereStr); + psFree(queryWhereStr); psFree(query); } @@ -201,6 +214,18 @@ } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // Parse the list of exposures ready to magic - if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -215,7 +240,9 @@ psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier + bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types! // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0); + magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label, + dvodb, registered, 0); if (!run) { psAbort("failed to alloc magicRun object"); @@ -296,4 +323,5 @@ // optional + PXOPT_LOOKUP_BOOL(inverse, config->args, "-inverse", false); PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); @@ -305,4 +333,5 @@ exp_id, diff_id ? diff_id : PS_MAX_S64, + inverse, "reg", // state workdir, @@ -363,5 +392,4 @@ // required PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); @@ -369,5 +397,4 @@ config->dbh, magic_id, - diff_id, node ); @@ -563,5 +590,5 @@ PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); - psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0"); + psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0"); if (psListLength(where->list)) { @@ -584,7 +611,13 @@ PS_ASSERT_PTR_NON_NULL(config, false); - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-node", "node", "=="); + // Regrettably, there are multiple WHERE hooks which call the same things different names + psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template + psMetadata *magicWhere = psMetadataAlloc(); // WHERE for selecting magic runs + + PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "=="); + + PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "=="); + PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -597,10 +630,20 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template + psString magicWhereStr = psStringCopy(""); // WHERE for selecting magic runs + + if (psListLength(templatesWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL); + psStringAppend(&templatesWhereStr, "\nAND %s", whereClause); psFree(whereClause); } - psFree(where); + psFree(templatesWhere); + + if (psListLength(magicWhere->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL); + psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause); + psFree(whereClause); + } + psFree(magicWhere); // treat limit == 0 as "no limit" @@ -611,9 +654,15 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, + templatesWhereStr, templatesWhereStr, + magicWhereStr, magicWhereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(templatesWhereStr); + psFree(magicWhereStr); psFree(query); return false; } + psFree(templatesWhereStr); + psFree(magicWhereStr); psFree(query); @@ -704,9 +753,9 @@ bool status = false; - psS32 done = psMetadataLookupS32(&status, data, "done"); + psS64 done = psMetadataLookupS64(&status, data, "done"); if (!status) { psAbort("failed to lookup value for done column"); } - psS16 bad = psMetadataLookupS16(&status, data, "bad"); + psS64 bad = psMetadataLookupS64(&status, data, "bad"); if (!status) { psAbort("failed to lookup value for bad column"); @@ -738,5 +787,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "=="); @@ -794,5 +843,5 @@ if (limit) { if (psArrayLength(output) >= limit) { - // we've found enough (not that limit was applied to the query so '> limit' won't happen) + // we've found enough (note that limit was applied to the query so '> limit' won't happen) // negative simple so the default is true if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { @@ -936,5 +985,5 @@ // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); // default values @@ -944,5 +993,5 @@ magic_id, node, - uri, + path_base, fault )) { Index: /trunk/ippTools/src/magictoolConfig.c =================================================================== --- /trunk/ippTools/src/magictoolConfig.c (revision 24173) +++ /trunk/ippTools/src/magictoolConfig.c (revision 24174) @@ -53,4 +53,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL); @@ -58,4 +59,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false); // -definerun @@ -65,4 +67,5 @@ // following argument is for compatability psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff_id", 0); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-inverse", 0, "use the inverse subtraction?", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); @@ -78,5 +81,4 @@ psMetadata *addinputskyfileArgs = psMetadataAlloc(); psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); - psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0); psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); @@ -125,5 +127,5 @@ psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); - psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base", NULL); psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);