Index: trunk/dbconfig/background.md =================================================================== --- trunk/dbconfig/background.md (revision 28486) +++ trunk/dbconfig/background.md (revision 28486) @@ -0,0 +1,72 @@ +# Tables to support background replacement + +# Background replacement on a chipRun +chipBackgroundRun METADATA + chip_bg_id S64 0 + chip_id S64 0 + state STR 64 + workdir STR 255 + label STR 64 + data_group STR 64 + dist_group STR 64 + reduction STR 64 + note STR 255 + registered TAI NULL + magicked S64 0 +END + +# Results of background replacement from chipBackgroundRun +chipBackgroundImfile METADATA + chip_bg_id S64 0 + class_id STR 64 + path_base STR 255 + magicked S64 0 + dtime_script F32 0.0 + hostname STR 64 + quality S16 0 + fault S16 0 + software_ver STR 16 + bg F64 0.0 + bg_stdev F64 0.0 + maskfrac_npix F32 0.0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 +END + +# Background replacement on a warpRun (utilising chipBackgroundRun) +warpBackgroundRun METADATA + warp_bg_id S64 0 + warp_id S64 0 + chip_bg_id S64 0 + state STR 64 + workdir STR 255 + label STR 64 + data_group STR 64 + dist_group STR 64 + reduction STR 64 + note STR 255 + registered TAI NULL + magicked S64 0 +END + +# Results of background replacement from warpBackgroundRun +warpBackgroundSkyfile METADATA + warp_bg_id S64 0 + skycell_id STR 64 + path_base STR 255 + magicked S64 0 + dtime_script F32 0.0 + hostname STR 64 + quality S16 0 + fault S16 0 + software_ver STR 16 + bg F64 0.0 + bg_stdev F64 0.0 + maskfrac_npix F32 0.0 + maskfrac_static F32 0.0 + maskfrac_dynamic F32 0.0 + maskfrac_magic F32 0.0 + maskfrac_advisory F32 0.0 +END Index: trunk/dbconfig/changes.txt =================================================================== --- trunk/dbconfig/changes.txt (revision 28453) +++ trunk/dbconfig/changes.txt (revision 28486) @@ -1718,4 +1718,100 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Tables to support background restoration +-- Background replacement on a chipRun +CREATE TABLE chipBackgroundRun ( + chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + chip_id BIGINT NOT NULL, -- link to chipRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(chip_bg_id), + KEY(chip_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from chipBackgroundRun +CREATE TABLE chipBackgroundImfile ( + chip_bg_id BIGINT NOT NULL, -- unique identifier + class_id VARCHAR(64) NOT NULL, -- class (component) identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(chip_bg_id,class_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Background replacement on a warpRun (utilising chipBackgroundRun) +CREATE TABLE warpBackgroundRun ( + warp_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + warp_id BIGINT NOT NULL, -- link to warpRun + chip_bg_id BIGINT NOT NULL, -- link to chipBackgroundRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + PRIMARY KEY(warp_bg_id), + KEY(warp_id), + KEY(chip_bg_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from warpBackgroundRun +CREATE TABLE warpBackgroundSkyfile ( + warp_bg_id BIGINT NOT NULL, -- unique identifier + skycell_id VARCHAR(64) NOT NULL, -- skycell identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(warp_bg_id,skycell_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + -- Version 1.1.68 addition of minidvodb tables and changes to addTables Index: trunk/dbconfig/ipp.m4 =================================================================== --- trunk/dbconfig/ipp.m4 (revision 28453) +++ trunk/dbconfig/ipp.m4 (revision 28486) @@ -33,3 +33,4 @@ include(receive.md) include(publish.md) +include(background.md) include(diffphot.md) Index: trunk/ippScripts/Build.PL =================================================================== --- trunk/ippScripts/Build.PL (revision 28453) +++ trunk/ippScripts/Build.PL (revision 28486) @@ -109,4 +109,6 @@ scripts/bundle_detrends.pl scripts/ipp_cluster_load_monitor.pl + scripts/background_chip.pl + scripts/background_warp.pl scripts/skycell_jpeg.pl scripts/diffphot.pl Index: trunk/ippScripts/scripts/background_chip.pl =================================================================== --- trunk/ippScripts/scripts/background_chip.pl (revision 28486) +++ trunk/ippScripts/scripts/background_chip.pl (revision 28486) @@ -0,0 +1,259 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $ppBackground = can_run('ppBackground') or (warn "Can't find ppBackground" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my @ARGS = @ARGV; + +# Parse the command-line arguments +my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose, + $threads, $no_update, $save_temps, $no_op, $redirect, $deburned ); +GetOptions( + 'chip_bg_id=s' => \$chip_bg_id, # chipBackgroundRun identifier + 'class_id=s' => \$class_id, # Class identifier + 'camera|c=s' => \$camera, # Camera + 'outroot|w=s' => \$outroot, # output file base name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'threads=s' => \$threads, # Number of threads to use + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --chip_bg_id --class_id --camera --outroot", + -exitval => 3) unless + defined $chip_bg_id and + defined $class_id and + defined $camera and + defined $outroot; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $chip_bg_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $chip_bg_id, $class_id, $PS_EXIT_CONFIG_ERROR); + +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR ); + print STDOUT "\n\n"; + print STDOUT "Starting script $0 on $host\n\n"; + print STDOUT "FULL COMMAND: $0 @ARGS\n\n"; +} + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_ppBackground = $ipprc->reduction($reduction, 'BACKGROUND_PPBACKGROUND'); # ppBackground recipe +unless ($recipe_ppBackground) { + &my_die("Couldn't find selected reduction for BACKGROUND_PPBACKGROUND: $reduction\n", $chip_bg_id, $class_id, $PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Get inputs +my $in_path; # Input path +my $magicked; # Input is magicked? +{ + my $command = "bgtool -chipinputs -chip_bg_id $chip_bg_id -class_id $class_id"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to get inputs: $error_code\n"); + exit($error_code); + } + + my $inputs = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $chip_bg_id, $class_id, $PS_EXIT_PROG_ERROR); + &my_die("Input list does not contain exactly one entry", $chip_bg_id, $class_id, $PS_EXIT_PROG_ERROR) unless scalar @$inputs == 1; + my $input = $$inputs[0]; # Input of interest + $in_path = $input->{path_base}; + $magicked = $input->{magicked}; +} + +my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id); +my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); +my $in_pattern = $ipprc->filename("PPIMAGE.PATTERN", $in_path, $class_id); +my $in_config = $ipprc->filename("PPIMAGE.CONFIG", $in_path, $class_id); + +# Determine what to apply +my ($apply_bg, $apply_pattern); # Apply the background and pattern? +{ + &my_die("Cannot find input file: $in_config", $chip_bg_id, $class_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($in_config); + + my $command = "$ppConfigDump -ipprc $in_config -dump-recipe PPIMAGE -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR); + } + my $recipe = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR); + + $apply_bg = metadataLookupBool($recipe, "BACKGROUND"); + my $row = metadataLookupBool($recipe, "PATTERN.ROW"); + my $cell = metadataLookupBool($recipe, "PATTERN.CELL"); + $apply_pattern = ($row or $cell); +} + +# Set up files +&my_die("Couldn't find input file: $in_image\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_image); +&my_die("Couldn't find input file: $in_mask\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_mask); +&my_die("Couldn't find input file: $in_bg\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_bg) or !$apply_bg; +&my_die("Couldn't find input file: $in_pattern\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_pattern) or !$apply_pattern; + +$ipprc->outroot_prepare($outroot); +my $out_image = $ipprc->filename("PPBACKGROUND.OUTPUT", $outroot, $class_id); +my $out_mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $outroot, $class_id); +my $out_stats = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id); +my $out_config = $ipprc->filename("PPIMAGE.CONFIG", $outroot, $class_id); +my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id); + +# Run ppImage +unless ($no_op) { + my $command = "$ppBackground $outroot"; + $command .= " -image $in_image"; + $command .= " -mask $in_mask"; + $command .= " -stats $out_stats"; + $command .= " -background $in_bg" if $apply_bg; + $command .= " -pattern $in_pattern" if $apply_pattern; + $command .= " -recipe PPBACKGROUND $recipe_ppBackground"; + $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -dumpconfig $out_config"; + $command .= " -tracedest $traceDest -log $logDest"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppBackground: $error_code", $chip_bg_id, $class_id, $error_code); + } +} + +# Gather command-line arguments from statistics +my $cmdflags; # Command-line flags to add +my $quality; # Quality flag +{ + &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); + + my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $chip_bg_id, $class_id, $error_code); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag +} + +if (!$quality and !$no_op) { + &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); + &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); + &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config); + +} + +# Update database +{ + my $command = "$bgtool -addchip"; + $command .= " -chip_bg_id $chip_bg_id"; + $command .= " -class_id $class_id"; + $command .= " -path_base $outroot"; + $command .= " -set_magicked $magicked" if $magicked; + $command .= " -hostname $host" if defined $host; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -dbname $dbname" if defined $dbname; + + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $command\n"; + } +} + + +### Pau. + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $chip_bg_id = shift; # Chiptool identifier + my $class_id = shift; # Class identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + if (defined $chip_bg_id and defined $class_id and not $no_update) { + my $command = "$bgtool -addchip"; + $command .= " -chip_bg_id $chip_bg_id"; + $command .= " -class_id $class_id"; + $command .= " -fault $exit_code"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -dbname $dbname" if defined $dbname; + system($command); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippScripts/scripts/background_warp.pl =================================================================== --- trunk/ippScripts/scripts/background_warp.pl (revision 28486) +++ trunk/ippScripts/scripts/background_warp.pl (revision 28486) @@ -0,0 +1,251 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +# Look for programs we need +my $missing_tools; +my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1); +my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($warp_bg_id, $skycell_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'warp_bg_id|i=s' => \$warp_bg_id, # Warp identifier + 'skycell_id|s=s' => \$skycell_id, # Skycell identifier + 'tess_dir|s=s' => \$tess_dir, # Tesselation identifier + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class + 'outroot=s' => \$outroot, # Output root name + 'threads=s' => \$threads, # Number of threads to use for pswarp + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --warp_bg_id --skycell_id --tess_dir --camera --outroot", + -exitval => 3, +) unless defined $warp_bg_id + and defined $skycell_id + and defined $tess_dir + and defined $camera + and defined $outroot; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR ); + +$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_pswarp = $ipprc->reduction($reduction, 'BACKGROUND_PSWARP'); # Recipe to use +my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use +unless ($recipe_pswarp and $recipe_psastro) { + &my_die("Couldn't find selected reduction: $reduction\n", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR); +} + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Where do we get the astrometry source from? +my $astromSource; # The astrometry source +{ + my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", $warp_bg_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); +} + +# Get list of filenames +my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id"; +my ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); +my ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); +my $astrometry; # Astrometry filename +my $magicked; # Magicked status +{ + my $command = "$bgtool -warpinputs"; + $command .= " -warp_bg_id $warp_bg_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to get input list: $error_code", $warp_bg_id, $skycell_id, $error_code); + } + + my $files = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR); + foreach my $file (@$files) { + my $chip_path = $file->{chip_path_base}; + my $class_id = $file->{class_id}; + my $image = $ipprc->filename("PPBACKGROUND.OUTPUT", $chip_path, $class_id); + my $mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $chip_path, $class_id ); + print $imageFile "$image\n"; + print $maskFile "$mask\n"; + + &my_die("Magic status don't match: $magicked vs $file->{magicked}", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $magicked and $magicked != $file->{magicked}; + $magicked = $file->{magicked}; + + my $cam_path = $file->{cam_path_base}; + my $astrom = $ipprc->filename($astromSource, $cam_path); + &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; + $astrometry = $astrom; + } +} +close $imageFile; +close $maskFile; + + + +my $out_image = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id ); +my $out_mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id); +my $out_stats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id ); +my $out_config = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id); +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id); + +my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id ); +$ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR); + +# Run pswarp +unless ($no_op) { + my $command = "$pswarp"; + $command .= " -list $imageName"; + $command .= " -masklist $maskName"; + $command .= " -astrom $astrometry"; + $command .= " $outroot $skyFile"; + $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; + $command .= " -recipe PSWARP $recipe_pswarp"; + $command .= " -recipe PPSTATS WARPSTATS"; + $command .= " -stats $out_stats"; + $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -threads $threads" if defined $threads; + $command .= " -dbname $dbname" if defined $dbname; + $command .= " -dumpconfig $out_config"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $warp_bg_id, $skycell_id, $error_code); + } +} + +# Read the statistics +my $cmdflags; +{ + &my_die("Couldn't find expected output file: $out_stats", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); + + my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_WARP"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStatsFromMetadata: $error_code", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } + foreach my $line (@$stdout_buf) { + $cmdflags .= " $line"; + } + chomp $cmdflags; + + my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + + if (!$quality) { + &my_die("Couldn't find expected output file: $out_image", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); + &my_die("Couldn't find expected output file: $out_mask", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); + &my_die("Couldn't find expected output file: $out_config", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config); + } +} + +unless ($no_update) { + my $command = "$bgtool -addwarp"; + $command .= " -warp_bg_id $warp_bg_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -path_base $outroot"; # needed for logfile lookups + $command .= " -set_magicked $magicked" if $magicked; + $command .= " -hostname $host"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " $cmdflags"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to update database: $error_code\n"); + exit($error_code); + } +} + +### Pau. + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $warp_bg_id = shift; # warpBackgroundRun identifier + my $skycell_id = shift; # Skycell identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $warp_bg_id and defined $skycell_id and not $no_update) { + my $command = "$bgtool -addwarp"; + $command .= " -warp_bg_id $warp_bg_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -path_base $outroot"; + $command .= " -hostname $host" if defined $host; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: trunk/ippTasks/Makefile.am =================================================================== --- trunk/ippTasks/Makefile.am (revision 28453) +++ trunk/ippTasks/Makefile.am (revision 28486) @@ -35,4 +35,5 @@ nightly_stacks.pro \ lossy_compress.pro \ + background.pro \ diffphot.pro Index: trunk/ippTasks/background.pro =================================================================== --- trunk/ippTasks/background.pro (revision 28486) +++ trunk/ippTasks/background.pro (revision 28486) @@ -0,0 +1,536 @@ +## background.pro : globals and support macros : -*- sh -*- + +check.globals + +book init chipBackgroundRuns +book init warpBackgroundRuns + +macro bg.status + book listbook chipBackgroundRuns + book listbook warpBackgroundRuns +end + +macro bg.reset + book init chipBackgroundRuns + book init warpBackgroundRuns +end + +macro bg.on + task bg.chip.load + active true + end + task bg.chip.run + active true + end + task bg.chip.advance + active true + end + task bg.chip.revert + active true + end + task bg.warp.load + active true + end + task bg.warp.run + active true + end + task bg.warp.advance + active true + end + task bg.warp.revert + active true + end +end + +macro bg.off + task bg.chip.load + active false + end + task bg.chip.run + active false + end + task bg.chip.advance + active false + end + task bg.chip.revert + active false + end + task bg.warp.load + active false + end + task bg.warp.run + active false + end + task bg.warp.advance + active false + end + task bg.warp.revert + active false + end +end + +macro bg.revert.on + task bg.chip.revert + active true + end + task bg.warp.revert + active true + end +end + +macro bg.revert.off + task bg.chip.revert + active false + end + task bg.warp.revert + active false + end +end + + +# These variables cycle through the known database names +$bg_chip_load_DB = 0 +$bg_chip_advance_DB = 0 +$bg_chip_revert_DB = 0 +$bg_warp_load_DB = 0 +$bg_warp_advance_DB = 0 +$bg_warp_revert_DB = 0 + +task bg.chip.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.load.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -tochip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_load_DB + $run = $run -dbname $DB:$bg_chip_load_DB + $bg_chip_load_DB ++ + if ($bg_chip_load_DB >= $DB:n) set bg_chip_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout chipBackgroundRuns -key chip_bg_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook chipBackgroundRuns + end + process_cleanup chipBackgroundRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.chip.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec $RUNEXEC + + book npages chipBackgroundRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage chipBackgroundRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword chipBackgroundRuns $pageName pantaskState RUN + book getword chipBackgroundRuns $pageName camera -var CAMERA + book getword chipBackgroundRuns $pageName chip_bg_id -var CHIP_BG_ID + book getword chipBackgroundRuns $pageName class_id -var CLASS_ID + book getword chipBackgroundRuns $pageName exp_tag -var EXP_TAG + book getword chipBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword chipBackgroundRuns $pageName dbname -var DBNAME + book getword chipBackgroundRuns $pageName reduction -var REDUCTION + + set.host.for.camera $CAMERA $CLASS_ID + set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s/%s.bgc.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID + stdout $LOGDIR/bg.chip.run.log + stderr $LOGDIR/bg.chip.run.log + + $run = background_chip.pl --threads @MAX_THREADS@ --chip_bg_id $CHIP_BG_ID --class_id $CLASS_ID --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit chipBackgroundRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + exec bgtool -addchip -dbname $DBNAME -chip_bg_id $CHIP_BG_ID -class_id $CLASS_ID -fault $EXIT_CRASH_ERR + process_exit chipBackgroundRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword chipBackgroundRuns $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task bg.chip.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -advancechip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$chip_advance_DB + $run = $run -dbname $DB:$bg_chip_advance_DB + $bg_chip_advance_DB ++ + if ($bg_chip_advance_DB >= $DB:n) set bg_chip_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.chip.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.chip.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -revertchip + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_chip_revert_DB + $run = $run -dbname $DB:$bg_chip_revert_DB + $bg_chip_revert_DB ++ + if ($bg_chip_revert_DB >= $DB:n) set bg_chip_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.load.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -towarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_load_DB + $run = $run -dbname $DB:$bg_warp_load_DB + $bg_warp_load_DB ++ + if ($bg_warp_load_DB >= $DB:n) set bg_warp_load_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout warpBackgroundRuns -key warp_bg_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook warpBackgroundRuns + end + process_cleanup warpBackgroundRuns + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60 + + task.exec + periods -exec $RUNEXEC + + book npages warpBackgroundRuns -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + book getpage warpBackgroundRuns 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword warpBackgroundRuns $pageName pantaskState RUN + book getword warpBackgroundRuns $pageName camera -var CAMERA + book getword warpBackgroundRuns $pageName warp_bg_id -var WARP_BG_ID + book getword warpBackgroundRuns $pageName skycell_id -var SKYCELL_ID + book getword warpBackgroundRuns $pageName tess_id -var TESS_DIR + book getword warpBackgroundRuns $pageName exp_tag -var EXP_TAG + book getword warpBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE + book getword warpBackgroundRuns $pageName dbname -var DBNAME + book getword warpBackgroundRuns $pageName reduction -var REDUCTION + + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s/%s.bgw.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID $SKYCELL_ID + stdout $LOGDIR/bg.warp.run.log + stderr $LOGDIR/bg.warp.run.log + + $run = background_warp.pl --threads @MAX_THREADS@ --warp_bg_id $WARP_BG_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit warpBackgroundRuns $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + exec bgtool -addwarp -dbname $DBNAME -warp_bg_id $WARP_BG_ID -skycell_id $SKYCELL_ID -fault $EXIT_CRASH_ERR + process_exit warpBackgroundRuns $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword warpBackgroundRuns $options:0 pantaskState TIMEOUT + end +end + +# advance exposures for which all imfiles have completed processing +# sets the exposure state to full and queues warp processing if requested +task bg.warp.advance + host local + + periods -poll $LOADPOLL +# periods -exec $LOADEXEC + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.advance.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -advancewarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$warp_advance_DB + $run = $run -dbname $DB:$bg_warp_advance_DB + $bg_warp_advance_DB ++ + if ($bg_warp_advance_DB >= $DB:n) set bg_warp_advance_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task bg.warp.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + + stdout NULL + stderr $LOGDIR/bg.warp.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = bgtool -revertwarp + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$bg_warp_revert_DB + $run = $run -dbname $DB:$bg_warp_revert_DB + $bg_warp_revert_DB ++ + if ($bg_warp_revert_DB >= $DB:n) set bg_warp_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: trunk/ippTools/configure.ac =================================================================== --- trunk/ippTools/configure.ac (revision 28453) +++ trunk/ippTools/configure.ac (revision 28486) @@ -19,6 +19,7 @@ PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63]) +PKG_CHECK_MODULES([PPSTAMP], [ippdb >= 1.1.63]) -PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" +PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${PPSTAMP_CFLAGS=} ${IPPDB_CFLAGS=}" PXTOOLS_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=} ${IPPDB_LIBS=}" AC_SUBST(PXTOOLS_CFLAGS,[$PXTOOLS_CFLAGS]) Index: trunk/ippTools/share/Makefile.am =================================================================== --- trunk/ippTools/share/Makefile.am (revision 28453) +++ trunk/ippTools/share/Makefile.am (revision 28486) @@ -4,334 +4,349 @@ dist_pkgdata_DATA = \ - addtool_checkminidvodbaddrun.sql \ - addtool_donecleanup.sql \ - addtool_find_cam_id.sql \ - addtool_find_pendingexp.sql \ - addtool_find_pendingmergeprocess.sql \ - addtool_find_processedexp.sql \ - addtool_find_minidvodbprocessed.sql \ - addtool_find_minidvodbrun.sql \ - addtool_pendingcleanupexp.sql \ - addtool_pendingcleanuprun.sql \ - addtool_queue_cam_id.sql \ - addtool_queue_minidvodbrun.sql \ - addtool_revertminidvodbprocessed.sql \ - addtool_revertprocessedexp.sql \ - camtool_addprocessedexp.sql \ - camtool_donecleanup.sql \ - camtool_find_chip_id.sql \ - camtool_pendingexp.sql \ - camtool_find_pendingimfile.sql \ - camtool_find_processedexp.sql \ - camtool_pendingcleanupexp.sql \ - camtool_pendingcleanuprun.sql \ - camtool_queue_chip_id.sql \ - camtool_revertprocessedexp.sql \ - camtool_export_run.sql \ - camtool_export_processed_exp.sql \ - chiptool_change_exp_state.sql \ - chiptool_change_imfile_data_state.sql \ - chiptool_completely_processed_exp.sql \ - chiptool_coalesce_run.sql \ - chiptool_definecopy.sql \ - chiptool_donecleanup.sql \ - chiptool_find_rawexp.sql \ - chiptool_listrun.sql \ - chiptool_pendingcleanupimfile.sql \ - chiptool_pendingcleanuprun.sql \ - chiptool_pendingimfile.sql \ - chiptool_processedimfile.sql \ - chiptool_revertprocessedimfile.sql \ - chiptool_revertupdatedimfile.sql \ - chiptool_run.sql \ - chiptool_runstate.sql \ - chiptool_setimfiletoupdate.sql \ - chiptool_export_run.sql \ - chiptool_export_imfile.sql \ - chiptool_export_processed_imfile.sql \ - chiptool_unmasked.sql \ - detselect_search.sql \ - detselect_select.sql \ - dettool_addprocessedexp.sql \ - dettool_childlessrun.sql \ - dettool_definebydetrun.sql \ - dettool_detrunsummary.sql \ - dettool_find_completed_runs.sql \ - dettool_input.sql \ - dettool_normalizedexp.sql \ - dettool_normalizedimfile.sql \ - dettool_normalizedstat.sql \ - dettool_pending.sql \ - dettool_pendingcleanup_detrunsummary.sql \ - dettool_pendingcleanup_normalizedexp.sql \ - dettool_pendingcleanup_normalizedimfile.sql \ - dettool_pendingcleanup_normalizedstat.sql \ - dettool_pendingcleanup_processedexp.sql \ - dettool_pendingcleanup_processedimfile.sql \ - dettool_pendingcleanup_residexp.sql \ - dettool_pendingcleanup_residimfile.sql \ - dettool_pendingcleanup_stacked.sql \ - dettool_processedimfile.sql \ - dettool_raw.sql \ - dettool_residexp.sql \ - dettool_residimfile.sql \ - dettool_revertdetrunsummary.sql \ - dettool_revertnormalizedexp.sql \ - dettool_revertnormalizedimfile.sql \ - dettool_revertnormalizedstat.sql \ - dettool_revertprocessedexp.sql \ - dettool_revertprocessedimfile.sql \ - dettool_revertresidexp.sql \ - dettool_revertresidimfile.sql \ - dettool_revertstacked.sql \ - dettool_runs.sql \ - dettool_stacked.sql \ - dettool_start_new_iteration.sql \ - dettool_stop_completed_correct_runs.sql \ - dettool_tocorrectexp.sql \ - dettool_tocorrectimfile.sql \ - dettool_todetrunsummary.sql \ - dettool_tonormalize.sql \ - dettool_tonormalizedexp.sql \ - dettool_tonormalizedstat.sql \ - dettool_toprocessedexp.sql \ - dettool_toprocessedimfile.sql \ - dettool_toresidexp.sql \ - dettool_toresidimfile.sql \ - dettool_tostacked.sql \ - difftool_change_skyfile_data_state.sql \ - difftool_change_run_state.sql \ - difftool_completed_runs.sql \ - difftool_coalesce_run.sql \ - difftool_definewarpstack_part1.sql \ - difftool_definewarpstack_part2.sql \ - difftool_definewarpstack_temp_create.sql \ - difftool_definewarpwarp_temp_create.sql \ - difftool_definewarpwarp_temp_insert.sql \ - difftool_definewarpwarp_select.sql \ - difftool_definewarpwarp_insert.sql \ - difftool_definestackstack_part0.sql \ - difftool_definestackstack_part1.sql \ - difftool_donecleanup.sql \ - difftool_export_input_skyfile.sql \ - difftool_export_run.sql \ - difftool_export_skyfile.sql \ - difftool_inputskyfile.sql \ - difftool_listrun.sql \ - difftool_pendingcleanuprun.sql \ - difftool_pendingcleanupskyfile.sql \ - difftool_revertdiffskyfile_delete.sql \ - difftool_revertdiffskyfile_updated.sql \ - difftool_setskyfiletoupdate.sql \ - difftool_skyfile.sql \ - difftool_todiffskyfile.sql \ - difftool_tosummary.sql \ - difftool_addsummary.sql \ - disttool_definebyquery_camera.sql \ - disttool_definebyquery_chip.sql \ - disttool_definebyquery_diff.sql \ - disttool_definebyquery_fake.sql \ - disttool_definebyquery_raw.sql \ - disttool_definebyquery_stack.sql \ - disttool_definebyquery_warp.sql \ - disttool_definebyquery_SSdiff.sql \ - disttool_defineinterest.sql \ - disttool_listfilesets.sql \ - disttool_listinterests.sql \ - disttool_pending_camera.sql \ - disttool_pending_chip.sql \ - disttool_pending_diff.sql \ - disttool_pending_fake.sql \ - disttool_pending_raw.sql \ - disttool_pending_stack.sql \ - disttool_pending_warp.sql \ - disttool_pending_SSdiff.sql \ - disttool_pendingfileset.sql \ - disttool_pendingdest.sql \ - disttool_pendingcleanup.sql \ - disttool_processedcomponent.sql \ - disttool_queuercrun.sql \ - disttool_revertrcrun.sql \ - disttool_revertrun.sql \ - disttool_revertcomponent.sql \ - disttool_revertfileset.sql \ - disttool_toadvance.sql \ - disttool_updateinterest.sql \ - disttool_updatercrun.sql \ - dqstatstool_createbundle.sql \ - dqstatstool_definebyquery.sql \ - dqstatstool_get_contents.sql \ - dqstatstool_get_run.sql \ - faketool_change_exp_state.sql \ - faketool_change_imfile_data_state.sql \ - faketool_completely_processed_exp.sql \ - faketool_donecleanup.sql \ - faketool_export_processed_imfile.sql \ - faketool_export_run.sql \ - faketool_find_camrun.sql \ - faketool_find_pendingexp.sql \ - faketool_pendingcleanupimfile.sql \ - faketool_pendingcleanuprun.sql \ - faketool_pendingimfile.sql \ - faketool_processedimfile.sql \ - faketool_queue_cam_id.sql \ - faketool_revertprocessedimfile.sql \ - faketool_unmasked.sql \ - flatcorr_chiprundone.sql \ - flatcorr_camerarundone.sql \ - flatcorr_pendingprocess.sql \ - flatcorr_inputimfile.sql \ - flatcorr_dropchip.sql \ - flatcorr_dropcamera.sql \ - magictool_addmask.sql \ - magictool_restore_camera.sql \ - magictool_restore_chip.sql \ - magictool_restore_diff.sql \ - magictool_restore_raw.sql \ - magictool_restore_warp.sql \ - magictool_create_tmp_warpcomplete.sql \ - magictool_definebyquery_insert.sql \ - magictool_definebyquery_select.sql \ - magictool_inputs.sql \ - magictool_inputskyfile.sql \ - magictool_mask.sql \ - magictool_tomask.sql \ - magictool_toprocess_inputs.sql \ - magictool_toprocess_runs.sql \ - magictool_toprocess_tree.sql \ - magictool_toskyfilemask.sql \ - magictool_totree.sql \ - magictool_diffskyfile.sql \ - magictool_warpskyfile.sql \ - magictool_chipprocessedimfile.sql \ - magictool_rawimfile.sql \ - magictool_revertnode.sql \ - magictool_exposure.sql \ - magicdstool_clearstatefaults.sql \ - magicdstool_completed_runs.sql \ - magicdstool_completedrevert.sql \ - magicdstool_definebyquery_raw.sql \ - magicdstool_definebyquery_chip.sql \ - magicdstool_definebyquery_camera.sql \ - magicdstool_definebyquery_warp.sql \ - magicdstool_definebyquery_diff.sql \ - magicdstool_definecopy_chip.sql \ - magicdstool_definecopy_warp.sql \ - magicdstool_getrunids.sql \ - magicdstool_getskycells.sql \ - magicdstool_revertdestreakedfile.sql \ - magicdstool_tocleanup.sql \ - magicdstool_todestreak_camera.sql \ - magicdstool_todestreak_chip.sql \ - magicdstool_todestreak_diff.sql \ - magicdstool_todestreak_raw.sql \ - magicdstool_todestreak_warp.sql \ - magicdstool_toremove.sql \ - magicdstool_torevert_raw.sql \ - magicdstool_torevert_chip.sql \ - magicdstool_torevert_camera.sql \ - magicdstool_torevert_warp.sql \ - magicdstool_torevert_diff.sql \ - pstamptool_completedreq.sql \ - pstamptool_datastore.sql \ - pstamptool_getdependent.sql \ - pstamptool_listjob.sql \ - pstamptool_pendingcleanup.sql \ - pstamptool_pendingdependent.sql \ - pstamptool_pendingjob.sql \ - pstamptool_pendingreq.sql \ - pstamptool_project.sql \ - pstamptool_revertdependent.sql \ - pstamptool_revertjob.sql \ - pstamptool_revertreq.sql \ - pstamptool_revertreq_deletejobs.sql \ - pstamptool_updatejob.sql \ - pxadmin_create_tables.sql \ - pxadmin_create_mirror_tables.sql \ - pxadmin_drop_tables.sql \ - pxadmin_update_version.sql \ - pubtool_definerun.sql \ - pubtool_pending.sql \ - pubtool_revert.sql \ - pztool_find_completed_exp.sql \ - pztool_pendingimfile.sql \ - pztool_revert_downloadimfile_faults.sql \ - pztool_revert_fileset_faults.sql \ - pztool_revertcopied.sql \ - receivetool_list.sql \ - receivetool_addfileset.sql \ - receivetool_pendingfileset.sql \ - receivetool_pendingfile.sql \ - receivetool_revert.sql \ - receivetool_toadvance.sql \ - regtool_create_dup_table.sql \ - regtool_export_exp.sql \ - regtool_export_imfile.sql \ - regtool_pendingexp.sql \ - regtool_pendingimfile.sql \ - regtool_populate_dup_table.sql \ - regtool_processedexp.sql \ - regtool_processedimfile.sql \ - regtool_revertprocessedexp.sql \ - regtool_revertprocessedimfile.sql \ - regtool_updateprocessedimfile.sql \ - regtool_pendingcompressimfile.sql \ - regtool_finishcompressexp.sql \ - regtool_updatebyquery.sql \ - regtool_updatebyqueryimfile.sql \ - stacktool_associationdefine_select.sql \ - stacktool_definebyquery_insert.sql \ - stacktool_definebyquery_insert_random_part1.sql \ - stacktool_definebyquery_insert_random_part2.sql \ - stacktool_definebyquery_select.sql \ - stacktool_definebyquery_test.sql \ - stacktool_donecleanup.sql \ - stacktool_export_input_skyfile.sql \ - stacktool_export_run.sql \ - stacktool_export_sum_skyfile.sql \ - stacktool_find_complete_warps.sql \ - stacktool_inputskyfile.sql \ - stacktool_pendingcleanuprun.sql \ - stacktool_pendingcleanupskyfile.sql \ - stacktool_revertsumskyfile_delete.sql \ - stacktool_sumskyfile.sql \ - stacktool_sassskyfile.sql \ - stacktool_tosum.sql \ - stacktool_tosummary.sql \ - stacktool_addsummary.sql \ - staticskytool_definebyquery_select.sql \ - staticskytool_definebyquery_inputs.sql \ - staticskytool_inputs.sql \ - staticskytool_todo.sql \ - staticskytool_result.sql \ - staticskytool_revert.sql \ - warptool_change_skyfile_data_state.sql \ - warptool_change_run_state.sql \ - warptool_definebyquery.sql \ - warptool_donecleanup.sql \ - warptool_exp.sql \ - warptool_export_imfile.sql \ - warptool_export_run.sql \ - warptool_export_skyfile.sql \ - warptool_export_skycell_map.sql \ - warptool_finished_run_select.sql \ - warptool_finish_run.sql \ - warptool_coalesce_run.sql \ - warptool_imfile.sql \ - warptool_listrun.sql \ - warptool_pendingcleanuprun.sql \ - warptool_pendingcleanupskyfile.sql \ - warptool_revertoverlap.sql \ - warptool_revertwarped_delete.sql \ - warptool_revertwarped_updated.sql \ - warptool_runstate.sql \ - warptool_scmap.sql \ - warptool_setskyfiletoupdate.sql \ - warptool_tooverlap.sql \ - warptool_towarped.sql \ - warptool_updateskyfile.sql \ - warptool_warped.sql \ - warptool_tosummary.sql \ - warptool_addsummary.sql \ + addtool_checkminidvodbaddrun.sql \ + addtool_donecleanup.sql \ + addtool_find_cam_id.sql \ + addtool_find_pendingexp.sql \ + addtool_find_pendingmergeprocess.sql \ + addtool_find_processedexp.sql \ + addtool_find_minidvodbprocessed.sql \ + addtool_find_minidvodbrun.sql \ + addtool_pendingcleanupexp.sql \ + addtool_pendingcleanuprun.sql \ + addtool_queue_cam_id.sql \ + addtool_queue_minidvodbrun.sql \ + addtool_revertminidvodbprocessed.sql \ + addtool_revertprocessedexp.sql \ + bgtool_advancechip.sql \ + bgtool_advancewarp.sql \ + bgtool_chipinputs.sql \ + bgtool_chip.sql \ + bgtool_cleanedchip.sql \ + bgtool_cleanedwarp.sql \ + bgtool_definechip.sql \ + bgtool_definewarp.sql \ + bgtool_revertchip.sql \ + bgtool_revertwarp.sql \ + bgtool_tochip.sql \ + bgtool_tocleanchip.sql \ + bgtool_tocleanwarp.sql \ + bgtool_towarp.sql \ + bgtool_warpinputs.sql \ + bgtool_warp.sql \ + camtool_donecleanup.sql \ + camtool_find_chip_id.sql \ + camtool_pendingexp.sql \ + camtool_find_pendingimfile.sql \ + camtool_find_processedexp.sql \ + camtool_pendingcleanupexp.sql \ + camtool_pendingcleanuprun.sql \ + camtool_queue_chip_id.sql \ + camtool_revertprocessedexp.sql \ + camtool_export_run.sql \ + camtool_export_processed_exp.sql \ + chiptool_change_exp_state.sql \ + chiptool_change_imfile_data_state.sql \ + chiptool_completely_processed_exp.sql \ + chiptool_coalesce_run.sql \ + chiptool_definecopy.sql \ + chiptool_donecleanup.sql \ + chiptool_find_rawexp.sql \ + chiptool_listrun.sql \ + chiptool_pendingcleanupimfile.sql \ + chiptool_pendingcleanuprun.sql \ + chiptool_pendingimfile.sql \ + chiptool_processedimfile.sql \ + chiptool_revertprocessedimfile.sql \ + chiptool_revertupdatedimfile.sql \ + chiptool_run.sql \ + chiptool_runstate.sql \ + chiptool_setimfiletoupdate.sql \ + chiptool_export_run.sql \ + chiptool_export_imfile.sql \ + chiptool_export_processed_imfile.sql \ + chiptool_unmasked.sql \ + detselect_search.sql \ + detselect_select.sql \ + dettool_addprocessedexp.sql \ + dettool_childlessrun.sql \ + dettool_definebydetrun.sql \ + dettool_detrunsummary.sql \ + dettool_find_completed_runs.sql \ + dettool_input.sql \ + dettool_normalizedexp.sql \ + dettool_normalizedimfile.sql \ + dettool_normalizedstat.sql \ + dettool_pending.sql \ + dettool_pendingcleanup_detrunsummary.sql \ + dettool_pendingcleanup_normalizedexp.sql \ + dettool_pendingcleanup_normalizedimfile.sql \ + dettool_pendingcleanup_normalizedstat.sql \ + dettool_pendingcleanup_processedexp.sql \ + dettool_pendingcleanup_processedimfile.sql \ + dettool_pendingcleanup_residexp.sql \ + dettool_pendingcleanup_residimfile.sql \ + dettool_pendingcleanup_stacked.sql \ + dettool_processedimfile.sql \ + dettool_raw.sql \ + dettool_residexp.sql \ + dettool_residimfile.sql \ + dettool_revertdetrunsummary.sql \ + dettool_revertnormalizedexp.sql \ + dettool_revertnormalizedimfile.sql \ + dettool_revertnormalizedstat.sql \ + dettool_revertprocessedexp.sql \ + dettool_revertprocessedimfile.sql \ + dettool_revertresidexp.sql \ + dettool_revertresidimfile.sql \ + dettool_revertstacked.sql \ + dettool_runs.sql \ + dettool_stacked.sql \ + dettool_start_new_iteration.sql \ + dettool_stop_completed_correct_runs.sql \ + dettool_tocorrectexp.sql \ + dettool_tocorrectimfile.sql \ + dettool_todetrunsummary.sql \ + dettool_tonormalize.sql \ + dettool_tonormalizedexp.sql \ + dettool_tonormalizedstat.sql \ + dettool_toprocessedexp.sql \ + dettool_toprocessedimfile.sql \ + dettool_toresidexp.sql \ + dettool_toresidimfile.sql \ + dettool_tostacked.sql \ + difftool_change_skyfile_data_state.sql \ + difftool_change_run_state.sql \ + difftool_completed_runs.sql \ + difftool_coalesce_run.sql \ + difftool_definewarpstack_part1.sql \ + difftool_definewarpstack_part2.sql \ + difftool_definewarpstack_temp_create.sql \ + difftool_definewarpwarp_temp_create.sql \ + difftool_definewarpwarp_temp_insert.sql \ + difftool_definewarpwarp_select.sql \ + difftool_definewarpwarp_insert.sql \ + difftool_definestackstack_part0.sql \ + difftool_definestackstack_part1.sql \ + difftool_donecleanup.sql \ + difftool_export_input_skyfile.sql \ + difftool_export_run.sql \ + difftool_export_skyfile.sql \ + difftool_inputskyfile.sql \ + difftool_listrun.sql \ + difftool_pendingcleanuprun.sql \ + difftool_pendingcleanupskyfile.sql \ + difftool_revertdiffskyfile_delete.sql \ + difftool_revertdiffskyfile_updated.sql \ + difftool_setskyfiletoupdate.sql \ + difftool_skyfile.sql \ + difftool_todiffskyfile.sql \ + difftool_tosummary.sql \ + difftool_addsummary.sql \ + disttool_definebyquery_camera.sql \ + disttool_definebyquery_chip.sql \ + disttool_definebyquery_diff.sql \ + disttool_definebyquery_fake.sql \ + disttool_definebyquery_raw.sql \ + disttool_definebyquery_stack.sql \ + disttool_definebyquery_warp.sql \ + disttool_definebyquery_SSdiff.sql \ + disttool_defineinterest.sql \ + disttool_listfilesets.sql \ + disttool_listinterests.sql \ + disttool_pending_camera.sql \ + disttool_pending_chip.sql \ + disttool_pending_diff.sql \ + disttool_pending_fake.sql \ + disttool_pending_raw.sql \ + disttool_pending_stack.sql \ + disttool_pending_warp.sql \ + disttool_pending_SSdiff.sql \ + disttool_pendingfileset.sql \ + disttool_pendingdest.sql \ + disttool_pendingcleanup.sql \ + disttool_processedcomponent.sql \ + disttool_queuercrun.sql \ + disttool_revertrcrun.sql \ + disttool_revertrun.sql \ + disttool_revertcomponent.sql \ + disttool_revertfileset.sql \ + disttool_toadvance.sql \ + disttool_updateinterest.sql \ + disttool_updatercrun.sql \ + dqstatstool_createbundle.sql \ + dqstatstool_definebyquery.sql \ + dqstatstool_get_contents.sql \ + dqstatstool_get_run.sql \ + faketool_change_exp_state.sql \ + faketool_change_imfile_data_state.sql \ + faketool_completely_processed_exp.sql \ + faketool_donecleanup.sql \ + faketool_export_processed_imfile.sql \ + faketool_export_run.sql \ + faketool_find_camrun.sql \ + faketool_find_pendingexp.sql \ + faketool_pendingcleanupimfile.sql \ + faketool_pendingcleanuprun.sql \ + faketool_pendingimfile.sql \ + faketool_processedimfile.sql \ + faketool_queue_cam_id.sql \ + faketool_revertprocessedimfile.sql \ + faketool_unmasked.sql \ + flatcorr_chiprundone.sql \ + flatcorr_camerarundone.sql \ + flatcorr_pendingprocess.sql \ + flatcorr_inputimfile.sql \ + flatcorr_dropchip.sql \ + flatcorr_dropcamera.sql \ + magictool_addmask.sql \ + magictool_restore_camera.sql \ + magictool_restore_chip.sql \ + magictool_restore_diff.sql \ + magictool_restore_raw.sql \ + magictool_restore_warp.sql \ + magictool_create_tmp_warpcomplete.sql \ + magictool_definebyquery_insert.sql \ + magictool_definebyquery_select.sql \ + magictool_inputs.sql \ + magictool_inputskyfile.sql \ + magictool_mask.sql \ + magictool_tomask.sql \ + magictool_toprocess_inputs.sql \ + magictool_toprocess_runs.sql \ + magictool_toprocess_tree.sql \ + magictool_toskyfilemask.sql \ + magictool_totree.sql \ + magictool_diffskyfile.sql \ + magictool_warpskyfile.sql \ + magictool_chipprocessedimfile.sql \ + magictool_rawimfile.sql \ + magictool_revertnode.sql \ + magictool_exposure.sql \ + magicdstool_clearstatefaults.sql \ + magicdstool_completed_runs.sql \ + magicdstool_completedrevert.sql \ + magicdstool_definebyquery_raw.sql \ + magicdstool_definebyquery_chip.sql \ + magicdstool_definebyquery_camera.sql \ + magicdstool_definebyquery_warp.sql \ + magicdstool_definebyquery_diff.sql \ + magicdstool_definecopy_chip.sql \ + magicdstool_definecopy_warp.sql \ + magicdstool_getrunids.sql \ + magicdstool_getskycells.sql \ + magicdstool_revertdestreakedfile.sql \ + magicdstool_tocleanup.sql \ + magicdstool_todestreak_camera.sql \ + magicdstool_todestreak_chip.sql \ + magicdstool_todestreak_diff.sql \ + magicdstool_todestreak_raw.sql \ + magicdstool_todestreak_warp.sql \ + magicdstool_toremove.sql \ + magicdstool_torevert_raw.sql \ + magicdstool_torevert_chip.sql \ + magicdstool_torevert_camera.sql \ + magicdstool_torevert_warp.sql \ + magicdstool_torevert_diff.sql \ + pstamptool_completedreq.sql \ + pstamptool_datastore.sql \ + pstamptool_getdependent.sql \ + pstamptool_listjob.sql \ + pstamptool_pendingcleanup.sql \ + pstamptool_pendingdependent.sql \ + pstamptool_pendingjob.sql \ + pstamptool_pendingreq.sql \ + pstamptool_project.sql \ + pstamptool_revertdependent.sql \ + pstamptool_revertjob.sql \ + pstamptool_revertreq.sql \ + pstamptool_revertreq_deletejobs.sql \ + pstamptool_updatejob.sql \ + pxadmin_create_tables.sql \ + pxadmin_create_mirror_tables.sql \ + pxadmin_drop_tables.sql \ + pxadmin_update_version.sql \ + pubtool_definerun.sql \ + pubtool_pending.sql \ + pubtool_revert.sql \ + pztool_find_completed_exp.sql \ + pztool_pendingimfile.sql \ + pztool_revert_downloadimfile_faults.sql \ + pztool_revert_fileset_faults.sql \ + pztool_revertcopied.sql \ + receivetool_list.sql \ + receivetool_addfileset.sql \ + receivetool_pendingfileset.sql \ + receivetool_pendingfile.sql \ + receivetool_revert.sql \ + receivetool_toadvance.sql \ + regtool_create_dup_table.sql \ + regtool_export_exp.sql \ + regtool_export_imfile.sql \ + regtool_pendingexp.sql \ + regtool_pendingimfile.sql \ + regtool_populate_dup_table.sql \ + regtool_processedexp.sql \ + regtool_processedimfile.sql \ + regtool_revertprocessedexp.sql \ + regtool_revertprocessedimfile.sql \ + regtool_updateprocessedimfile.sql \ + regtool_pendingcompressimfile.sql \ + regtool_finishcompressexp.sql \ + regtool_updatebyquery.sql \ + regtool_updatebyqueryimfile.sql \ + stacktool_associationdefine_select.sql \ + stacktool_definebyquery_insert.sql \ + stacktool_definebyquery_insert_random_part1.sql \ + stacktool_definebyquery_insert_random_part2.sql \ + stacktool_definebyquery_select.sql \ + stacktool_definebyquery_test.sql \ + stacktool_donecleanup.sql \ + stacktool_export_input_skyfile.sql \ + stacktool_export_run.sql \ + stacktool_export_sum_skyfile.sql \ + stacktool_find_complete_warps.sql \ + stacktool_inputskyfile.sql \ + stacktool_pendingcleanuprun.sql \ + stacktool_pendingcleanupskyfile.sql \ + stacktool_revertsumskyfile_delete.sql \ + stacktool_sumskyfile.sql \ + stacktool_sassskyfile.sql \ + stacktool_tosum.sql \ + stacktool_tosummary.sql \ + stacktool_addsummary.sql \ + staticskytool_definebyquery_select.sql \ + staticskytool_definebyquery_inputs.sql \ + staticskytool_inputs.sql \ + staticskytool_todo.sql \ + staticskytool_result.sql \ + staticskytool_revert.sql \ + warptool_change_skyfile_data_state.sql \ + warptool_change_run_state.sql \ + warptool_definebyquery.sql \ + warptool_donecleanup.sql \ + warptool_exp.sql \ + warptool_export_imfile.sql \ + warptool_export_run.sql \ + warptool_export_skyfile.sql \ + warptool_export_skycell_map.sql \ + warptool_finished_run_select.sql \ + warptool_finish_run.sql \ + warptool_coalesce_run.sql \ + warptool_imfile.sql \ + warptool_listrun.sql \ + warptool_pendingcleanuprun.sql \ + warptool_pendingcleanupskyfile.sql \ + warptool_revertoverlap.sql \ + warptool_revertwarped_delete.sql \ + warptool_revertwarped_updated.sql \ + warptool_runstate.sql \ + warptool_scmap.sql \ + warptool_setskyfiletoupdate.sql \ + warptool_tooverlap.sql \ + warptool_towarped.sql \ + warptool_updateskyfile.sql \ + warptool_warped.sql \ + warptool_tosummary.sql \ + warptool_addsummary.sql \ diffphottool_definerun.sql \ diffphottool_input.sql \ Index: trunk/ippTools/share/bgtool_advancechip.sql =================================================================== --- trunk/ippTools/share/bgtool_advancechip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_advancechip.sql (revision 28486) @@ -0,0 +1,15 @@ +SELECT DISTINCT + chipBackgroundRun.chip_bg_id, + chipBackgroundImfile.magicked +FROM chipBackgroundRun +JOIN chipProcessedImfile USING(chip_id) +LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id) +WHERE chipBackgroundRun.state = 'new' + AND chipProcessedImfile.quality = 0 + AND chipProcessedImfile.fault = 0 +-- WHERE hook %s +GROUP BY chip_bg_id +HAVING + COUNT(chipBackgroundImfile.class_id) = COUNT(chipProcessedImfile.class_id) + AND SUM(IF(chipBackgroundImfile.fault > 0, 1, 0)) = 0 + AND SUM(IF(chipBackgroundImfile.quality > 0, 1, 0)) != COUNT(*) Index: trunk/ippTools/share/bgtool_advancewarp.sql =================================================================== --- trunk/ippTools/share/bgtool_advancewarp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_advancewarp.sql (revision 28486) @@ -0,0 +1,15 @@ +SELECT DISTINCT + warpBackgroundRun.warp_bg_id, + warpBackgroundSkyfile.magicked +FROM warpBackgroundRun +JOIN warpSkyfile USING(warp_id) +LEFT JOIN warpBackgroundSkyfile USING(warp_bg_id, skycell_id) +WHERE warpBackgroundRun.state = 'new' + AND warpSkyfile.quality = 0 + AND warpSkyfile.fault = 0 +-- WHERE hook %s +GROUP BY warp_bg_id +HAVING + COUNT(warpBackgroundSkyfile.skycell_id) = COUNT(warpSkyfile.skycell_id) + AND SUM(IF(warpBackgroundSkyfile.fault > 0, 1, 0)) = 0 + AND SUM(IF(warpBackgroundSkyfile.quality > 0, 1, 0)) != COUNT(*) Index: trunk/ippTools/share/bgtool_chip.sql =================================================================== --- trunk/ippTools/share/bgtool_chip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_chip.sql (revision 28486) @@ -0,0 +1,17 @@ +SELECT + chipBackgroundImfile.*, + chipBackgroundRun.state, + chipBackgroundRun.workdir, + chipBackgroundRun.label, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM chipBackgroundRun +JOIN chipBackgroundImfile USING(chip_bg_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: trunk/ippTools/share/bgtool_chipinputs.sql =================================================================== --- trunk/ippTools/share/bgtool_chipinputs.sql (revision 28486) +++ trunk/ippTools/share/bgtool_chipinputs.sql (revision 28486) @@ -0,0 +1,8 @@ +SELECT + chipProcessedImfile.* +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN chipProcessedImfile USING(chip_id) +WHERE chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 Index: trunk/ippTools/share/bgtool_cleanedchip.sql =================================================================== --- trunk/ippTools/share/bgtool_cleanedchip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_cleanedchip.sql (revision 28486) @@ -0,0 +1,3 @@ +UPDATE chipBackgroundRun +SET state = '%s' +WHERE state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') Index: trunk/ippTools/share/bgtool_cleanedwarp.sql =================================================================== --- trunk/ippTools/share/bgtool_cleanedwarp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_cleanedwarp.sql (revision 28486) @@ -0,0 +1,3 @@ +UPDATE warpBackgroundRun +SET state = '%s' +WHERE state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') Index: trunk/ippTools/share/bgtool_definechip.sql =================================================================== --- trunk/ippTools/share/bgtool_definechip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_definechip.sql (revision 28486) @@ -0,0 +1,6 @@ +SELECT + chipRun.* +FROM chipRun +JOIN rawExp USING(exp_id) +LEFT JOIN chipBackgroundRun USING(chip_id) +WHERE chipRun.state = 'full' Index: trunk/ippTools/share/bgtool_definewarp.sql =================================================================== --- trunk/ippTools/share/bgtool_definewarp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_definewarp.sql (revision 28486) @@ -0,0 +1,11 @@ +SELECT + warpRun.*, + chipBackgroundRun.chip_bg_id +FROM warpRun +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN chipBackgroundRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN warpBackgroundRun USING(chip_bg_id) +WHERE chipBackgroundRun.state = 'full' Index: trunk/ippTools/share/bgtool_revertchip.sql =================================================================== --- trunk/ippTools/share/bgtool_revertchip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_revertchip.sql (revision 28486) @@ -0,0 +1,4 @@ +DELETE chipBackgroundImfile +FROM chipBackgroundRun +JOIN chipBackgroundImfile USING(chip_bg_id) +WHERE chipBackgroundImfile.fault != 0 Index: trunk/ippTools/share/bgtool_revertwarp.sql =================================================================== --- trunk/ippTools/share/bgtool_revertwarp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_revertwarp.sql (revision 28486) @@ -0,0 +1,4 @@ +DELETE warpBackgroundSkyfile +FROM warpBackgroundRun +JOIN warpBackgroundSkyfile USING(warp_bg_id) +WHERE warpBackgroundSkyfile.fault != 0 Index: trunk/ippTools/share/bgtool_tochip.sql =================================================================== --- trunk/ippTools/share/bgtool_tochip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_tochip.sql (revision 28486) @@ -0,0 +1,18 @@ +SELECT + chipBackgroundRun.*, + class_id, + rawExp.exp_tag, + rawExp.camera +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +JOIN chipProcessedImfile USING(chip_id) +LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id) +LEFT JOIN Label ON chipBackgroundRun.label = Label.label +WHERE chipBackgroundImfile.chip_bg_id IS NULL + AND chipRun.state = 'full' + AND chipProcessedImfile.fault = 0 + AND chipProcessedImfile.quality = 0 + AND (Label.active OR Label.active IS NULL) +-- WHERE hook %s +ORDER BY priority DESC, chip_bg_id Index: trunk/ippTools/share/bgtool_tocleanchip.sql =================================================================== --- trunk/ippTools/share/bgtool_tocleanchip.sql (revision 28486) +++ trunk/ippTools/share/bgtool_tocleanchip.sql (revision 28486) @@ -0,0 +1,9 @@ +SELECT + chipBackgroundRun.chip_bg_id, + chipBackgroundRun.state, + rawExp.camera +FROM chipBackgroundRun +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE chipBackgroundRun.state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') + Index: trunk/ippTools/share/bgtool_tocleanwarp.sql =================================================================== --- trunk/ippTools/share/bgtool_tocleanwarp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_tocleanwarp.sql (revision 28486) @@ -0,0 +1,12 @@ +SELECT + warpBackgroundRun.warp_bg_id, + warpBackgroundRun.state, + rawExp.camera +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE warpBackgroundRun.state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged') + Index: trunk/ippTools/share/bgtool_towarp.sql =================================================================== --- trunk/ippTools/share/bgtool_towarp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_towarp.sql (revision 28486) @@ -0,0 +1,23 @@ +SELECT + warpBackgroundRun.*, + warpRun.tess_id, + skycell_id, + rawExp.exp_tag, + rawExp.camera +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap USING(warp_id) +JOIN warpSkyfile USING(warp_id, skycell_id) +JOIN chipBackgroundRun USING(chip_bg_id) +JOIN chipBackgroundImfile USING(chip_bg_id, class_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN warpBackgroundSkyfile USING(warp_bg_id, skycell_id) +LEFT JOIN Label ON Label.label = warpBackgroundRun.label +WHERE warpBackgroundSkyfile.warp_bg_id IS NULL + AND chipBackgroundRun.state = 'full' + AND chipBackgroundImfile.fault = 0 + AND chipBackgroundImfile.quality = 0 + AND (Label.active OR Label.active IS NULL) +-- WHERE hook %s +ORDER BY priority DESC, chip_bg_id Index: trunk/ippTools/share/bgtool_warp.sql =================================================================== --- trunk/ippTools/share/bgtool_warp.sql (revision 28486) +++ trunk/ippTools/share/bgtool_warp.sql (revision 28486) @@ -0,0 +1,20 @@ +SELECT + warpBackgroundSkyfile.*, + warpBackgroundRun.state, + warpBackgroundRun.workdir, + warpBackgroundRun.label, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.filter, + rawExp.dateobs, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM warpBackgroundRun +JOIN warpBackgroundImfile USING(warp_bg_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) Index: trunk/ippTools/share/bgtool_warpinputs.sql =================================================================== --- trunk/ippTools/share/bgtool_warpinputs.sql (revision 28486) +++ trunk/ippTools/share/bgtool_warpinputs.sql (revision 28486) @@ -0,0 +1,17 @@ +SELECT + chipBackgroundImfile.path_base AS chip_path_base, + camProcessedExp.path_base AS cam_path_base +FROM warpBackgroundRun +JOIN warpRun USING(warp_id) +JOIN warpSkyCellMap USING(warp_id) +JOIN warpSkyfile USING(warp_id, skycell_id) +JOIN chipBackgroundRun USING(chip_bg_id) +JOIN chipBackgroundImfile USING(chip_bg_id, class_id) +JOIN fakeRun USING(fake_id) +JOIN camProcessedExp USING(cam_id) +WHERE warpRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND chipBackgroundRun.state = 'full' + AND chipBackgroundImfile.fault = 0 + AND chipBackgroundImfile.quality = 0 Index: trunk/ippTools/share/pxadmin_create_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_create_tables.sql (revision 28453) +++ trunk/ippTools/share/pxadmin_create_tables.sql (revision 28486) @@ -1715,4 +1715,6 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Processing labels with their priorities CREATE TABLE Label ( label VARCHAR(64), @@ -1723,4 +1725,101 @@ KEY(priority), KEY(active) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Tables to support background restoration + +-- Background replacement on a chipRun +CREATE TABLE chipBackgroundRun ( + chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + chip_id BIGINT NOT NULL, -- link to chipRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + magicked BIGINT DEFAULT 0 NOT NULL, -- magic status + PRIMARY KEY(chip_bg_id), + KEY(chip_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from chipBackgroundRun +CREATE TABLE chipBackgroundImfile ( + chip_bg_id BIGINT NOT NULL, -- unique identifier + class_id VARCHAR(64) NOT NULL, -- class (component) identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(chip_bg_id,class_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Background replacement on a warpRun (utilising chipBackgroundRun) +CREATE TABLE warpBackgroundRun ( + warp_bg_id BIGINT AUTO_INCREMENT, -- unique identifier + warp_id BIGINT NOT NULL, -- link to warpRun + chip_bg_id BIGINT NOT NULL, -- link to chipBackgroundRun + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + note VARCHAR(255), -- note + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + magicked BIGINT DEFAULT 0 NOT NULL, -- magic status + PRIMARY KEY(warp_bg_id), + KEY(warp_id), + KEY(chip_bg_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), + FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Results of background replacement from warpBackgroundRun +CREATE TABLE warpBackgroundSkyfile ( + warp_bg_id BIGINT NOT NULL, -- unique identifier + skycell_id VARCHAR(64) NOT NULL, -- skycell identifier + path_base VARCHAR(255) NOT NULL, -- root name for outputs + magicked BIGINT, -- magic_id if magicked + dtime_script FLOAT, -- elapsed time for script + hostname VARCHAR(64) NOT NULL, -- host that executed script + quality SMALLINT NOT NULL, -- bad quality flag + fault SMALLINT NOT NULL, -- fault code + software_ver VARCHAR(16), -- software version + bg FLOAT, -- background level + bg_stdev FLOAT, -- stdev of background + maskfrac_npix FLOAT, -- Number of pixels masked + maskfrac_static FLOAT, -- Fraction masked static + maskfrac_dynamic FLOAT, -- Fraction masked dynamic + maskfrac_magic FLOAT, -- Fraction masked magic + maskfrac_advisory FLOAT, -- Fraction masked advisory + PRIMARY KEY(warp_bg_id,skycell_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: trunk/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- trunk/ippTools/share/pxadmin_drop_tables.sql (revision 28453) +++ trunk/ippTools/share/pxadmin_drop_tables.sql (revision 28486) @@ -91,4 +91,8 @@ DROP TABLE IF EXISTS Label; DROP TABLE IF EXISTS pstampWebRequest; +DROP TABLE IF EXISTS chipBackgroundRun; +DROP TABLE IF EXISTS chipBackgroundImfile; +DROP TABLE IF EXISTS warpBackgroundRun; +DROP TABLE IF EXISTS warpBackgroundSkyfile; DROP TABLE IF EXISTS diffPhotRun; DROP TABLE IF EXISTS diffPhotSkyfile; Index: trunk/ippTools/src/Makefile.am =================================================================== --- trunk/ippTools/src/Makefile.am (revision 28453) +++ trunk/ippTools/src/Makefile.am (revision 28486) @@ -1,4 +1,5 @@ bin_PROGRAMS = \ addtool \ + bgtool \ caltool \ camtool \ @@ -48,4 +49,5 @@ noinst_HEADERS = \ addtool.h \ + bgtool.h \ caltool.h \ camtool.h \ @@ -117,4 +119,10 @@ addtoolConfig.c +bgtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +bgtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +bgtool_SOURCES = \ + bgtool.c \ + bgtoolConfig.c + caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la Index: trunk/ippTools/src/bgtool.c =================================================================== --- trunk/ippTools/src/bgtool.c (revision 28486) +++ trunk/ippTools/src/bgtool.c (revision 28486) @@ -0,0 +1,1799 @@ +/* + * bgtool.c + * + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVB_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "bgtool.h" + +static bool definechipMode(pxConfig *config); +static bool updatechipMode(pxConfig *config); +static bool tochipMode(pxConfig *config); +static bool chipinputsMode(pxConfig *config); +static bool addchipMode(pxConfig *config); +static bool chipMode(pxConfig *config); +static bool advancechipMode(pxConfig *config); +static bool revertchipMode(pxConfig *config); +static bool definewarpMode(pxConfig *config); +static bool updatewarpMode(pxConfig *config); +static bool towarpMode(pxConfig *config); +static bool warpinputsMode(pxConfig *config); +static bool addwarpMode(pxConfig *config); +static bool warpMode(pxConfig *config); +static bool advancewarpMode(pxConfig *config); +static bool revertwarpMode(pxConfig *config); +static bool tocleanchipMode(pxConfig *config); +static bool cleanedchipMode(pxConfig *config); +static bool tocleanwarpMode(pxConfig *config); +static bool cleanedwarpMode(pxConfig *config); +static bool exportchipMode(pxConfig *config); +static bool importchipMode(pxConfig *config); +static bool exportwarpMode(pxConfig *config); +static bool importwarpMode(pxConfig *config); + +// Tables to import/export +typedef struct { + const char *name; // Table name + void* (*parse)(); // Parsing function + bool (*insert)(); // Insertion function +} tableData; +static const tableData chipTables[] = { + { "chipBackgroundRun", (void*)&chipBackgroundRunObjectFromMetadata, &chipBackgroundRunInsertObject }, + { "chipBackgroundImfile", (void*)&chipBackgroundImfileObjectFromMetadata, &chipBackgroundImfileInsertObject }, + { NULL, NULL, NULL } +}; +static const tableData warpTables[] = { + { "warpBackgroundRun", (void*)&warpBackgroundRunObjectFromMetadata, &warpBackgroundRunInsertObject }, + { "warpBackgroundImfile", (void*)&warpBackgroundSkyfileObjectFromMetadata, &warpBackgroundSkyfileInsertObject }, + { NULL, NULL, NULL } +}; + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = bgtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(BGTOOL_MODE_DEFINECHIP, definechipMode); + MODECASE(BGTOOL_MODE_UPDATECHIP, updatechipMode); + MODECASE(BGTOOL_MODE_TOCHIP, tochipMode); + MODECASE(BGTOOL_MODE_CHIPINPUTS, chipinputsMode); + MODECASE(BGTOOL_MODE_ADDCHIP, addchipMode); + MODECASE(BGTOOL_MODE_CHIP, chipMode); + MODECASE(BGTOOL_MODE_ADVANCECHIP, advancechipMode); + MODECASE(BGTOOL_MODE_REVERTCHIP, revertchipMode); + MODECASE(BGTOOL_MODE_DEFINEWARP, definewarpMode); + MODECASE(BGTOOL_MODE_UPDATEWARP, updatewarpMode); + MODECASE(BGTOOL_MODE_TOWARP, towarpMode); + MODECASE(BGTOOL_MODE_WARPINPUTS, warpinputsMode); + MODECASE(BGTOOL_MODE_ADDWARP, addwarpMode); + MODECASE(BGTOOL_MODE_WARP, warpMode); + MODECASE(BGTOOL_MODE_ADVANCEWARP, advancewarpMode); + MODECASE(BGTOOL_MODE_REVERTWARP, revertwarpMode); + MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode); + MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode); + MODECASE(BGTOOL_MODE_TOCLEANWARP, tocleanwarpMode); + MODECASE(BGTOOL_MODE_CLEANEDWARP, cleanedwarpMode); + MODECASE(BGTOOL_MODE_EXPORTCHIP, exportchipMode); + MODECASE(BGTOOL_MODE_IMPORTCHIP, importchipMode); + MODECASE(BGTOOL_MODE_EXPORTWARP, exportwarpMode); + MODECASE(BGTOOL_MODE_IMPORTWARP, importwarpMode); + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// General functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool exportTables(pxConfig *config, // Configuration (with DB handle) + const char *filename, // Filename to which to write + const tableData tables[], // Tables to export (NULL terminated) + const psMetadata *where, // WHERE restrictions + bool clean // Write as cleaned? + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + FILE *file = fopen(filename, "w"); + if (!file) { + psError(PXTOOLS_ERR_SYS, true, "failed to open output file %s", filename); + return false; + } + + if (!pxExportVersion(config, file)) { + psError(psErrorCodeLast(), false, "failed to write dbversion to output file %s", filename); + return false; + } + + for (int i = 0; tables[i].name; i++) { + const char *name = tables[i].name; // Name of table + psString query = NULL; + psStringAppend(&query, "SELECT * FROM %s", name); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PXTOOLS_ERR_CONFIG, true, "no rows found"); + psFree(output); + return false; + } + + if (clean && + (strcmp(name, "chipBackgroundRun") == 0 || + strcmp(name, "warpBackgroundRun") == 0) && + !pxSetStateCleaned(name, "state", output)) { + psFree(output); + psError(psErrorCodeLast(), false, "pxSetStateClean failed for table %s", name); + return false; + } + + if (!ippdbPrintMetadatas(file, output, name, true)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + fclose(file); + + return true; +} + +static bool importTables(pxConfig *config, // Configuration + const char *filename, // + const tableData tables[] // Tables to read in + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + unsigned int badLines = 0; // Number of bad lines + psMetadata *input = psMetadataConfigRead(NULL, &badLines, filename, false); // Input file contents + if (!input) { + psError(psErrorCodeLast(), false, "Unable to parse input file %s", filename); + return false; + } + if (badLines > 0) { + psWarning("%d bad lines encountered when parsing %s", badLines, filename); + } + + if (!pxCheckImportVersion(config, input)) { + psError(psErrorCodeLast(), false, "pxCheckImportVersion failed"); + return false; + } + + // Import primary table + for (int i = 0; tables[i].name; i++) { + const char *name = tables[i].name; // Name of table + psMetadataItem *item = psMetadataLookup(input, name); // Item from input + psAssert(item, "%s not in input", name); + psAssert(item->type == PS_DATA_METADATA_MULTI, "%s not MULTI type", name); + psAssert(psListLength(item->data.list) == 1, "%s has multiple entries", name); + psMetadataItem *entry = psListGet(item->data.list, PS_LIST_HEAD); // Entry of interest + void *data = tables[i].parse(entry); // Parsed entry + if (!data) { + psError(PXTOOLS_ERR_CONFIG, false, "Unable to parse entry %s", name); + psFree(input); + return false; + } + if (!tables[0].insert(config->dbh, data)) { + psError(psErrorCodeLast(), false, "Unable to insert entry %s", name); + psFree(input); + return false; + } + } + + psFree(input); + + return true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Functions for chip stage +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool definechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + pxAddLabelSearchArgs(config, where, "-label", "chipRun.label", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // Get chip runs to promote to chipBackgroundRun + + psString query = pxDataGet("bgtool_definechip.sql"); // Query to execute + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!rerun) { + psStringAppend(&query, "\nAND chipBackgroundRun.chip_bg_id IS NULL"); + if (label) { + psStringAppend(&query, + "\nAND (chipBackgroundRun.label = '%s'" + " OR chipBackgroundRun.label IS NULL", + label); + } + if (data_group) { + psStringAppend(&query, + "\nAND (chipBackgroundRun.data_group = '%s'" + " OR chipBackgroundRun.data_group IS NULL", + data_group); + } + if (dist_group) { + psStringAppend(&query, + "\nAND (chipBackgroundRun.dist_group = '%s'" + " OR chipBackgroundRun.dist_group IS NULL", + dist_group); + } + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); // Matching rows + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + chipRunRow *row = chipRunObjectFromMetadata(md); + if (!row) { + psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, "new", + workdir ? workdir : row->workdir, + label ? label : row->label, + data_group ? data_group : row->data_group, + dist_group ? dist_group : row->dist_group, + reduction ? reduction : row->reduction, + note ? note : row->note, + NULL, 0)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(row); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + return true; +} + +static bool updatechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipBackgroundRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "chipBackgroundRun.data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipBackgroundRun.dist_group","LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "chipBackgroundRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "chipBackgroundRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "chipBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE chipBackgroundRun JOIN chipBackgroundImfile USING(chip_bg_id)"); + bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id", + "chipBackgroundImfile", true); + + psFree(query); + psFree(where); + + return result; +} + +static bool tochipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("bgtool_tochip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereStr, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(whereStr); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool chipinputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_chipinputs.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules); + ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground); + ver_code = pxMergeCodeVersions(ver_code, ver_ppstats); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script, + hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, + maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool chipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-data_group", "chipBackgroundRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, where, "-dist_group", "chipBackgroundRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_chip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString magicWhere = NULL; + if (!pxmagicAddWhere(config, &magicWhere, "chipBackgroundImfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !magicWhere) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + if (magicWhere) { + psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere); + } + psFree(magicWhere); + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + + return true; +} + + +static bool advancechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString select = pxDataGet("bgtool_advancechip.sql"); + if (!select) { + psError(psErrorCodeLast(), false, "failed to retrieve SQL statement"); + return false; + } + + psString selectWhere = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&selectWhere, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&select, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(select); + psFree(selectWhere); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(select); + psFree(selectWhere); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = true; // Status of MD lookup + psS64 chip_bg_id = psMetadataLookupS64(&status, row, "chip_bg_id"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for chip_bg_id"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE chipBackgroundRun " + "SET state = 'full', magicked = %" PRId64 " " + " WHERE chip_bg_id = %" PRId64, + magicked, chip_bg_id)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool revertchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "chipBackgroundImfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psString query = pxDataGet("bgtool_revertchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted); + + return true; +} + +static bool tocleanchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + + psString query = pxDataGet("bgtool_tocleanchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool cleanedchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + if (!pxIsValidCleanedState(state)) { + psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); + return false; + } + + psString query = pxDataGet("bgtool_cleanedchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + psFree(query); + + return true; +} + +static bool exportchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); + + bool status = exportTables(config, outfile, chipTables, where, clean); + + psFree(where); + return status; +} + +static bool importchipMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + return importTables(config, infile, chipTables); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Functions for warp stage +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool definewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + pxAddLabelSearchArgs(config, where, "-warp_label", "warpRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-bg_label", "chipBackgroundRun.label", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // Get warp runs to promote to warpBackgroundRun + + psString query = pxDataGet("bgtool_definewarp.sql"); // Query to execute + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!rerun) { + psStringAppend(&query, "\nAND warpBackgroundRun.warp_bg_id IS NULL"); + if (label) { + psStringAppend(&query, + "\nAND (warpBackgroundRun.label = '%s'" + " OR warpBackgroundRun.label IS NULL", + label); + } + if (data_group) { + psStringAppend(&query, + "\nAND (warpBackgroundRun.data_group = '%s'" + " OR warpBackgroundRun.data_group IS NULL", + data_group); + } + if (dist_group) { + psStringAppend(&query, + "\nAND (warpBackgroundRun.dist_group = '%s'" + " OR warpBackgroundRun.dist_group IS NULL", + dist_group); + } + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); // Matching rows + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id"); + + warpRunRow *row = warpRunObjectFromMetadata(md); + if (!row) { + psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!warpBackgroundRunInsert(config->dbh, 0, row->warp_id, chip_bg_id, "new", + workdir ? workdir : row->workdir, + label ? label : row->label, + data_group ? data_group : row->data_group, + dist_group ? dist_group : row->dist_group, + reduction ? reduction : row->reduction, + note ? note : row->note, + NULL, 0)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(row); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + return true; +} + +static bool updatewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpBackgroundRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "warpBackgroundRun.data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "warpBackgroundRun.dist_group","LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpBackgroundRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "warpBackgroundRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "warpBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE warpBackgroundRun JOIN warpBackgroundSkyfile USING(warp_bg_id)"); + bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id", + "warpBackgroundSkyfile", true); + + psFree(query); + psFree(where); + + return result; +} + +static bool towarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpBackgroundRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("bgtool_towarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString whereStr = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereStr, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereStr)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(whereStr); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool warpinputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_warpinputs.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules); + ver_code = pxMergeCodeVersions(ver_code, ver_pswarp); + ver_code = pxMergeCodeVersions(ver_code, ver_ppstats); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script, + hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, + maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool warpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-data_group", "warpBackgroundRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, where, "-dist_group", "warpBackgroundRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_warp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString magicWhere = NULL; + if (!pxmagicAddWhere(config, &magicWhere, "warpBackgroundSkyfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else if (!all && !magicWhere) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + if (magicWhere) { + psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere); + } + psFree(magicWhere); + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + + return true; +} + + +static bool advancewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString select = pxDataGet("bgtool_advancewarp.sql"); + if (!select) { + psError(psErrorCodeLast(), false, "failed to retrieve SQL statement"); + return false; + } + + psString selectWhere = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&selectWhere, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&select, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(select); + psFree(selectWhere); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(select); + psFree(selectWhere); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = true; // Status of MD lookup + psS64 warp_bg_id = psMetadataLookupS64(&status, row, "warp_bg_id"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for warp_bg_id"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE warpBackgroundRun " + "SET state = 'full', magicked = %" PRId64 " " + " WHERE warp_bg_id = %" PRId64, + magicked, warp_bg_id)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool revertwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpBackgroundSkyfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psString query = pxDataGet("bgtool_revertwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d warpBackgroundSkyfiles", numDeleted); + + return true; +} + +static bool tocleanwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); + + psString query = pxDataGet("bgtool_tocleanwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool cleanedwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + if (!pxIsValidCleanedState(state)) { + psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); + return false; + } + + psString query = pxDataGet("bgtool_cleanedwarp.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + psFree(query); + + return true; +} + +static bool exportwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); + + bool status = exportTables(config, outfile, warpTables, where, clean); + + psFree(where); + return status; +} + +static bool importwarpMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + return importTables(config, infile, warpTables); +} Index: trunk/ippTools/src/bgtool.h =================================================================== --- trunk/ippTools/src/bgtool.h (revision 28486) +++ trunk/ippTools/src/bgtool.h (revision 28486) @@ -0,0 +1,59 @@ +/* + * bgtool.h + * + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef BGTOOL_H +#define BGTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + BGTOOL_MODE_NONE = 0x0, + // Chip stage + BGTOOL_MODE_DEFINECHIP, + BGTOOL_MODE_UPDATECHIP, + BGTOOL_MODE_TOCHIP, + BGTOOL_MODE_CHIPINPUTS, + BGTOOL_MODE_ADDCHIP, + BGTOOL_MODE_CHIP, + BGTOOL_MODE_ADVANCECHIP, + BGTOOL_MODE_REVERTCHIP, + // Warp stage + BGTOOL_MODE_DEFINEWARP, + BGTOOL_MODE_UPDATEWARP, + BGTOOL_MODE_TOWARP, + BGTOOL_MODE_WARPINPUTS, + BGTOOL_MODE_ADDWARP, + BGTOOL_MODE_WARP, + BGTOOL_MODE_ADVANCEWARP, + BGTOOL_MODE_REVERTWARP, + // Cleanups + BGTOOL_MODE_TOCLEANCHIP, + BGTOOL_MODE_CLEANEDCHIP, + BGTOOL_MODE_TOCLEANWARP, + BGTOOL_MODE_CLEANEDWARP, + // Exporting + BGTOOL_MODE_EXPORTCHIP, + BGTOOL_MODE_IMPORTCHIP, + BGTOOL_MODE_EXPORTWARP, + BGTOOL_MODE_IMPORTWARP, +} bgtoolMode; + +pxConfig *bgtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // BGTOOL_H Index: trunk/ippTools/src/bgtoolConfig.c =================================================================== --- trunk/ippTools/src/bgtoolConfig.c (revision 28486) +++ trunk/ippTools/src/bgtoolConfig.c (revision 28486) @@ -0,0 +1,420 @@ +/* + * bgtoolConfig.c + * + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "bgtool.h" + +pxConfig *bgtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + // -definechip + psMetadata *definechipArgs = psMetadataAlloc(); + psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-rerun", 0, "re-run data?", false); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + + // -updatechip + psMetadata *updatechipArgs = psMetadataAlloc(); + psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL); + psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_begin", 0, "search by registration time (>=)", NULL); + psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL); + psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -tochip + psMetadata *tochipArgs = psMetadataAlloc(); + psMetadataAddS64(tochipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(tochipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(tochipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tochipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -chipinputs + psMetadata *chipinputsArgs = psMetadataAlloc(); + psMetadataAddS64(chipinputsArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(chipinputsArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0); + psMetadataAddU64(chipinputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(chipinputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addchip + psMetadata *addchipArgs = psMetadataAlloc(); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "define chip_bg_id (required)", 0); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id (required)", NULL); + + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location (required)", NULL); + psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddS16(addchipArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddS16(addchipArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_ppbackground", 0, "define ppBackground version", NULL); + psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddF64(addchipArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addchipArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddS32(addchipArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -chip + psMetadata *chipArgs = psMetadataAlloc(); + psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddS16(chipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL); + psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by dist_group", NULL); + pxmagicAddArguments(chipArgs); + psMetadataAddBool(chipArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(chipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(chipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -advancechip + psMetadata *advancechipArgs = psMetadataAlloc(); + psMetadataAddS64(advancechipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(advancechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL); + psMetadataAddU64(advancechipArgs, PS_LIST_TAIL, "-limit", 0, "search limit", 0); + + // -revertchip + psMetadata *revertchipArgs = psMetadataAlloc(); + psMetadataAddS64(revertchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(revertchipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL); + psMetadataAddStr(revertchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddS16(revertchipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertchipArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -tocleanchip + psMetadata *tocleanchipArgs = psMetadataAlloc(); + psMetadataAddStr(tocleanchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(tocleanchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(tocleanchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -cleanedchip + psMetadata *cleanedchipArgs = psMetadataAlloc(); + psMetadataAddS64(cleanedchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); + psMetadataAddStr(cleanedchipArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + + // -exportchip + psMetadata *exportchipArgs = psMetadataAlloc(); + psMetadataAddS64(exportchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "export this chip_bg_id (required)", 0); + psMetadataAddStr(exportchipArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddBool(exportchipArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importchip + psMetadata *importchipArgs = psMetadataAlloc(); + psMetadataAddStr(importchipArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + + // -definewarp + psMetadata *definewarpArgs = psMetadataAlloc(); + psMetadataAddS64(definewarpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(definewarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); + psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN); + psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); + psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-bg_label", PS_META_DUPLICATE_OK, "search on warpBackgroundRun label", NULL); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + + psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + + // -updatewarp + psMetadata *updatewarpArgs = psMetadataAlloc(); + psMetadataAddS64(updatewarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL); + psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_begin", 0, "search by registration time (>=)", NULL); + psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL); + psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -towarp + psMetadata *towarpArgs = psMetadataAlloc(); + psMetadataAddS64(towarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(towarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddU64(towarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(towarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -warpinputs + psMetadata *warpinputsArgs = psMetadataAlloc(); + psMetadataAddS64(warpinputsArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(warpinputsArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", 0); + psMetadataAddU64(warpinputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(warpinputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -addwarp + psMetadata *addwarpArgs = psMetadataAlloc(); + psMetadataAddS64(addwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "define warp_bg_id (required)", 0); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell_id (required)", NULL); + + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0); + psMetadataAddS64(addwarpArgs, PS_LIST_TAIL, "-set_magicked", 0, "define if this skycell has been magicked", 0); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0); + psMetadataAddS16(addwarpArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + psMetadataAddS16(addwarpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_pswarp", 0, "define pswarp version", NULL); + psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddF64(addwarpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); + psMetadataAddF64(addwarpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); + psMetadataAddS32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN); + psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN); + + // -warp + psMetadata *warpArgs = psMetadataAlloc(); + psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddS16(warpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL); + psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by dist_group", NULL); + pxmagicAddArguments(warpArgs); + psMetadataAddBool(warpArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(warpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(warpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -advancewarp + psMetadata *advancewarpArgs = psMetadataAlloc(); + psMetadataAddS64(advancewarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(advancewarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL); + psMetadataAddU64(advancewarpArgs, PS_LIST_TAIL, "-limit", 0, "search limit", 0); + + // -revertwarp + psMetadata *revertwarpArgs = psMetadataAlloc(); + psMetadataAddS64(revertwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(revertwarpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); + psMetadataAddStr(revertwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddS16(revertwarpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertwarpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -tocleanwarp + psMetadata *tocleanwarpArgs = psMetadataAlloc(); + psMetadataAddStr(tocleanwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(tocleanwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(tocleanwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -cleanedwarp + psMetadata *cleanedwarpArgs = psMetadataAlloc(); + psMetadataAddS64(cleanedwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddStr(cleanedwarpArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL); + + // -exportwarp + psMetadata *exportwarpArgs = psMetadataAlloc(); + psMetadataAddS64(exportwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "export this warp_bg_id (required)", 0); + psMetadataAddStr(exportwarpArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddBool(exportwarpArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false); + + // -importwarp + psMetadata *importwarpArgs = psMetadataAlloc(); + psMetadataAddStr(importwarpArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + + psFree(now); + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definechip", "", BGTOOL_MODE_DEFINECHIP, definechipArgs); + PXOPT_ADD_MODE("-updatechip", "", BGTOOL_MODE_UPDATECHIP, updatechipArgs); + PXOPT_ADD_MODE("-tochip", "", BGTOOL_MODE_TOCHIP, tochipArgs); + PXOPT_ADD_MODE("-chipinputs", "", BGTOOL_MODE_CHIPINPUTS, chipinputsArgs); + PXOPT_ADD_MODE("-addchip", "", BGTOOL_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-chip", "", BGTOOL_MODE_CHIP, chipArgs); + PXOPT_ADD_MODE("-advancechip", "", BGTOOL_MODE_ADVANCECHIP, advancechipArgs); + PXOPT_ADD_MODE("-revertchip", "", BGTOOL_MODE_REVERTCHIP, revertchipArgs); + PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs); + PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs); + PXOPT_ADD_MODE("-exportchip", "", BGTOOL_MODE_EXPORTCHIP, exportchipArgs); + PXOPT_ADD_MODE("-importchip", "", BGTOOL_MODE_IMPORTCHIP, importchipArgs); + + PXOPT_ADD_MODE("-definewarp", "", BGTOOL_MODE_DEFINEWARP, definewarpArgs); + PXOPT_ADD_MODE("-updatewarp", "", BGTOOL_MODE_UPDATEWARP, updatewarpArgs); + PXOPT_ADD_MODE("-towarp", "", BGTOOL_MODE_TOWARP, towarpArgs); + PXOPT_ADD_MODE("-warpinputs", "", BGTOOL_MODE_WARPINPUTS, warpinputsArgs); + PXOPT_ADD_MODE("-addwarp", "", BGTOOL_MODE_ADDWARP, addwarpArgs); + PXOPT_ADD_MODE("-warp", "", BGTOOL_MODE_WARP, warpArgs); + PXOPT_ADD_MODE("-advancewarp", "", BGTOOL_MODE_ADVANCEWARP, advancewarpArgs); + PXOPT_ADD_MODE("-revertwarp", "", BGTOOL_MODE_REVERTWARP, revertwarpArgs); + PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs); + PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs); + PXOPT_ADD_MODE("-exportwarp", "", BGTOOL_MODE_EXPORTWARP, exportwarpArgs); + PXOPT_ADD_MODE("-importwarp", "", BGTOOL_MODE_IMPORTWARP, importwarpArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + // do this last so we don't setup a connection before CLI options are + // validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: trunk/ippTools/src/pxtools.c =================================================================== --- trunk/ippTools/src/pxtools.c (revision 28453) +++ trunk/ippTools/src/pxtools.c (revision 28486) @@ -48,4 +48,14 @@ } +bool pxIsValidCleanedState(const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + if (!strcmp(state, "cleaned")) return true; + if (!strcmp(state, "purged")) return true; + if (!strcmp(state, "scrubbed")) return true; + return false; +} + psString pxMergeCodeVersions(psString version1, psString version2) { @@ -211,9 +221,9 @@ // change the value for tableName.columName from 'full' to 'cleaned' if necessary -bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows) +bool pxSetStateCleaned(const char *tableName, const char *columnName, psArray *rows) { for (long i = 0; i < psArrayLength(rows); i++) { psMetadata *row = rows->data[i]; - psString state = psMetadataLookupStr(NULL, row, columnName); + const char *state = psMetadataLookupStr(NULL, row, columnName); if (!state) { psError(PS_ERR_PROGRAMMING, false, "%s not found in row %ld of table %s", Index: trunk/ippTools/src/pxtools.h =================================================================== --- trunk/ippTools/src/pxtools.h (revision 28453) +++ trunk/ippTools/src/pxtools.h (revision 28486) @@ -53,20 +53,21 @@ bool pxIsValidState(const char *state); +bool pxIsValidCleanedState(const char *state); psString pxMergeCodeVersions(psString version1, psString version2); bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver, - psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, - psF32 *maskfrac_magic, psF32 *maskfrac_advisory); + psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic, + psF32 *maskfrac_magic, psF32 *maskfrac_advisory); bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psString software_ver); + psString software_ver); bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, - psF32 maskfrac_magic, psF32 maskfrac_advisory); + psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic, + psF32 maskfrac_magic, psF32 maskfrac_advisory); bool pxCamSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id, - psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic, - psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory, - psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic, - psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory); - -bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows); + psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic, + psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory, + psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic, + psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory); + +bool pxSetStateCleaned(const char *tableName, const char *columnName, psArray *rows); bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op); Index: trunk/ippconfig/recipes/Makefile.am =================================================================== --- trunk/ippconfig/recipes/Makefile.am (revision 28453) +++ trunk/ippconfig/recipes/Makefile.am (revision 28486) @@ -30,5 +30,6 @@ ppSkycell.config \ ppVizPSF.mdc \ - nightly_science.config + nightly_science.config \ + ppBackground.mdc install_DATA = $(install_files) Index: trunk/ippconfig/recipes/ppBackground.mdc =================================================================== --- trunk/ippconfig/recipes/ppBackground.mdc (revision 28486) +++ trunk/ippconfig/recipes/ppBackground.mdc (revision 28486) @@ -0,0 +1,5 @@ +# Recipe options for ppBackground + + +BACKGROUND METADATA +END Index: trunk/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- trunk/ippconfig/recipes/ppStatsFromMetadata.config (revision 28453) +++ trunk/ippconfig/recipes/ppStatsFromMetadata.config (revision 28486) @@ -326,4 +326,27 @@ END + +BACKGROUND_WARP METADATA + ENTRY MULTI + TYPE VAL KEYWORD TYPE STATISTIC FLAG + ENTRY VAL ROBUST_MEDIAN F32 ROBUST_MEDIAN -bg + ENTRY VAL ROBUST_STDEV F32 RMS -bg_stdev + ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + ENTRY VAL WARP_V STR CONSTANT -ver_pswarp + + # Mask stats values + ENTRY VAL MASKFRAC_NPIX S32 CONSTANT -maskfrac_npix + ENTRY VAL MASKFRAC_STATIC F32 CONSTANT -maskfrac_static + ENTRY VAL MASKFRAC_DYNAMIC F32 CONSTANT -maskfrac_dynamic + ENTRY VAL MASKFRAC_MAGIC F32 CONSTANT -maskfrac_magic + ENTRY VAL MASKFRAC_ADVISORY F32 CONSTANT -maskfrac_advisory +END + + STACK_SKYCELL METADATA ENTRY MULTI Index: trunk/ippconfig/recipes/reductionClasses.mdc =================================================================== --- trunk/ippconfig/recipes/reductionClasses.mdc (revision 28453) +++ trunk/ippconfig/recipes/reductionClasses.mdc (revision 28486) @@ -158,4 +158,7 @@ PSASTRO STR DEFAULT_RECIPE STACKPHOT STR STACKPHOT + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + END Index: trunk/ippconfig/system.config =================================================================== --- trunk/ippconfig/system.config (revision 28453) +++ trunk/ippconfig/system.config (revision 28486) @@ -59,3 +59,4 @@ PPVIZPSF STR recipes/ppVizPSF.mdc # PSF visualisation NIGHTLY_SCIENCE STR recipes/nightly_science.config # Nightly Science + PPBACKGROUND STR recipes/ppBackground.mdc # Background restoration END Index: trunk/psconfig/tagsets/ipp-2.9.dist =================================================================== --- trunk/psconfig/tagsets/ipp-2.9.dist (revision 28453) +++ trunk/psconfig/tagsets/ipp-2.9.dist (revision 28486) @@ -73,4 +73,5 @@ YYYYY ppTranslate ipp-2-9 -0 YYYYY ppViz ipp-2-9 -0 + YYYYY ppBackground ipp-2-9 -0 YYYYY ppSkycell ipp-2-9 -0 Index: trunk/pstamp/Makefile.am =================================================================== --- trunk/pstamp/Makefile.am (revision 28453) +++ trunk/pstamp/Makefile.am (revision 28486) @@ -1,3 +1,9 @@ SUBDIRS = src scripts +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA= ppstamp.pc + +EXTRA_DIST = \ + ppstamp.pc.in + CLEANFILES = *~ core core.* Index: trunk/pstamp/configure.ac =================================================================== --- trunk/pstamp/configure.ac (revision 28453) +++ trunk/pstamp/configure.ac (revision 28486) @@ -33,4 +33,6 @@ IPP_STDOPTS CFLAGS="${CFLAGS} -Wall -Werror" +PPSTAMP_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=}" +PPSTAMP_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=}" echo "PPSTAMP_CFLAGS: $PPSTAMP_CFLAGS" echo "PPSTAMP_LIBS: $PPSTAMP_LIBS" @@ -43,4 +45,5 @@ src/Makefile scripts/Makefile + ppstamp.pc ]) Index: trunk/pstamp/ppstamp.pc.in =================================================================== --- trunk/pstamp/ppstamp.pc.in (revision 28486) +++ trunk/pstamp/ppstamp.pc.in (revision 28486) @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@/@PACKAGE_NAME@ + +Name: @PACKAGE_NAME@ +Description: Pan-STARRS Postage Stamper +Version: @VERSION@ +Requires: pslib psmodules +Libs: -L${libdir} @PPSTAMP_LIBS@ +Cflags: -I${includedir} @PPSTAMP_CFLAGS@