Index: /branches/eam_branches/ipp-20130307/DataStoreServer/web/php/getsmf.php =================================================================== --- /branches/eam_branches/ipp-20130307/DataStoreServer/web/php/getsmf.php (revision 35350) +++ /branches/eam_branches/ipp-20130307/DataStoreServer/web/php/getsmf.php (revision 35350) @@ -0,0 +1,133 @@ +$command\n
"; + } + $command = escapeshellcmd($command); + $output = array(); + + exec($command, $output, $command_status); + + if ($command_status == 0) { + // we only expect one line of output + $len = count($output); + if ($len == 1) { + list($filename, $pathname) = explode(" ", $output[0]); + if ($filename && $pathname) { + if (!$debug) { + $gotFile = 1; + } else { + echo "$filename $pathname\n"; + echo "$command\n"; + } + } + } else { + echo "unexpected output from $command: $output[0] $output[1]\n"; + } + } else { + if ($debug) { + echo "command failed $command_status\n"; + } + } +} else { +} + +if ($gotFile) { + if (!$rvar_list) { + // All systems are go. Time to write the output. + // First set up the header + header('Content-type: application/fits'); + header("Content-Disposition: attachment; filename=\"$filename\""); + $filesize = filesize($pathname); + header("Content-Length: $filesize"); + header('Expires: now'); + + // copy the contents of the file to the stream + readfile($pathname); + } else { + echo "smf file name is $filename
\nfile is $pathname\n"; + } + +} else { + // XXX: Figure out how to stop wget from redirecting these error + // messages to the nasty filename + $error_string="Could not find smf"; + if ($rvar_cam_id) { + $error_string .= " for cam_id: $rvar_cam_id"; + } elseif ($rvar_exp_name) { + $error_string .= " for exposure: $rvar_exp_name"; + } else { + $error_string .= ". No cam_id or exp_name provided"; + } + + echo "$error_string.\n"; +} + +function getVar($var) { + if ($_SERVER['REQUEST_METHOD'] == 'POST') { + $rvar = $_POST[$var]; + } else { + $rvar = $_GET[$var]; + } + $rvar = stripslashes($rvar); + $rvar = htmlentities($rvar); + $rvar = strip_tags($rvar); + return $rvar; +} + + +if ($list) { + // print lots of information + // phpinfo(-1); + + // print the most useful variables + // phpinfo(32); +} + +?> Index: /branches/eam_branches/ipp-20130307/Ohana/src/delstar/src/delstar.c =================================================================== --- /branches/eam_branches/ipp-20130307/Ohana/src/delstar/src/delstar.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/Ohana/src/delstar/src/delstar.c (revision 35350) @@ -21,5 +21,6 @@ switch (MODE) { case MODE_DUP_IMAGES: - delete_duplicate_images (0, NULL, &db); + if (!delete_duplicate_images (0, NULL, &db)) exit (1); + exit (0); break; case MODE_IMAGEFILE: Index: /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35349) +++ /branches/eam_branches/ipp-20130307/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 35350) @@ -223,5 +223,14 @@ my $choose_components = 0; if ($stage eq "raw") { - $command = "$regtool -processedimfile -dbname $imagedb"; + if ($component or $use_imfile_id or !$skycenter) { + $command = "$regtool -processedimfile -dbname $imagedb"; + if ($component and $component ne 'all') { + $class_id = $component; + $component_args = " -class_id $class_id"; + } + } else { + $command = "$regtool -processedexp -dbname $imagedb"; + $choose_components = 1; + } # XXX: for now restrict lookups to type object # are stamps of detrend exposures interesting? Index: /branches/eam_branches/ipp-20130307/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20130307/dbconfig/changes.txt (revision 35349) +++ /branches/eam_branches/ipp-20130307/dbconfig/changes.txt (revision 35350) @@ -2388,6 +2388,9 @@ CREATE TABLE lapGroup ( seq_id BIGINT, + tess_id VARCHAR(64), projection_cell VARCHAR(64), state VARCHAR(16), + label VARCHAR(654), + registered TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP fault SMALLINT, PRIMARY KEY(seq_id, projection_cell), @@ -2399,2 +2402,12 @@ UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP(); + +-- new Postage Stamp Request columns +-- Note: These do not aply to the gpc1 database because the pstamp tables were +-- deleted from that database. +ALTER TABLE pstampRequest ADD COLUMN username VARCHAR(255) AFTER outdir; +ALTER TABLE pstampRequest ADD COLUMN proj_id BIGINT AFTER username; +ALTER TABLE pstampRequest ADD COLUMN registered DATETIME AFTER proj_id; + +-- default to the gpc1 project +UPDATE pstampRequest SET proj_id = 1; Index: /branches/eam_branches/ipp-20130307/dbconfig/lap.md =================================================================== --- /branches/eam_branches/ipp-20130307/dbconfig/lap.md (revision 35349) +++ /branches/eam_branches/ipp-20130307/dbconfig/lap.md (revision 35350) @@ -33,6 +33,9 @@ lapGroup METADATA seq_id S64 0 + tess_id STR 64 projection_cell STR 64 state STR 64 + label STR 64 + registered TAI NULL fault S16 0 end Index: /branches/eam_branches/ipp-20130307/dbconfig/pstamp.md =================================================================== --- /branches/eam_branches/ipp-20130307/dbconfig/pstamp.md (revision 35349) +++ /branches/eam_branches/ipp-20130307/dbconfig/pstamp.md (revision 35350) @@ -32,4 +32,7 @@ uri STR 255 outdir STR 255 + username STR 255 + proj_id S64 0 + registered UTC 0001-01-01T00:00:00Z timestamp UTC 0001-01-01T00:00:00Z fault S32 0 Index: /branches/eam_branches/ipp-20130307/ippMonitor/def/distRun.d =================================================================== --- /branches/eam_branches/ipp-20130307/ippMonitor/def/distRun.d (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippMonitor/def/distRun.d (revision 35350) @@ -8,4 +8,5 @@ WHERE distRun.dist_id = distComponent.dist_id UNRESTRICTED AND 0=1 +RESTRICT_COUNT AND 0>1 # field size format name show link to extras Index: /branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php =================================================================== --- /branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippMonitor/raw/heathers_plot.php (revision 35350) @@ -10,4 +10,5 @@ menu('ipp.czar.dat', 'useful tools for the czar', 'ipp.css', $ID['link'], $ID['proj']); +echo "These images are generated every day at 7:00 HST (17:00 GMT)\n"; // document body echo "\n"; Index: /branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.imfiles.dat (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.imfiles.dat (revision 35350) @@ -24,5 +24,4 @@ menutop | menutop | link | czartool - ipptopsps | czartool_ipptopsps.php menutop | menutop | link | mask stats | maskStats.php -menutop | menutop | link | Heather's Plots | heathers_plot.php menutop | menutop | plain |   | Index: /branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.plots.dat =================================================================== --- /branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.plots.dat (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippMonitor/raw/ipp.plots.dat (revision 35350) @@ -8,4 +8,5 @@ menulink | menuselect | link | Simple plot - staticsky | simplePlotStaticskyImage.php menulink | menuselect | link | Simple plot - skycal | simplePlotSkycalImage.php +menulink | menuselect | link | Heather's Plots | heathers_plot.php menutop | menutop | plain |   | Index: /branches/eam_branches/ipp-20130307/ippScripts/Build.PL =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/Build.PL (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippScripts/Build.PL (revision 35350) @@ -130,4 +130,5 @@ scripts/regenerate_background.pl scripts/relgroup_exp_list.pl + scripts/queuestaticsky.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /branches/eam_branches/ipp-20130307/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/scripts/detrend_process_imfile.pl (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippScripts/scripts/detrend_process_imfile.pl (revision 35350) @@ -129,5 +129,5 @@ my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); unless ($repair_success) { - &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id $PS_EXIT_SYS_ERROR); + &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR); } Index: /branches/eam_branches/ipp-20130307/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/scripts/diff_skycell.pl (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippScripts/scripts/diff_skycell.pl (revision 35350) @@ -113,16 +113,19 @@ my ($inputMagic, $templateMagic); # Are the inputs been magicked? my ($saveInConv, $saveRefConv); # Save the input or reference convolved images? -# Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries. -if ((${ $files }[0]->{warp_id} == 0)&& - (${ $files }[1]->{warp_id} == 0)) { - # Both are zero, so stack stack diff; - unless ($use_convolved) { - $use_convolved = 0; - } -} -else { - # We're in some sort of warp stack or warp warp (don't care about the last one.) - $use_convolved = 1; -} +# CZW 2013-03-21: We only want to use unconvolved inputs. Ever, if I understand correctly. +unless ($use_convolved) { + $use_convolved = 0; +} + +# # Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries. +# if ((${ $files }[0]->{warp_id} != 0)&& +# (${ $files }[1]->{warp_id} != 0)) { +# # Both are zero, so stack stack diff; +# } +# else { + +# # We're in some sort of warp stack or warp warp (don't care about the last one.) +# $use_convolved = 1; ## This is a hack to do a test for mops, and should not be committed. +# } # ppSub does (input) - (template) after PSF-match convolution. Index: /branches/eam_branches/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippScripts/scripts/mergedvodb_merge.pl (revision 35350) @@ -156,5 +156,5 @@ $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; } else { - $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + $merge_command = "$dvomerge -parallel $minidvodb_path into $mergedvodb_path"; } print "\n$merge_command\n"; Index: /branches/eam_branches/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippScripts/scripts/minidvodb_premerge.pl (revision 35350) @@ -16,4 +16,5 @@ my $dtime_relphot; my $dtime_script; +my $dtime_delstar; use vars qw( $VERSION ); @@ -33,7 +34,9 @@ my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $delstar = can_run('delstar') or (warn "Can't find delstar" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); + my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); @@ -80,5 +83,26 @@ unless ($no_op) { - #this is chopped into several parts: addstar, relphot + #this is chopped into several parts: delstar,addstar, relphot, relastro, dvoverify + #delstar - first step: are there duplicates, if so remove them + { + my $command = "$delstar -update -dup-images "; + #$command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + my $mjd_delstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform delstar: $error_code", $minidvodb_id, $error_code); + } + print $full_buf; + $dtime_delstar = 86400.0*(DateTime->now->mjd - $mjd_delstar_start); + # MJD of starting script + print "delstar time $dtime_delstar\n"; + } + + + #addstar { @@ -94,6 +118,6 @@ &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code); } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; - # MJD of starting script + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); + # MJD of starting script $dtime_resort = $dtime_addstar; print "addstar -resort time $dtime_addstar\n"; Index: /branches/eam_branches/ipp-20130307/ippScripts/scripts/queuestaticsky.pl =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/scripts/queuestaticsky.pl (revision 35350) +++ /branches/eam_branches/ipp-20130307/ippScripts/scripts/queuestaticsky.pl (revision 35350) @@ -0,0 +1,273 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use Carp; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use POSIX qw(strftime); +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +my ($seq_id, $tess_id, $projection_cell, $label, $dist_group, $dbname, $pretend, $simple, $verbose, $no_update); + +GetOptions( + 'seq_id=s' => \$seq_id, + 'tess_id=s' => \$tess_id, + 'projection_cell=s' => \$projection_cell, + 'label=s' => \$label, + 'dist_group=s' => \$dist_group, + 'dbname=s' => \$dbname, + 'pretend' => \$pretend, + 'simple' => \$simple, + 'no-update' => \$no_update, + 'verbose|v' => \$verbose, +) or pod2usage(2); + +unless (defined $seq_id and defined $tess_id and defined $projection_cell and defined $label and defined $dist_group) { + warn ("label, seq_id, tess_id, and projection_cell are required\n"); + pod2usage(2); +}; + + +my $missing_tools; +my $laptool = can_run('laptool') or (warn "Can't find laptool" and $missing_tools = 1); +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $PS_EXIT_CONFIG_ERROR ); + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# XXX get from site.config +my $workdirBase = "neb://\@HOST\@.0/gpc1"; + +my @filters; +{ + my $command = "$laptool -filtersforgroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => 0); + unless ($success) { + $error_code = $error_code >> 8; + my_die("failed to run $command $error_code\n", $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR); + my $list = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $PS_EXIT_PROG_ERROR); + + foreach my $entry (@$list) { + push @filters, $entry->{filter}; + } +} + +my $nFilters = scalar @filters; +print STDERR "lapGroup $seq_id $tess_id $projection_cell has $nFilters filters\n"; + +&my_die("No filters found for $seq_id $tess_id $projection_cell", $PS_EXIT_PROG_ERROR) unless $nFilters; +&my_die("Unexpected number of filters found for $seq_id $tess_id $projection_cell", $PS_EXIT_PROG_ERROR) unless $nFilters <= 5; + +my $datestr = strftime "%Y/%m/%d", gmtime; + +my $staticsky_command="staticskytool -definebyquery" + . " -select_label $label" + . " -select_tess_id $tess_id" + . " -select_skycell_id $projection_cell%" + . " -set_workdir $workdirBase/$label/$datestr" + . " -set_label $label" + . " -set_data_group $label" + . " -set_dist_group $dist_group"; + +$staticsky_command .= " -pretend" if $pretend; +$staticsky_command .= " -simple" if $simple; +$staticsky_command .= " -dbname $dbname" if $dbname; + +for (my $num = $nFilters; $num > 0; $num--) { + + # set up the possible combinations of $num filters from @filters + + my $filter_combos = setup_filter_combos($num); + + # printcombos($filter_combos) if $verbose; + + foreach my $filter_combo (@$filter_combos) { + my $command = $staticsky_command; + foreach my $f (@$filter_combo) { + $command .= " -select_filter $f"; + } + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + my_die("failed to run $command $error_code\n", $error_code); + } + } +} +{ + my $command = "$laptool -updategroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -set_state full"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + carp("failed to run $command $error_code\n", $error_code); + exit $error_code; + } + } else { + print STDERR "Skipping $command\n"; + } +} + +exit 0; + +sub my_die +{ + my $msg = shift; + my $exit_code = shift; + + $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code; + + carp($msg); + + my $command = "$laptool -updategroup"; + $command .= " -seq_id $seq_id"; + $command .= " -tess_id $tess_id"; + $command .= " -projection_cell $projection_cell"; + $command .= " -set_fault $exit_code"; + $command .= " -dbname $dbname" if defined $dbname; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = $error_code >> 8; + carp("failed to run $command $error_code\n", $error_code); + exit $error_code; + } + } else { + print STDERR "Skipping $command\n"; + } + exit $exit_code; +} + + +# The rest of this file contains some subroutines for setting up the filter combinations + + +sub setup_filter_combos { + my $combos = shift; + + my_die( "invalid combos arg: $combos", $PS_EXIT_PROG_ERROR) + unless defined $combos and ($combos eq "5" or $combos eq "4" or $combos eq "3" or $combos eq "2" or $combos eq "1"); + + if ($combos eq 5) { + my @ary; + push @ary, \@filters; + return \@ary; + } elsif ($combos eq 4) { + return makecombo4(); + } elsif ($combos eq 3) { + return makecombo3(); + } elsif ($combos eq 2) { + return makecombo2(); + } elsif ($combos eq 1) { + my @ary; + foreach my $f (@filters) { + my @ary2 = ($f); + push @ary, \@ary2 + } + return \@ary; + } else { + die "how did we get here?"; + } +} + + + +sub printcombos { + my $combos = shift; + foreach my $c (@$combos) { + my $str; + foreach my $f (@$c) { + $str .= " -select_filter $f"; + } + print "$str\n"; + } +} + + +sub makecombo4 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + for (my $k = $j + 1; $k < $numFilters; $k++) { + for (my $l = $k + 1; $l < $numFilters; $l++) { + my @combo = ($filters[$i], $filters[$j], $filters[$k], $filters[$l]); + push @combos, \@combo; + } + } + } + } + return \@combos; +} + +sub makecombo3 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + for (my $k = $j + 1; $k < $numFilters; $k++) { + my @combo = ($filters[$i], $filters[$j], $filters[$k]); + push @combos, \@combo; + } + } + } + return \@combos; +} + +sub makecombo2 { + my $numFilters = scalar @filters; + my @combos; + + for (my $i = 0; $i < $numFilters; $i++) { + for (my $j = $i + 1; $j < $numFilters; $j++) { + my @combo = ($filters[$i], $filters[$j]); + push @combos, \@combo; + } + } + return \@combos; +} + + +__END__ + +=pod + +=head1 NAME + +queuesskylap - queue LAP staticsky runs + +=head1 SYNOPSIS + + XXX: pod TODO + + queuesskylap --ra_min --ra_max --dec_min --dec_max [--go] + + Index: /branches/eam_branches/ipp-20130307/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20130307/ippScripts/scripts/warp_skycell.pl (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippScripts/scripts/warp_skycell.pl (revision 35350) @@ -247,5 +247,5 @@ if (($doBackground)) { # &&($imfile->{background_model} == 1) - if ($imfile->{cam_background_model}) { + if (($imfile->{cam_background_model})&&($imfile->{cam_background_model} != 32767)) { $bkg = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{cam_path_base}, $imfile->{class_id}); } Index: /branches/eam_branches/ipp-20130307/ippTasks/addstar.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/addstar.pro (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTasks/addstar.pro (revision 35350) @@ -6,4 +6,9 @@ check.globals + +if ($?ADDSTAGES:n == 0) set ADDSTAGES:n = 0 + + + book init addPendingExp @@ -17,14 +22,5 @@ macro addstar.on - task addstar.exp.load.stack - active true - end - task addstar.exp.load.cam - active true - end - task addstar.exp.load.staticsky - active true - end - task addstar.exp.load.skycal + task addstar.exp.load active true end @@ -35,14 +31,5 @@ macro addstar.off - task addstar.exp.load.stack - active false - end - task addstar.exp.load.cam - active false - end - task addstar.exp.load.staticsky - active false - end - task addstar.exp.load.skycal + task addstar.exp.load active false end @@ -83,6 +70,69 @@ +#addstar stages + +macro add.addstages + if ($0 != 2) + echo "USAGE: add.addstages (addstages)" + break + end + if ($?ADDSTAGES:n == 0) + list ADDSTAGES -add $1 + return + end + + local found + $found = 0 + for i 0 $ADDSTAGES:n + if ($ADDSTAGES:$i == $1) + $found = 1 + echo "$ADDSTAGES:$i set" + last + end + end + + if ($found == 0) + list ADDSTAGES -add $1 + end +end + +macro del.addstages + if ($0 != 2) + echo "USAGE: del.addstages (addstages)" + break + end + if ($?ADDSTAGES:n == 0) + return + end + + list ADDSTAGES -del $1 +end + +macro show.addstages + if ($0 != 1) + echo "USAGE: show.addstages" + break + end + if ($?ADDSTAGES:n == 0) + echo "no addstar stages defined" + end + if ($ADDSTAGES:n == 0) + echo "no addstar stages defined" + end + + local i + for i 0 $ADDSTAGES:n + echo $ADDSTAGES:$i + end +end + +macro stuff + echo $LOADEXEC_ADD + echo $LOADPOLL +end + # this variable will cycle through the known database names $addstar_DB = 0 +$addstar_stages_DB = 0 #this may not work for more databases (addstar) will do that later) $addstar_revert_DB_C = 0 @@ -92,6 +142,6 @@ $addstar_revert_DB_SC = 0 -$LOADEXEX_ADD = 300 - +#$LOADEXEC_ADD = 300 +$LOADEXEC_ADD = 10 @@ -99,5 +149,5 @@ # new entries are added to addPendingExp # skip already-present entries -task addstar.exp.load.stack +task addstar.exp.load host local @@ -111,17 +161,24 @@ task.exec - # if ($LABEL:n == 0) break - $run = addtool -pendingexp -stage stack - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_DB - $run = $run -dbname $DB:$addstar_DB - $addstar_DB ++ - if ($addstar_DB >= $DB:n) set addstar_DB = 0 - end + #if no stages defined we can't run + if ($ADDSTAGES:n == 0) echo "what" + # if ($ADDSTAGES:n == 0) break + $run = addtool -pendingexp + + #option $ADDSTAGES:$addstar_stages_DB + $run = $run -stage $ADDSTAGES:$addstar_stages_DB + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_stages_DB ++ + if ($addstar_stages_DB >= $ADDSTAGES:n) + set addstar_stages_DB = 0 + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + #go to next stage. if run out of stages, go to 0 stage and + #cycle through dbname, if run out of dbnames, go to 0 dbname + end add_poll_args run add_poll_labels run + echo $run command $run end @@ -153,168 +210,4 @@ end end - -task addstar.exp.load.cam - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC_ADD - periods -timeout 300 - npending 1 - - stdout NULL - stderr $LOGDIR/addstar.exp.log - - task.exec - # if ($LABEL:n == 0) break - $run = addtool -pendingexp -stage cam - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_DB - $run = $run -dbname $DB:$addstar_DB - $addstar_DB ++ - if ($addstar_DB >= $DB:n) set addstar_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook addPendingExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup addPendingExp - end - - # default exit status - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task addstar.exp.load.staticsky - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC_ADD - periods -timeout 300 - npending 1 - - stdout NULL - stderr $LOGDIR/addstar.exp.log - - task.exec - # if ($LABEL:n == 0) break - $run = addtool -pendingexp -stage staticsky - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_DB - $run = $run -dbname $DB:$addstar_DB - $addstar_DB ++ - if ($addstar_DB >= $DB:n) set addstar_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook addPendingExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup addPendingExp - end - - # default exit status - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task addstar.exp.load.skycal - host local - periods -poll $LOADPOLL - periods -exec $LOADEXEC_ADD - periods -timeout 300 - npending 1 - - stdout NULL - stderr $LOGDIR/addstar.exp.log - - task.exec - # if ($LABEL:n == 0) break - $run = addtool -pendingexp -stage skycal - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_DB - $run = $run -dbname $DB:$addstar_DB - $addstar_DB ++ - if ($addstar_DB >= $DB:n) set addstar_DB = 0 - end - add_poll_args run - add_poll_labels run - command $run - end - - # success - task.exit 0 - # convert 'stdout' to book format - ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT - if ($VERBOSE > 2) - book listbook addPendingExp - end - - # delete existing entries in the appropriate pantaskStates - process_cleanup addPendingExp - end - - # default exit status - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - # run the addstar script on pending exposures Index: /branches/eam_branches/ipp-20130307/ippTasks/lapgroup.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/lapgroup.pro (revision 35350) +++ /branches/eam_branches/ipp-20130307/ippTasks/lapgroup.pro (revision 35350) @@ -0,0 +1,161 @@ +## lapgroup.pro : tasks for lap group management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/lapgroup +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendinglapGroup + +### Database lists +$lapgroup_DB = 0 + +### Check status of lapgroup tasks +macro lapgroup.status + book listbook pendinglapGroup +end + +### Reset lapgroup tasks +macro lapgroup.reset + book init pendinglapGroup +end + +### Turn lapgroup tasks on +macro lapgroup.on + task lapgroup.load + active true + end + task lapgroup.run + active true + end +end + +### Turn lapgroup tasks off +macro lapgroup.off + task lapgroup.load + active false + end + task lapgroup.run + active false + end +end + +### Load jobs for lapGroup +### Tasks are loaded into pendinglapGroup. +task lapgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/lapgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = laptool -pendinggroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$lapgroup_DB + $run = $run -dbname $DB:$lapgroup_DB + $lapgroup_DB ++ + if ($lapgroup_DB >= $DB:n) set lapgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendinglapGroup -key seq_id:tess_id:projection_cell -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendinglapGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendinglapGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task lapgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendinglapGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendinglapGroup (pantaskState == INIT) + book getpage pendinglapGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendinglapGroup $pageName pantaskState RUN + book getword pendinglapGroup $pageName seq_id -var SEQ_ID + book getword pendinglapGroup $pageName tess_id -var TESS_ID + book getword pendinglapGroup $pageName projection_cell -var PROJECTION_CELL + book getword pendinglapGroup $pageName label -var LABEL + book getword pendinglapGroup $pageName dist_group -var DIST_GROUP + book getword pendinglapGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/lapgroup.log + stderr $LOGSUBDIR/lapgroup.log + + host anyhost + + $run = queuestaticsky.pl --seq_id $SEQ_ID --tess_id $TESS_ID --projection_cell $PROJECTION_CELL --label $LABEL --dist_group $DIST_GROUP + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendinglapGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendinglapGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendinglapGroup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/ipp-20130307/ippTasks/minidvodb.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/minidvodb.pro (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTasks/minidvodb.pro (revision 35350) @@ -145,4 +145,5 @@ book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1 book setword MINIDVODB_PREMERGE $1 DVODB $3 + book setword MINIDVODB_PREMERGE $1 CAMERA $5 book setword MINIDVODB_PREMERGE $1 STATE PENDING #active @@ -443,9 +444,10 @@ book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbPreMergelist $pageName camera -var CAMERA book getword minidvodbPreMergelist $pageName state -var STATE stdout $LOGDIR/minidvodb.premerge.run.log stderr $LOGDIR/minidvodb.premerge.run.log - $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + $run = minidvodb_premerge.pl --camera $CAMERA --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID if ($DB:n == 0) Index: /branches/eam_branches/ipp-20130307/ippTasks/release.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/release.pro (revision 35350) +++ /branches/eam_branches/ipp-20130307/ippTasks/release.pro (revision 35350) @@ -0,0 +1,167 @@ +## release.pro : tasks for release management : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/release +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init pendingrelGroup + +### Database lists +$relgroup_DB = 0 + +### Check status of release tasks +macro release.status + book listbook pendingrelGroup +end + +### Reset release tasks +macro release.reset + book init pendingrelgroup +end + +### Turn release tasks on +macro release.on + task relgroup.load + active true + end + task relgroup.run + active true + end +end + +### Turn release tasks off +macro release.off + task relgroup.load + active false + end + task relgroup.run + active false + end +end + +### Load jobs for relGroup +### Tasks are loaded into pendingrelGroup. +task relgroup.load + host local + + periods -poll $LOADPOLL + periods -exec 30 + periods -timeout 60 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/relgroup.log + + task.exec + if ($LABEL:n == 0) break + $run = releasetool -pendingrelgroup + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$relgroup_DB + $run = $run -dbname $DB:$relgroup_DB + $relgroup_DB ++ + if ($relgroup_DB >= $DB:n) set relgroup_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pendingrelGroup -key group_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook pendingrelGroup + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup pendingrelGroup + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task relgroup.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 120 + npending 1 + + task.exec + book npages pendingrelGroup -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in pendingrelGroup (pantaskState == INIT) + book getpage pendingrelGroup 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword pendingrelGroup $pageName pantaskState RUN + book getword pendingrelGroup $pageName group_id -var GROUP_ID + book getword pendingrelGroup $pageName rel_id -var REL_ID + book getword pendingrelGroup $pageName group_type -var GROUP_TYPE + book getword pendingrelGroup $pageName lap_id -var LAP_ID + book getword pendingrelGroup $pageName group_name -var GROUP_NAME + book getword pendingrelGroup $pageName release_name -var RELEASE_NAME + book getword pendingrelGroup $pageName dbname -var DBNAME + + stdout $LOGSUBDIR/relgroup.log + stderr $LOGSUBDIR/relgroup.log + + host anyhost + + $run = relgroup_exp_list.pl --group_id $GROUP_ID --group_type $GROUP_TYPE --release_name $RELEASE_NAME + if ("$GROUP_TYPE" == "lap") + $run = $run --lap_id $LAP_ID + else + $run = $run --group_name $GROUP_NAME + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pendingrelGroup $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword pendingrelGroup $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pendingrelGroup $options:0 pantaskState TIMEOUT + end +end + Index: /branches/eam_branches/ipp-20130307/ippTasks/survey.pro =================================================================== --- /branches/eam_branches/ipp-20130307/ippTasks/survey.pro (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTasks/survey.pro (revision 35350) @@ -18,4 +18,5 @@ book create SURVEY_STATICSKYSINGLE book create SURVEY_SKYCAL + book create SURVEY_LAPGROUP $haveSurveyBooks = TRUE end @@ -33,4 +34,5 @@ $SURVEY_PUBLISH_DB = 0 $SURVEY_SKYCAL_DB = 0 +$SURVEY_LAPGROUP_DB = 0 $SURVEY_STATICSKYSINGLE_DB = 0 $SURVEY_EXEC = 120 @@ -78,4 +80,7 @@ active true end + task survey.lapgroup + active true + end end @@ -118,4 +123,7 @@ end task survey.skycal + active false + end + task survey.lapgroup active false end @@ -494,8 +502,35 @@ macro survey.show.skycal if ($0 != 1) - echo "USAGE: survey.show.skyacl" + echo "USAGE: survey.show.skycal" break end book listbook SURVEY_SKYCAL +end + +macro survey.add.lapgroup + if ($0 != 3) + echo "USAGE: survey.add.lapgroup (label) (seq_id)" + break + end + book newpage SURVEY_LAPGROUP $1 + book setword SURVEY_LAPGROUP $1 LABEL $1 + book setword SURVEY_LAPGROUP $1 SEQ_ID $2 + book setword SURVEY_LAPGROUP $1 STATE PENDING +end + +macro survey.del.lapgroup + if ($0 != 2) + echo "USAGE: survey.del.lapgroup (label)" + break + end + book delpage SURVEY_LAPGROUP $1 +end + +macro survey.show.lapgroup + if ($0 != 1) + echo "USAGE: survey.show.lapgroup" + break + end + book listbook SURVEY_LAPGROUP end @@ -1561,2 +1596,72 @@ end end + +task survey.lapgroup + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.lapgroup.log + stderr $LOGDIR/survey.lapgroup.log + + task.exec + book npages SURVEY_LAPGROUP -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW + if ("$label" == "NULL") + # All labels have been done --- reset +# echo "Resetting labels" + for i 0 $N + book getpage SURVEY_LAPGROUP $i -var label + book setword SURVEY_LAPGROUP $label STATE NEW + end + book getpage SURVEY_LAPGROUP 0 -var label -key STATE NEW + + # Select different database + $SURVEY_LAPGROUP ++ + if ($SURVEY_LAPGROUP >= $DB:n) set SURVEY_LAPGROUP = 0 + end + + book setword SURVEY_LAPGROUP $label STATE DONE + book getword SURVEY_LAPGROUP $label SEQ_ID -var SEQ_ID + + # For now the list of filters + $run = laptool -definegroup -seq_id $SEQ_ID -set_label $label -filter g.00000 -filter r.00000 -filter i.00000 -filter z.00000 -filter y.00000 + + if ($DB:n == 0) + option DEFAULT + else + $run = $run -dbname $DB:$SURVEY_LAPGROUP_DB + option $DB:$SURVEY_LAPGROUP_DB + end + + # echo $run + command $run + end + + # success + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/cleanup.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/cleanup.py (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/cleanup.py (revision 35350) @@ -73,13 +73,13 @@ self.logger.infoTitle("Cleaning up batch type %s" % batchType) - loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType) - deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType) - mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType) + mergedLocalIDs = self.ippToPspsDb.getMergedReadyToDeleteFromLocalDisk(batchType) + purgedLocalIDs = self.ippToPspsDb.getPurgedReadyToDeleteFromLocalDisk(batchType) + deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs - purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType) - purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType) + loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMReadyToDeleteFromDatastore(batchType) + purgedDatastoreIDs = self.ippToPspsDb.getPurgedReadyToDeleteFromDatastore(batchType) + deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs - deleteFromDatastoreIDs = loadedDatastoreIDs + purgedDatastoreIDs - deleteFromLocalIDs = mergedLocalIDs + purgedLocalIDs + deleteFromDxLayerIDs = self.ippToPspsDb.getLoadedToODMReadyToDeleteFromDxLayer(batchType) # report to log what we are going to do Index: /branches/eam_branches/ipp-20130307/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/eam_branches/ipp-20130307/ippToPsps/jython/ipptopspsdb.py (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippToPsps/jython/ipptopspsdb.py (revision 35350) @@ -30,8 +30,8 @@ ''' - Returns a list of merged batch IDs that are merged but not yet deleted - ''' - def getMergedButNotDeletedBatchIDs(self, batchType, column): - + Returns a list of merged batch IDs that are merged but not yet deleted (but allowed to be deleted) + ''' + def getMergedReadyToDeleteFromLocalDisk(self, batchType): + sql = "SELECT DISTINCT batch_id \ FROM batch \ @@ -40,6 +40,7 @@ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND merged = 1 \ - AND " + column + " = 0" - + AND deleted_local = 0 \ + AND block_delete_local = 0 " + ids = [] try: @@ -49,14 +50,14 @@ except: self.logger.exception("Can't query for merged batch ids in ipptopsps Db") - + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) - + return ids ''' - Returns a list of batch IDs marked as 'purged' but not yet deleted - ''' - def getPurgedButNotDeletedBatchIDs(self, batchType, column): - + Returns a list of batch IDs marked as 'purged' but not yet deleted (but allowed to be deleted) + ''' + def getPurgedReadyToDeleteFromLocalDisk(self, batchType): + sql = "SELECT DISTINCT batch_id \ FROM batch \ @@ -65,6 +66,7 @@ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND purged = 1 \ - AND " + column + " = 0" - + AND deleted_local = 0 \ + AND block_delete_local = 0 " + ids = [] try: @@ -74,15 +76,15 @@ except: self.logger.exception("Can't query for merged batch ids in ipptopsps Db") - - + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) - + return ids ''' - Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted - ''' - def getLoadedToODMButNotDeletedBatchIDs(self, batchType, column): - + Returns a list of batch IDs marked as 'purged' but not yet deleted (but allowed to be deleted) + ''' + def getPurgedReadyToDeleteFromDatastore(self, batchType): + sql = "SELECT DISTINCT batch_id \ FROM batch \ @@ -90,6 +92,33 @@ AND batch_type = '" + batchType + "' \ AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + AND purged = 1 \ + AND deleted_datastore = 0 \ + AND block_delete_datastore = 0 " + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted from the datastore + ''' + def getLoadedToODMReadyToDeleteFromDatastore(self, batchType): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ - AND " + column + " = 0" + AND deleted_datastore = 0 AND block_delete_datastore = 0" ids = [] @@ -106,38 +135,141 @@ ''' - Returns a list of merged batch IDs that are not deleted from local disk - ''' - def getMergedButNotDeletedFromLocalDisk(self, batchType): - return self.getMergedButNotDeletedBatchIDs(batchType, "deleted_local") - - ''' - Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk - ''' - def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType): - return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_local") - - ''' - Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore - ''' - def getLoadedToODMButNotDeletedFromDatastore(self, batchType): - return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_datastore") - - ''' - Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer - ''' - def getLoadedToODMButNotDeletedFromDXLayer(self, batchType): - return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_dxlayer") - - ''' - Returns a list of purged batch IDs that not deleted from local disk - ''' - def getPurgedButNotDeletedFromLocalDisk(self, batchType): - return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_local") - - ''' - Returns a list of purged batch IDs that are not deleted from datastore - ''' - def getPurgedButNotDeletedFromDatastore(self, batchType): - return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_datastore") + Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted from the dxlayer + ''' + def getLoadedToODMReadyToDeleteFromDxLayer(self, batchType): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + self.skychunk.epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ + AND deleted_dxlayer = 0 and block_delete_dxlayer = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ## ''' + ## XXX EAM : removed this function and made it more explicit (with an additional block) + ## Returns a list of merged batch IDs that are merged but not yet deleted + ## ''' + ## def getMergedButNotDeletedBatchIDs(self, batchType, column): + ## + ## sql = "SELECT DISTINCT batch_id \ + ## FROM batch \ + ## WHERE timestamp > '" + self.skychunk.epoch + "' \ + ## AND batch_type = '" + batchType + "' \ + ## AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + ## AND merged = 1 \ + ## AND " + column + " = 0" + ## + ## ids = [] + ## try: + ## rs = self.executeQuery(sql) + ## while (rs.next()): ids.append(rs.getInt(1)) + ## rs.close() + ## except: + ## self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + ## + ## self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + ## + ## return ids + + ## ''' + ## Returns a list of batch IDs marked as 'purged' but not yet deleted + ## XXX EAM : removed this function and made it more explicit (with an additional block) + ## ''' + ## def getPurgedButNotDeletedBatchIDs(self, batchType, column): + ## + ## sql = "SELECT DISTINCT batch_id \ + ## FROM batch \ + ## WHERE timestamp > '" + self.skychunk.epoch + "' \ + ## AND batch_type = '" + batchType + "' \ + ## AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + ## AND purged = 1 \ + ## AND " + column + " = 0" + ## + ## ids = [] + ## try: + ## rs = self.executeQuery(sql) + ## while (rs.next()): ids.append(rs.getInt(1)) + ## rs.close() + ## except: + ## self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + ## + ## + ## self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + ## + ## return ids + + ## ''' + ## Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted + ## ''' + ## def getLoadedToODMButNotDeletedBatchIDs(self, batchType, column): + ## + ## sql = "SELECT DISTINCT batch_id \ + ## FROM batch \ + ## WHERE timestamp > '" + self.skychunk.epoch + "' \ + ## AND batch_type = '" + batchType + "' \ + ## AND dvo_db = '" + self.skychunk.dvoLabel + "' \ + ## AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ + ## AND " + column + " = 0" + ## + ## ids = [] + ## try: + ## rs = self.executeQuery(sql) + ## while (rs.next()): ids.append(rs.getInt(1)) + ## rs.close() + ## except: + ## self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + ## + ## self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + ## + ## return ids + + # ''' + # Returns a list of merged batch IDs that are not deleted from local disk + # ''' + # def getMergedButNotDeletedFromLocalDisk(self, batchType): + # return self.getMergedButNotDeletedBatchIDs(batchType, "deleted_local") + + ## ''' + ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk + ## ''' + ## def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType): + ## return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_local") + ## + ## ''' + ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore + ## ''' + ## def getLoadedToODMButNotDeletedFromDatastore(self, batchType): + ## return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_datastore") + ## + ## ''' + ## Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer + ## ''' + ## def getLoadedToODMButNotDeletedFromDXLayer(self, batchType): + ## return self.getLoadedToODMButNotDeletedBatchIDs(batchType, "deleted_dxlayer") + + ## ''' + ## Returns a list of purged batch IDs that not deleted from local disk + ## ''' + ## def getPurgedButNotDeletedFromLocalDisk(self, batchType): + ## return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_local") + ## + ## ''' + ## Returns a list of purged batch IDs that are not deleted from datastore + ## ''' + ## def getPurgedButNotDeletedFromDatastore(self, batchType): + ## return self.getPurgedButNotDeletedBatchIDs(batchType, "deleted_datastore") Index: /branches/eam_branches/ipp-20130307/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/Makefile.am (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/share/Makefile.am (revision 35350) @@ -457,4 +457,7 @@ laptool_WSdiff_check.sql \ laptool_stacks.sql \ + laptool_pendinggroup.sql \ + laptool_revertgroup.sql \ + laptool_filtersforgroup.sql \ vptool_find_rawexp.sql \ vptool_pendingimfile.sql \ Index: /branches/eam_branches/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/share/addtool_find_processedexp_cam.sql (revision 35350) @@ -1,5 +1,11 @@ SELECT addProcessedExp.*, - addRun.workdir + addRun.label, + addRun.workdir, + camRun.cam_id, + camRun.label as cam_label, + camRun.data_group as cam_data_group, + rawExp.exp_id, + exp_name FROM addProcessedExp JOIN addRun Index: /branches/eam_branches/ipp-20130307/ippTools/share/laptool_filtersforgroup.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/laptool_filtersforgroup.sql (revision 35350) +++ /branches/eam_branches/ipp-20130307/ippTools/share/laptool_filtersforgroup.sql (revision 35350) @@ -0,0 +1,2 @@ +SELECT DISTINCT filter +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) Index: /branches/eam_branches/ipp-20130307/ippTools/share/laptool_pendinggroup.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/laptool_pendinggroup.sql (revision 35350) +++ /branches/eam_branches/ipp-20130307/ippTools/share/laptool_pendinggroup.sql (revision 35350) @@ -0,0 +1,4 @@ +SELECT DISTINCT lapGroup.*, + lapRun.dist_group +FROM lapGroup JOIN lapRun USING(seq_id, tess_id, projection_cell) +WHERE lapGroup.state ='new' AND lapGroup.fault = 0 Index: /branches/eam_branches/ipp-20130307/ippTools/share/laptool_revertgroup.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/laptool_revertgroup.sql (revision 35350) +++ /branches/eam_branches/ipp-20130307/ippTools/share/laptool_revertgroup.sql (revision 35350) @@ -0,0 +1,3 @@ +UPDATE lapGroup +SET fault = 0 +WHERE fault != 0 Index: /branches/eam_branches/ipp-20130307/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/pxadmin_create_tables.sql (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/share/pxadmin_create_tables.sql (revision 35350) @@ -1441,5 +1441,8 @@ outProduct VARCHAR(64), uri VARCHAR(255), - outdir VARCHAR(255), + outdir VARCHAR(255), + username VARCHAR(255), + proj_id BIGINT, + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, timestamp TIMESTAMP, fault SMALLINT, @@ -1448,5 +1451,6 @@ KEY(state), KEY(fault), - KEY(label) + KEY(label), + KEY(username) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -2194,4 +2198,5 @@ KEY (state), KEY (fault), + KEY (group_id), FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), @@ -2253,8 +2258,11 @@ CREATE TABLE lapGroup ( seq_id BIGINT, + tess_id VARCHAR(64), projection_cell VARCHAR(64), state VARCHAR(16), + label VARCHAR(64), + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time group was registered fault SMALLINT, - PRIMARY KEY(seq_id, projection_cell), + PRIMARY KEY(seq_id, tess_id, projection_cell), KEY(state), KEY(fault), Index: /branches/eam_branches/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (revision 35350) @@ -1,4 +1,6 @@ SELECT relExp.relexp_id -FROM relGroup JOIN lapRun USING(lap_id) +FROM relGroup + JOIN lapRun USING(lap_id) + JOIN lapExp USING(lap_id) JOIN relExp using(rel_id, exp_id) WHERE relExp.group_id = 0 Index: /branches/eam_branches/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/share/releasetool_pendingrelgroup.sql (revision 35350) @@ -1,3 +1,5 @@ -SELECT relGroup.* +SELECT + relGroup.*, + ippRelease.release_name FROM relGroup JOIN ippRelease USING(rel_id) Index: /branches/eam_branches/ipp-20130307/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/addtool.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/addtool.c (revision 35350) @@ -876,5 +876,5 @@ psTrace("addtool", PS_LOG_INFO, "no rows found"); psFree(output); - return true; + return false; } Index: /branches/eam_branches/ipp-20130307/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/camtool.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/camtool.c (revision 35350) @@ -599,5 +599,5 @@ psTrace("camtool", PS_LOG_INFO, "no rows found"); psFree(output); - return true; + return false; } Index: /branches/eam_branches/ipp-20130307/ippTools/src/laptool.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/laptool.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/laptool.c (revision 35350) @@ -34,4 +34,12 @@ static bool inactiveexpMode(pxConfig *config); +// Groups +static bool definegroupMode(pxConfig *config); +static bool pendinggroupMode(pxConfig *config); +static bool filtersforgroupMode(pxConfig *config); +static bool updategroupMode(pxConfig *config); +static bool revertgroupMode(pxConfig *config); +static bool listgroupMode(pxConfig *config); + # define MODECASE(caseName, func) \ case caseName: \ @@ -67,4 +75,12 @@ MODECASE(LAPTOOL_MODE_INACTIVEEXP, inactiveexpMode); + + MODECASE(LAPTOOL_MODE_DEFINEGROUP, definegroupMode); + MODECASE(LAPTOOL_MODE_PENDINGGROUP, pendinggroupMode); + MODECASE(LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupMode); + MODECASE(LAPTOOL_MODE_UPDATEGROUP, updategroupMode); + MODECASE(LAPTOOL_MODE_REVERTGROUP, revertgroupMode); + MODECASE(LAPTOOL_MODE_LISTGROUP, listgroupMode); + default: psAbort("invalid option (this should not happen)"); @@ -900,3 +916,443 @@ return(true); } - +// --------------------------- +// Group level (a collection of completed lapRuns for a given lapSequence projection cell and collection of filters + +static bool definegroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // the following insures that the -filter argument has been set up properly (adapted from pxAddLabelSearchArgs) + psMetadataItem *item = psMetadataLookup(config->args, "-filter"); + psAssert (item, "-filter argument not found in config->args"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", "filter"); + psAssert (item->data.list->n, "%s should at least have a place-holder", "filter"); + psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data; + psAssert (entry, "%s should at least have a place-holder", "filter"); + // end of checking + + // Now if the ony entry is the place-holder then the user supplied no -filter arguments + // which are required + if (!entry->data.str) { + psError(PXTOOLS_ERR_ARGUMENTS, true, "at least one -filter is required"); + return false; + } + + int nFilters = item->data.list->n; + + psString query = psStringCopy("SELECT seq_id, tess_id, projection_cell,\n"); + + // We construct a query joining completed lap runs with the same lapSequence.seq_id, + // tess_id, and projection_cell + + // select the lap_ids for the various filters. These are only used for debugging. + psStringAppend(&query, "lap_id_%d", 0); + for (int i = 1; i < nFilters; i++) { + psStringAppend(&query, ", lap_id_%d", i); + } + + + // sub query for each supplied filter + char * lapRunForFilter = "(\nSELECT seq_id, tess_id, projection_cell, lap_id as 'lap_id_%d'\n" + "FROM lapRun\n" + "WHERE filter LIKE '%s'\n" + " AND lapRun.seq_id = %"PRId64"\n" + " AND (lapRun.state = 'done' or lapRun.state = 'full')\n" + " ) as lap_%d\n"; + + // loop over supplied filters and flesh out the query using the format above + psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true); + psMetadataItem *filterItem = NULL; + int i = -1; + while ((filterItem = psListGetAndIncrement(iter))) { + ++i; + if (i == 0) { + psStringAppend(&query, "\nFROM\n"); + } else { + psStringAppend(&query, "\nJOIN\n"); + } + + psString filter = filterItem->data.str; + psStringAppend(&query, lapRunForFilter, i, filter, seq_id, i); + + if (i != 0) { + psStringAppend(&query, "USING (seq_id, tess_id, projection_cell)\n"); + } + } + // now join to lapGroup + psStringAppend(&query, "\nLEFT JOIN lapGroup USING(seq_id, tess_id, projection_cell)\n"); + + // we only want projection cells which do not already of an entry + psStringAppend(&query, "\nWHERE lapGroup.projection_cell IS NULL\n"); + psStringAppend(&query, "\nORDER by projection_cell\n"); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + if (!ippdbPrintMetadatas(stdout, output, "new_lapGroups", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + + psString last_projection_cell = NULL; + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psString tess_id = psMetadataLookupStr(NULL, row, "tess_id"); + psString projection_cell = psMetadataLookupStr(NULL, row, "projection_cell"); + if (last_projection_cell && !strcmp(last_projection_cell, projection_cell)) { + // duplicate lap runs for a filter will generate multiple rows + // Since we care about projection_cells not lapRuns per se this is not problem. + // Skip any duplicates. + continue; + } + last_projection_cell = projection_cell; + + lapGroupRow *group = lapGroupRowAlloc( + seq_id, + tess_id, + projection_cell, + "new", // state + label, + now, // registered + 0 // fault + ); + if (!group) { + psError(PS_ERR_UNKNOWN, false, "failed to alloc lapGroup object"); + psFree(output); + psFree(now); + return(false); + } + + + if (!lapGroupInsertObject(config->dbh, group)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return(true); + } + } + + // point of no return + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(now); + psFree(output); + + return(true); +} + +static bool pendinggroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "lapRun.label", "=="); + + psString query = pxDataGet("laptool_pendinggroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapGroup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool filtersforgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + psString query = pxDataGet("laptool_filtersforgroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "filters", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +static bool updategroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // check that all of the keys that define a lapGroup are supplied + PXOPT_LOOKUP_S64(seq_id, config->args, "-seq_id", true, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(projection_cell, config->args, "-projection_cell", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + psMetadata *values = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, values, "-set_state", "state", "=="); + PXOPT_COPY_STR(config->args, values, "-set_label", "label", "=="); + PXOPT_COPY_S16(config->args, values, "-set_fault", "fault", "=="); + + if (!psListLength(values->list)) { + psFree(values); + psFree(where); + psError(PXTOOLS_ERR_ARGUMENTS, true, "must set at least one column"); + return false; + } + + long rows = psDBUpdateRows(config->dbh, "lapGroup", where, values); + if (rows < 1) { + psFree(values); + psError(PXTOOLS_ERR_SYS, true, "failed to update lapGroup for %" PRId64 " %s %s", seq_id, tess_id, projection_cell); + return false; + } + psFree(values); + + return(true); +} + +static bool revertgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-projection_cell", "projection_cell", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required"); + return false; + } + + psString query = pxDataGet("laptool_revertgroup.sql"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("laptool", PS_LOG_INFO, "Updated %" PRIu64 " lapGroups", affected); + + return true; +} + +static bool listgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psError(PXTOOLS_ERR_SYS, true, "not yet implemented"); + return false; + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-seq_name", "name", "LIKE"); + + psString query = pxDataGet("laptool_listsequence.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapSequence", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: /branches/eam_branches/ipp-20130307/ippTools/src/laptool.h =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/laptool.h (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/laptool.h (revision 35350) @@ -20,5 +20,11 @@ LAPTOOL_MODE_UPDATEEXP, LAPTOOL_MODE_DIFFCHECK, - LAPTOOL_MODE_INACTIVEEXP + LAPTOOL_MODE_INACTIVEEXP, + LAPTOOL_MODE_DEFINEGROUP, + LAPTOOL_MODE_PENDINGGROUP, + LAPTOOL_MODE_FILTERSFORGROUP, + LAPTOOL_MODE_UPDATEGROUP, + LAPTOOL_MODE_REVERTGROUP, + LAPTOOL_MODE_LISTGROUP, } laptoolMode; Index: /branches/eam_branches/ipp-20130307/ippTools/src/laptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/laptoolConfig.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/laptoolConfig.c (revision 35350) @@ -97,5 +97,4 @@ ADD_OPT(S64, updaterunArgs, "-set_quick_sass_id", "set quick stack sass_id", 0); ADD_OPT(S64, updaterunArgs, "-set_final_sass_id", "set final stack sass_id", 0); - // -pendingexp @@ -150,4 +149,52 @@ ADD_OPT(U64, inactiveexpArgs, "-limit", "limit result set to N items", 0); + // -definegroup + psMetadata *definegroupArgs = psMetadataAlloc(); + ADD_OPT(S64, definegroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, definegroupArgs, "-set_label", "set label (required)", 0); + + psMetadataAddStr(definegroupArgs, PS_LIST_TAIL, + "-filter", PS_META_DUPLICATE_OK, + "search by filter (LIKE comparison, multiple OK)", NULL); + ADD_OPT(Bool,definegroupArgs, "-pretend", "do not actuallym modify the database", false); + ADD_OPT(Bool,definegroupArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, definegroupArgs, "-limit", "limit result set to N items", 0); + + // -pendinggroup + psMetadata *pendinggroupArgs = psMetadataAlloc(); + psMetadataAddStr(pendinggroupArgs, PS_LIST_TAIL, + "-label", PS_META_DUPLICATE_OK, + "search by label (multiple OK)", NULL); + ADD_OPT(S64, pendinggroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + + ADD_OPT(Bool,pendinggroupArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, pendinggroupArgs, "-limit", "limit result set to N items", 0); + + // -updategroup + psMetadata *updategroupArgs = psMetadataAlloc(); + ADD_OPT(S64, updategroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, updategroupArgs, "-tess_id", "search by tess_id (required)", 0); + ADD_OPT(Str, updategroupArgs, "-projection_cell", "search by projection_cell (required)", 0); + ADD_OPT(Str, updategroupArgs, "-set_state", "set state", 0); + ADD_OPT(S16, updategroupArgs, "-set_fault", "set fault code", INT16_MAX); + ADD_OPT(Str, updategroupArgs, "-set_label", "set label", 0); + + // -revertgroup + psMetadata *revertgroupArgs = psMetadataAlloc(); + psMetadataAddStr(revertgroupArgs, PS_LIST_TAIL, + "-label", PS_META_DUPLICATE_OK, + "search by label (multiple OK)", NULL); + ADD_OPT(S64, revertgroupArgs, "-seq_id", "search by lap sequence ID", 0); + ADD_OPT(Str, revertgroupArgs, "-tess_id", "search by tess_id", 0); + ADD_OPT(Str, revertgroupArgs, "-projection_cell", "search by projection_cell", 0); + ADD_OPT(S16, revertgroupArgs, "-fault", "fault code", INT16_MAX); + + // -filtersforgroup + psMetadata *filtersforgroupArgs = psMetadataAlloc(); + ADD_OPT(S64, filtersforgroupArgs, "-seq_id", "search by lap sequence ID (required)", 0); + ADD_OPT(Str, filtersforgroupArgs, "-tess_id", "search by tess_id (required)", 0); + ADD_OPT(Str, filtersforgroupArgs, "-projection_cell", "search by projection_cell (required)", 0); + ADD_OPT(Bool,filtersforgroupArgs, "-simple", "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); @@ -166,4 +213,9 @@ PXOPT_ADD_MODE("-diffcheck", "", LAPTOOL_MODE_DIFFCHECK, diffcheckArgs); PXOPT_ADD_MODE("-inactiveexp", "", LAPTOOL_MODE_INACTIVEEXP, inactiveexpArgs); + PXOPT_ADD_MODE("-definegroup", "", LAPTOOL_MODE_DEFINEGROUP, definegroupArgs); + PXOPT_ADD_MODE("-pendinggroup", "", LAPTOOL_MODE_PENDINGGROUP, pendinggroupArgs); + PXOPT_ADD_MODE("-filtersforgroup", "", LAPTOOL_MODE_FILTERSFORGROUP, filtersforgroupArgs); + PXOPT_ADD_MODE("-updategroup", "", LAPTOOL_MODE_UPDATEGROUP, updategroupArgs); + PXOPT_ADD_MODE("-revertgroup", "", LAPTOOL_MODE_REVERTGROUP, revertgroupArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/ipp-20130307/ippTools/src/pstamptool.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/pstamptool.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/pstamptool.c (revision 35350) @@ -287,4 +287,9 @@ PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + PXOPT_LOOKUP_STR(username, config->args, "-username", false, false); + PXOPT_LOOKUP_S64(proj_id, config->args, "-proj_id", false, false); + + psTime *now = psTimeGetNow(PS_TIME_TAI); + if (!pstampRequestInsert(config->dbh, 0, // req_id @@ -297,5 +302,8 @@ uri, NULL, // outdir - NULL, // timestamp + username, + proj_id, + now, // registered + now, // timestamp 0 // fault )) { @@ -444,5 +452,7 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-username", "username", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -450,5 +460,5 @@ if (!psListLength(where->list)) { - psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied"); + psError(PS_ERR_UNKNOWN, true, "search paramters are required"); return false; } @@ -482,8 +492,8 @@ if (!psArrayLength(output)) { psTrace("pstamptool", PS_LOG_INFO, "request not found"); - // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for + // This causes main to exit with PS_EXIT_DATA_ERROR which the pstamp scripts are looking for psError(PXTOOLS_ERR_CONFIG, true, "request not found"); psFree(output); - // we return false so that the caller can determine that a request does not exist + // we return false so that the caller can easily determine that a request does not exist return false; } @@ -567,12 +577,13 @@ PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(outProduct, config->args, "-set_outProduct", false, false); - PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_STR(uri, config->args, "-set_uri", false, false); - PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-set_outdir", false, false); PXOPT_LOOKUP_STR(name, config->args, "-set_name", false, false); + PXOPT_LOOKUP_STR(username, config->args, "-set_username", false, false); PXOPT_LOOKUP_STR(reqType, config->args, "-set_reqType", false, false); - PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault", false); - - if (!state && !label && !outProduct && !fault && !uri && !outdir && !name && !reqType && !clearfault) { + PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault", false); + + if (!state && !label && !outProduct && !fault && !uri && !outdir && !name && !username && !reqType && !clearfault) { psError(PS_ERR_UNKNOWN, true, "at least one set option is required"); return false; @@ -584,6 +595,7 @@ PXOPT_COPY_S32(config->args, where, "-fault", "fault", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); - PXOPT_COPY_STR(config->args, where, "-reqType", "reqType", "=="); - PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-reqType", "reqType", "=="); + PXOPT_COPY_STR(config->args, where, "-name", "name", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-username", "username", "=="); PXOPT_COPY_TIME(config->args, where, "-timestamp_begin", "timestamp", ">="); PXOPT_COPY_TIME(config->args, where, "-timestamp_end", "timestamp", "<="); Index: /branches/eam_branches/ipp-20130307/ippTools/src/pstamptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/pstamptoolConfig.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/pstamptoolConfig.c (revision 35350) @@ -74,5 +74,7 @@ psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-proj_id", 0, "define request proj_id", 0); psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-username", 0, "define user name", NULL); psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); @@ -88,4 +90,6 @@ psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-username", 0, "list by user name", NULL); + psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-state", 0, "list by state", NULL); psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -105,5 +109,6 @@ psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0, "search by reqType", NULL); - psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "search by reqType (LIKE comparsion)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0, "search by reqType (LIKE comparsion)", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-username", 0, "search by username", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddTime(updatereqArgs, PS_LIST_TAIL, "-timestamp_begin", 0, "search by timestamp (>=)", NULL); @@ -116,4 +121,5 @@ psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_uri", 0, "new uri", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_name", 0, "new name", NULL); + psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_username", 0, "new username", NULL); psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_reqType", 0, "new reqType", NULL); psMetadataAddBool(updatereqArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); Index: /branches/eam_branches/ipp-20130307/ippTools/src/releasetool.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/releasetool.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/releasetool.c (revision 35350) @@ -1270,4 +1270,7 @@ } if (!psArrayLength(exposures)) { + fprintf(stderr, "no exposures found for lap_id %" PRId64 "\n", lap_id); +#ifdef notdef + continue; psFree(now); psFree(output); @@ -1278,4 +1281,5 @@ } return false; +#endif } @@ -1301,5 +1305,5 @@ psFree(exposures); - // set state of relGroup to 'new' + // update the state of relGroup to 'new' if (!p_psDBRunQueryF(config->dbh, "UPDATE relGroup set state = 'new' WHERE group_id = %d", group_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1325,6 +1329,78 @@ static bool updaterelgroupMode(pxConfig *config) { - psError(PS_ERR_UNKNOWN, true, "not yet implemented"); - return false; + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S32(group_id, config->args, "-group_id", true, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(exp_list_path, config->args, "-set_exp_list_path", false, false); + PXOPT_LOOKUP_STR(new_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + + if (!state && !exp_list_path && !new_label && !fault && !clearfault) { + psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n"); + return false; + } + + if (fault && clearfault) { + psError(PXTOOLS_ERR_CONFIG, true, "cannot set and clear fault at same time\n"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-group_id", "relGroup.group_id", "=="); + // XXX TODO if label is enabled (for changing label or state for a block) we should + // disallow setting some parameters such as exp_list_path + // PXOPT_COPY_S64(config->args, where, "-label", "relGroup.label", "=="); + // make sure that we have enough parameters to identify the relGroup to change + if (!psListLength(where->list)){ + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + psString sep = ""; + psString comma = ", "; + psString query = psStringCopy("UPDATE relGroup SET"); + if (state) { + psStringAppend(&query, "%s relGroup.state = '%s'", sep, state); + sep = comma; + } + if (exp_list_path) { + psStringAppend(&query, "%s relGroup.exp_list_path = '%s'", sep, exp_list_path); + sep = comma; + } + if (new_label) { + psStringAppend(&query, "%s relGroup.label = '%s'", sep, new_label); + sep = comma; + } + if (fault) { + psStringAppend(&query, "%s relGroup.fault = %d", sep, fault); + sep = comma; + } + if (clearfault) { + psStringAppend(&query, "%s relGroup.fault = 0", sep); + sep = comma; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected); + + + psFree(query); + + return true; } Index: /branches/eam_branches/ipp-20130307/ippTools/src/releasetoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130307/ippTools/src/releasetoolConfig.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippTools/src/releasetoolConfig.c (revision 35350) @@ -217,5 +217,5 @@ // parameters of the relGroup psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_type", 0, "define group_type (required)", NULL); - psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0, "define group_name (required)", NULL); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0, "define group_name", NULL); psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label (required)", NULL); psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -240,5 +240,4 @@ psMetadata *updaterelgroupArgs = psMetadataAlloc(); - // set the target release psMetadataAddS32(updaterelgroupArgs, PS_LIST_TAIL, "-group_id", 0, "select by group ID (required)", 0); Index: /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppImage.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppImage.config (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppImage.config (revision 35350) @@ -1232,4 +1232,5 @@ BIAS BOOL FALSE # Bias subtraction DARK BOOL TRUE # Dark subtraction + NOISEMAP BOOL FALSE REMNANCE BOOL FALSE # Remnance masking SHUTTER BOOL FALSE # Shutter correction Index: /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippconfig/gpc1/ppStack.config (revision 35350) @@ -56,4 +56,10 @@ END +STACK_ALLDEEP METADATA # N>>20 inputs, typically MD fields, no cuts + OUTPUT.NOCOMP BOOL TRUE + OUTPUT.LOGFLUX BOOL FALSE + OUTPUT.REPLICATE BOOL TRUE +END + # Recipe overrides for STACK STACKPHOT METADATA Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStack.config (revision 35350) @@ -159,4 +159,7 @@ END +STACK_ALLDEEP METADATA +END + TEST1 METADATA PSF.AUTOSIZE BOOL FALSE # Determine output PSF size from input PSFs? Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStatsFromMetadata.config (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/ppStatsFromMetadata.config (revision 35350) @@ -563,6 +563,6 @@ ENTRY VAL ZPT_ERR F32 CONSTANT -zpt_stdev # don't know why these come out as F64 instead of F32 - ENTRY VAL FWHM_MAJ F64 CONSTANT -fhwm_major - ENTRY VAL FWHM_MIN F64 CONSTANT -fhwm_minor + ENTRY VAL FWHM_MAJ F64 CONSTANT -fwhm_major + ENTRY VAL FWHM_MIN F64 CONSTANT -fwhm_minor ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag Index: /branches/eam_branches/ipp-20130307/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/eam_branches/ipp-20130307/ippconfig/recipes/reductionClasses.mdc (revision 35349) +++ /branches/eam_branches/ipp-20130307/ippconfig/recipes/reductionClasses.mdc (revision 35350) @@ -550,4 +550,13 @@ STACK_PPSUB STR STACK STACK_PSPHOT STR STACK +END + +# all/deep stacks +ALLDEEP_STACK METADATA + STACK_PPSTACK STR STACK_ALLDEEP + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + STACKPHOT_PSPHOT STR STACKPHOT + STATICSKY_CALIBRATION STR STATICSKY_DEEPCAL END Index: /branches/eam_branches/ipp-20130307/psModules/src/detrend/pmPattern.c =================================================================== --- /branches/eam_branches/ipp-20130307/psModules/src/detrend/pmPattern.c (revision 35349) +++ /branches/eam_branches/ipp-20130307/psModules/src/detrend/pmPattern.c (revision 35350) @@ -997,6 +997,6 @@ pmChip *chip = out->fpa->chips->data[i]; pmCell *cell = chip->cells->data[0]; // Cell of interest - pmReadout *ro = cell->readouts->data[0]; // Readout of interest - + + psStatsInit(bgStats); @@ -1009,5 +1009,12 @@ int y = chipName[3] - '0'; - + if ((cell->readouts->n != 1)||(!chip->data_exists)) { + A->data.F64[y][x] = NAN; + B->data.F64[y][x] = NAN; + C->data.F64[y][x] = NAN; + D->data.F64[y][x] = NAN; + continue; + } + pmReadout *ro = cell->readouts->data[0]; // Readout of interest for (int j = 0; j < 4; j++) { if (j == 0) { // Region B @@ -1284,4 +1291,7 @@ pmChip *chip = out->fpa->chips->data[i]; pmCell *cell = chip->cells->data[0]; // Cell of interest + if ((cell->readouts->n != 1)||(!chip->data_exists)) { + continue; + } pmReadout *ro = cell->readouts->data[0]; // Readout of interest const char *chipName = psMetadataLookupStr(NULL,chip->concepts, "CHIP.NAME"); Index: /branches/eam_branches/ipp-20130307/tools/cleandsproduct.pl =================================================================== --- /branches/eam_branches/ipp-20130307/tools/cleandsproduct.pl (revision 35349) +++ /branches/eam_branches/ipp-20130307/tools/cleandsproduct.pl (revision 35350) @@ -52,5 +52,9 @@ -$datastore = "http://datastore.ipp.ifa.hawaii.edu" if !$datastore; + +# since this needs to run on the MHPCC cluster there is no need to go to the +# proxy to get the datastore listing +$datastore = "http://ippc17/ds" if !$datastore; +# $datastore = "http://datastore.ipp.ifa.hawaii.edu" if !$datastore; my $uri = "$datastore/$product/index.txt"; Index: /branches/eam_branches/ipp-20130307/tools/heather/stack_inspector.pl =================================================================== --- /branches/eam_branches/ipp-20130307/tools/heather/stack_inspector.pl (revision 35350) +++ /branches/eam_branches/ipp-20130307/tools/heather/stack_inspector.pl (revision 35350) @@ -0,0 +1,107 @@ +#!/usr/bin/env perl + +# +# Copy stack inputs to the current directory +# + +use warnings; +use strict; + +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; # Socket for mysql + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); + +my ($db_host, $db_name, $db_user, $db_pw); # Database details +my ($stack_id); # Stack of interest + +GetOptions( + 'dbhost=s' => \$db_host, # Database host name + 'dbname=s' => \$db_name, # Database name + 'dbuser=s' => \$db_user, # Database user + 'dbpass=s' => \$db_pw, # Database p/w + 'stack_id=s' => \$stack_id, # Stack identifier + ) or die "Unable to parse arguments.\n"; +die "Unknown option: @ARGV\n" if @ARGV; +die "Required options: --dbhost --dbname --dbuser --dbpass --stack_id\n" + unless defined $db_host + and defined $db_name + and defined $db_user + and defined $db_pw + and defined $stack_id; + +# Database connection +my $db = DBI->connect( "DBI:mysql:database=$db_name;host=$db_host;mysql_socket=" . DB_SOCKET(), + $db_user, + $db_pw, + { RaiseError => 1, AutoCommit => 1 } + ) or die "Unable to connect to database: $DBI::errstr"; + +# Query to run +my $sql = "SELECT warpSkyfile.path_base" . + " FROM stackRun" . + " JOIN stackInputSkyfile USING(stack_id)" . + " JOIN warpSkyfile USING(warp_id, skycell_id)" . + " WHERE stack_id = $stack_id"; + +# List of warps +my $warps = $db->selectcol_arrayref( $sql, + { Columns => [1] } + ) or die "Unable to execute SQL: $DBI::errstr"; + +$sql = "SELECT path_base FROM stackSumSkyfile WHERE stack_id = $stack_id"; + +# List of stacks +my $stacks = $db->selectcol_arrayref( $sql, + { Columns => [1] } + ) or die "Unable to execute SQL: $DBI::errstr"; + +$db->disconnect; + + + + +foreach my $stack_path ( @$stacks ) { + print "$stack_path\n"; + #make jpg of unconv + my @ext = (".fits", ".unconv.fits",".exp.fits", ".unconv.exp.fits" ); + # if ($db_name == "isp") { +# #add in other single chip stuff if needed +# @ext = (".exp.fits", ".unconv.exp.fits"); +# } + + + #make jpg of conv + foreach my $stuff (@ext) { + my $file = $stack_path.$stuff; + print "$file\n"; + `echo $file > list`; + my $base = $file; + $base =~ s/.fits//; + $base =~ s/.*\///g; + print $base; + `ppSkycell -images list $base`; + } + + +} + + + +foreach my $warp_path ( @$warps ) { + print "$warp_path\n"; + my @ext = (".fits"); + foreach my $stuff (@ext) { + my $file = $warp_path.$stuff; + print "$file\n"; + `echo $file > list`; + my $base = $file; + $base =~ s/.fits//; + $base =~ s/.*\///g; + print $base; + `ppSkycell -images list $base`; + } + + + +} Index: /branches/eam_branches/ipp-20130307/tools/run.relastro =================================================================== --- /branches/eam_branches/ipp-20130307/tools/run.relastro (revision 35349) +++ /branches/eam_branches/ipp-20130307/tools/run.relastro (revision 35350) @@ -6,5 +6,5 @@ endif -set REALRUN = 0 +set REALRUN = 1 set catdir = $argv[1] @@ -21,5 +21,5 @@ # -update-chips : determine improved astrometry calibrations # try 1000 meas per deg^2, but if memory if available, go to more -if (1) then +if (0) then set options = "-update-chips" set options = "$options -D RELASTRO_SRC_MEAS_TOOFEW 1" @@ -34,5 +34,5 @@ # enable parallel operations for parallel databases - # set options = "$options -parallel" + set options = "$options -parallel" set region = "-region $ra_min $ra_max $dec_min $dec_max" @@ -54,5 +54,5 @@ # enable parallel operations for parallel databases - # set options = "$options -parallel" + set options = "$options -parallel" set region = "-region $ra_min $ra_max $dec_min $dec_max" Index: /branches/eam_branches/ipp-20130307/tools/run.relphot =================================================================== --- /branches/eam_branches/ipp-20130307/tools/run.relphot (revision 35349) +++ /branches/eam_branches/ipp-20130307/tools/run.relphot (revision 35350) @@ -40,5 +40,5 @@ # activate parallel mode when catdir is parallel - # set options = "$options -parallel" + set options = "$options -parallel" # option to change the schema Index: /branches/eam_branches/ipp-20130307/tools/run.setphot =================================================================== --- /branches/eam_branches/ipp-20130307/tools/run.setphot (revision 35349) +++ /branches/eam_branches/ipp-20130307/tools/run.setphot (revision 35350) @@ -6,5 +6,5 @@ endif -set REALRUN = 0 +set REALRUN = 1 set catdir = $argv[1] @@ -22,5 +22,5 @@ # activate parallel mode when catdir is parallel -# set options = "$options -parallel" +set options = "$options -parallel" # the new ubercal file includes Eddie's offsets relative to JT (so, no need to supply -zpt-offsets)