Index: /branches/eam_branches/ipp-20130207/Nebulous-Server/bin/nebdiskd =================================================================== --- /branches/eam_branches/ipp-20130207/Nebulous-Server/bin/nebdiskd (revision 35235) +++ /branches/eam_branches/ipp-20130207/Nebulous-Server/bin/nebdiskd (revision 35236) @@ -392,5 +392,5 @@ my @data = split(/:/, $df_data); $df_info{"blocks"} = $data[1]; - $df_info{"used"} = $data[2]; + $df_info{"used"} = $data[1]-$data[3]; $log->debug("blocks = " . $df_info{"blocks"} . " / used = " . $df_info{"used"}); return \%df_info; Index: /branches/eam_branches/ipp-20130207/dbconfig/background.md =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/background.md (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/background.md (revision 35236) @@ -5,4 +5,5 @@ chip_bg_id S64 0 chip_id S64 0 + cam_id S64 0 state STR 64 workdir STR 255 Index: /branches/eam_branches/ipp-20130207/dbconfig/cam.md =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/cam.md (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/cam.md (revision 35236) @@ -118,4 +118,5 @@ deteff_uq F32 0 quality S16 0 + background_model S16 0 END Index: /branches/eam_branches/ipp-20130207/dbconfig/changes.txt =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/changes.txt (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/changes.txt (revision 35236) @@ -2267,8 +2267,24 @@ +-- Update camProcessedExp to hold a background_model value + +ALTER TABLE camProcessedExp add column background_model SMALLINT after quality; +UPDATE dbversion set schema_version = '1.1.74', updated= CURRENT_TIMESTAMP(); + + +-- Note the following 3 changes were implemented in the ipp-20121218 tag +-- without changing the schema_version from 1.1.74 + +-- Add cam_id to chipBackgroundRun +ALTER TABLE chipBackgroundRun ADD COLUMN cam_id BIGINT DEFAULT 0 AFTER chip_id; + +-- add fwhm measuements to skycalResult +ALTER TABLE skycalResult ADD COLUMN fwhm_major FLOAT AFTER zpt_stdev; +ALTER TABLE skycalResult ADD COLUMN fwhm_minor FLOAT AFTER fwhm_major; + -- new tables for managing the "release" of ipp data CREATE TABLE survey ( - surveyID INT(8), + surveyID INT, surveyName VARCHAR(16) NOT NULL, description VARCHAR(255), @@ -2277,14 +2293,15 @@ ) ENGINE=InnoDB CHARSET=latin1; - CREATE TABLE ippRelease ( - rel_id INT(8) AUTO_INCREMENT, - surveyID INT(8), - releaseName VARCHAR(64), - release_state VARCHAR(16), -- active, pending archive drop - dataRelease INT(8), -- PSPS dataRelease + rel_id INT AUTO_INCREMENT, + surveyID INT, + release_name VARCHAR(64), + release_state VARCHAR(16), -- active, pending, archive, drop + dataRelease INT, -- PSPS dataRelease + priority INT, PRIMARY KEY(rel_id), - KEY(releaseName), - KEY(state), + KEY(release_name), + KEY(release_state), + KEY(priority), FOREIGN KEY(surveyID) REFERENCES survey(surveyID) ) ENGINE=Innodb DEFAULT CHARSET=latin1; @@ -2292,14 +2309,16 @@ CREATE TABLE relExp ( relexp_id BIGINT AUTO_INCREMENT, - rel_id INT(8), + rel_id INT, exp_id BIGINT, chip_id BIGINT, -- links to the runs that supplied the data cam_id BIGINT, -- for this release + group_id INT, -- id of relGroup that this exposure is contained in state VARCHAR(16), -- released, pending, archive, drop - flags INT UNSIGNED, -- flags for relphot, relastro, ?? + flags INT UNSIGNED, -- flags for relphot, relastro, ?? zpt_obs FLOAT, -- calibrated zero point for this release of zpt_stdev FLOAT, -- this exposure path_base VARCHAR(255), -- path_base of any supporting files for this -- release of this exposure. + fault SMALLINT NOT NULL, registered DATETIME, -- insertion time for this row time_stamp DATETIME, -- time of last update for this row @@ -2307,4 +2326,5 @@ FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), KEY (state), + KEY (fault), FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), @@ -2314,9 +2334,11 @@ CREATE TABLE relStack ( relstack_id BIGINT AUTO_INCREMENT, - rel_id INT(8), + rel_id INT, + stack_id BIGINT, -- id of the stackRun skycal_id BIGINT, -- id of the sky calibration run that supplied the calibration - -- for this release. We get back to stack from here - skycell_id varchar(64), - tess_id varchar(64), + -- for this release of this skycell. + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(16), state VARCHAR(16), -- released, pending, archive, drop flags INT UNSIGNED, @@ -2324,14 +2346,55 @@ zpt_obs FLOAT, -- calibrated zero point for this release of this skycell zpt_stdev FLOAT, - fwhm_major FLOAT, -- seeing as measured by staticsky + mjd_obs INT UNSIGNED, -- for nightly stacks the mjd_obs, zero for deep and reference path_base VARCHAR(255), -- path_base of any supporting files, + fault SMALLINT NOT NULL, registered DATETIME, time_stamp DATETIME, PRIMARY KEY (relstack_id), KEY (tess_id, skycell_id), - KEY (rel_id, tess_id, skycell_id), + KEY (rel_id, tess_id, skycell_id, filter, mjd_obs), + KEY (stack_type), + KEY (mjd_obs), + KEY (state), + KEY (fault), FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), - FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) -) ENGINE=innodb DEFAULT CHARSET=latin1; - -UPDATE dbversion set schema_version = '1.1.74', updated= CURRENT_TIMESTAMP(); + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relGroup ( + group_id INT AUTO_INCREMENT, + rel_id INT, + group_type VARCHAR(16), + lap_id BIGINT, + group_name VARCHAR(16), + state VARCHAR(16), -- new full + label VARCHAR(64), + exp_list_path VARCHAR(255), + fault SMALLINT NOT NULL, + registered DATETIME, + PRIMARY KEY (group_id), + KEY (rel_id, group_name), + KEY (group_type), + KEY (lap_id), + KEY (group_name), + KEY (state), + KEY (label), + KEY (fault), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- new table grouping multi filter lap runs into a group. This is used to +-- determine when all 5 filters for a projection cell have completed processing +CREATE TABLE lapGroup ( + seq_id BIGINT, + projection_cell VARCHAR(64), + state VARCHAR(16), + fault SMALLINT, + PRIMARY KEY(seq_id, projection_cell), + KEY(state), + KEY(fault), + FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +UPDATE dbversion set schema_version = '1.1.75', updated= CURRENT_TIMESTAMP(); + Index: /branches/eam_branches/ipp-20130207/dbconfig/config.md =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/config.md (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/config.md (revision 35236) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.71717171717171 + pkg_version STR 1.1.75 END Index: /branches/eam_branches/ipp-20130207/dbconfig/lap.md =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/lap.md (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/lap.md (revision 35236) @@ -30,2 +30,9 @@ data_state STR 64 # Key END + +lapGroup METADATA + seq_id S64 0 + projection_cell STR 64 + state STR 64 + fault S16 0 +end Index: /branches/eam_branches/ipp-20130207/dbconfig/release.md =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/release.md (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/release.md (revision 35236) @@ -8,5 +8,5 @@ rel_id S32 0 # Primary Key AUTO_INCREMENT surveyID S32 0 # fkey(surveyID) ref survey(surveyID) - releaseName STR 64 + release_name STR 64 release_state STR 16 dataRelease S32 0 @@ -20,4 +20,5 @@ chip_id S64 0 cam_id S64 0 + group_id S32 0 state STR 16 flags U32 0 @@ -25,4 +26,5 @@ zpt_stdev F32 0 path_base STR 255 + fault S16 0 registered TAI NULL time_stamp UTC NULL @@ -32,7 +34,9 @@ relstack_id S64 0 # Primary Key AUTO_INCREMENT rel_id S32 0 # fkey(rel_id) ref ippRelease(rel_id) - skycal_id S64 0 # fkey(skycal_id) ref skycalRun(skycal_id) + stack_id S64 0 # fkey(stack_id) ref stackRun(stack_id) + skycal_id S64 0 skycell_id STR 64 tess_id STR 64 + filter STR 64 state STR 16 flags U32 0 @@ -40,7 +44,21 @@ zpt_obs F32 0 zpt_stdev F32 0 - fwhm_major F32 0 + mjd_obs U32 0 path_base STR 255 + fault S16 0 registered TAI NULL time_stamp TAI NULL END + +relGroup METADATA + group_id S32 0 # Primary Key AUTO_INCREMENT + rel_id S32 0 + group_type STR 16 + lap_id S64 0 + group_name STR 16 + state STR 16 + label STR 64 + exp_list_path STR 255 + fault S16 0 + registered TAI NULL +END Index: /branches/eam_branches/ipp-20130207/dbconfig/staticsky.md =================================================================== --- /branches/eam_branches/ipp-20130207/dbconfig/staticsky.md (revision 35235) +++ /branches/eam_branches/ipp-20130207/dbconfig/staticsky.md (revision 35236) @@ -59,4 +59,6 @@ zpt_obs F32 0.0 zpt_stdev F32 0.0 + fwhm_major F32 0.0 + fwhm_minor F32 0.0 quality S16 0 # Key software_ver STR 16 Index: /branches/eam_branches/ipp-20130207/ippMonitor/Makefile.in =================================================================== --- /branches/eam_branches/ipp-20130207/ippMonitor/Makefile.in (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippMonitor/Makefile.in (revision 35236) @@ -76,5 +76,6 @@ $(DESTWWW)/diskUsage.php \ $(DESTWWW)/version.php \ -$(DESTWWW)/show_log.php +$(DESTWWW)/show_log.php \ +$(DESTWWW)/heathers_plot.php DEFSRC = \ Index: /branches/eam_branches/ipp-20130207/ippMonitor/raw/czartool_getplot.php =================================================================== --- /branches/eam_branches/ipp-20130207/ippMonitor/raw/czartool_getplot.php (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippMonitor/raw/czartool_getplot.php (revision 35236) @@ -34,4 +34,8 @@ else if ($type=="id") $filePath = "$path/ippToPsps_density_".$label.".png"; +else if ($type=="hp_c") +$filePath = "$path/heathers_plots/cam.png"; +else if ($type=="hp_s") +$filePath = "$path/heathers_plots/skycal.png"; Index: /branches/eam_branches/ipp-20130207/ippMonitor/raw/heathers_plot.php =================================================================== --- /branches/eam_branches/ipp-20130207/ippMonitor/raw/heathers_plot.php (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippMonitor/raw/heathers_plot.php (revision 35236) @@ -0,0 +1,28 @@ +\n"; +echo "\n"; +echo "Camera\n"; +echo "SkyCal\n"; +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + +echo "\n"; + +menu_end(); + +?> Index: /branches/eam_branches/ipp-20130207/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- /branches/eam_branches/ipp-20130207/ippMonitor/raw/ipp.imfiles.dat (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippMonitor/raw/ipp.imfiles.dat (revision 35236) @@ -24,4 +24,5 @@ menutop | menutop | link | czartool - ipptopsps | czartool_ipptopsps.php menutop | menutop | link | mask stats | maskStats.php +menutop | menutop | link | Heather's Plots | heathers_plot.php menutop | menutop | plain |   | Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/automate_stacks.pl (revision 35236) @@ -117,5 +117,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks", + -msg => "Choose a mode: --check_registration --check_burntool --queue_chips --queue_stacks --dbname", -exitval => 3, ) unless @@ -124,5 +124,5 @@ defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or defined $test_mode or defined $clean_old or defined $check_mode or - defined $confirm_stacks or defined $burntool_stats; + defined $confirm_stacks or defined $burntool_stats or defined $dbname; # Configurable parameters from our config file. @@ -282,7 +282,4 @@ unless(defined($camera)) { $camera = 'GPC1'; -} -unless(defined($dbname)) { - $dbname = 'gpc1'; } Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/background_chip.pl (revision 35236) @@ -43,5 +43,5 @@ # Parse the command-line arguments my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose, - $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked ); + $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $cam_path_base, $magicked ); GetOptions( 'chip_bg_id=s' => \$chip_bg_id, # chipBackgroundRun identifier @@ -53,4 +53,5 @@ 'threads=s' => \$threads, # Number of threads to use 'chip_path_base=s' => \$chip_path_base, # optional chip_path_base + 'cam_path_base=s' => \$cam_path_base, # optional camera stage path_base 'magicked=s' => \$magicked, # magicked status of input 'verbose' => \$verbose, # Print to stdout @@ -94,4 +95,6 @@ my $dump_config; if ($chip_path_base and $no_update) { + # XXX: this path through the code has not been excercized in awhile and probably does not work + # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config # or do stats. Get path to input from command line. @@ -119,9 +122,15 @@ my $input = $$inputs[0]; # Input of interest $in_path = $input->{path_base}; + $cam_path_base = $input->{cam_path_base}; $magicked = $input->{magicked}; } my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id); -my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +my $in_mask; +if ($cam_path_base) { + $in_mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id); +} else { + $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id); +} my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id); my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id); @@ -177,4 +186,6 @@ my $apply_auxiliary_mask = 1; +# XXX Temporary lookup of auxiliary mask file from static location. +# At some point I'll convert this to register the masks in the database and use detselect if ($apply_auxiliary_mask) { my @auxmask_start_date = qw( @@ -238,5 +249,4 @@ print "Auxililary mask file is $auxmask\n"; } - @@ -272,5 +282,6 @@ &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); - my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP"; + my $resolved_stats = $ipprc->file_resolve($out_stats); + my $command = "$ppStatsFromMetadata $resolved_stats - BACKGROUND_CHIP"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/background_warp.pl (revision 35236) @@ -119,27 +119,6 @@ my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id"; -# XXX: get this from recipe -my $alternate_astrometry_label = 'M31.test.20130206.bgsub'; -# XXX: get exp_id from command line -my $exp_id = 196758; - if (!defined $imageName) { # go find our inputs - my $astrom_path_base; - if (defined $alternate_astrometry_label) { - my $command = "$camtool -processedexp -exp_id $exp_id -label $alternate_astrometry_label"; - $command .= " -dbname $dbname" if $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform camtool -processedexp: $error_code", $warp_bg_id, $error_code); - } - my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR); - my $entry = $entries->[0]; - $astrom_path_base = $entry->{path_base}; - } - my ($imageFile, $maskFile, $weightFile); ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); @@ -174,13 +153,8 @@ $magicked = $file->{magicked}; - if (!$astrom_path_base) { - my $cam_path = $file->{cam_path_base}; - my $astrom = $ipprc->filename($astromSource, $cam_path); - &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; - $astrometry = $astrom; - } elsif (!$astrometry) { - # using alternate astrometry - $astrometry = $ipprc->filename($astromSource, $astrom_path_base); - } + my $cam_path = $file->{cam_path_base}; + my $astrom = $ipprc->filename($astromSource, $cam_path); + &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry; + $astrometry = $astrom; } close $imageFile; @@ -238,5 +212,7 @@ &my_die("Couldn't find expected output file: $out_stats", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); - my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_WARP"; + my $real_stats = $ipprc->file_resolve($out_stats); + + my $command = "$ppStatsFromMetadata $real_stats - BACKGROUND_WARP"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/camera_exp.pl (revision 35236) @@ -379,4 +379,5 @@ check_output($bkgModel, $replicateOutputs); } + $cmdflags = $cmdflags . " -background_model 1 "; } } Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/diff_skycell.pl (revision 35236) @@ -299,4 +299,9 @@ my $jpeg1Name = prepare_output("PPSUB.OUTPUT.JPEG1", $outroot, 1); my $jpeg2Name = prepare_output("PPSUB.OUTPUT.JPEG2", $outroot, 1); + +# do we always need to prepare this file? +if ($reduction ne 'NOCONVDIFF') { + my $refConv = prepare_output("PPSUB.REF.CONV", $outroot, 1); +} my ($inverseName, $inverseMask, $inverseVariance, $inverseSources); Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/ipp_cleanup.pl (revision 35236) @@ -1957,4 +1957,5 @@ addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id); addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPBACKGROUND.OUTPUT.VARIANCE", $path_base, $class_id); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode @@ -2100,4 +2101,5 @@ addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id); addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename (\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/publish_file.pl (revision 35236) @@ -92,4 +92,6 @@ my $comment; # Comment for exposure +my $exp_name_1; # Name of exposure 1 +my $exp_name_2; # Name of exposure 2 if ($stage eq 'camera') { my $command = "camtool -processedexp -cam_id $stage_id"; @@ -168,5 +170,5 @@ my %negative; # Data for negative diff detections foreach my $comp ( @$components ) { - my $path_base = $comp->{path_base}; # Base name for file + my $path_base = $comp->{path_base}; # Base name for file if (!$need_magic and $comp->{magicked}) { # This client is authorized to receive uncensored detections @@ -178,4 +180,6 @@ print "Warning: mis-matched comments\n" if defined $comment and $comment ne $comp->{comment}; $comment = $comp->{comment} unless defined $comment; + $exp_name_1 = $comp->{exp_name_1} unless defined $exp_name_1; + $exp_name_2 = $comp->{exp_name_2} unless defined $exp_name_2; (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time}; @@ -192,49 +196,49 @@ &my_die("input file does not exist: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$ipprc->file_exists($filename); my $resolved = $ipprc->file_resolve($filename); - + &my_die("unable to resolve input file: $filename", $pub_id, $PS_EXIT_SYS_ERROR) if !$resolved; $filename = $resolved; - my $cam_id = $comp->{cam_id_1}; - #print "Getting info from camera stage $cam_id\n"; - my $cam_command = "$camtool -processedexp -cam_id $cam_id"; - $cam_command .= " -dbname $dbname" if defined $dbname; - my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) = - run(command => $cam_command, verbose => $verbose); - &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success; - my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or - &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); - my $cam_components = parse_md_list($cam_metadata) or - &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); - &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1; - my $cam_comp = $$cam_components[0]; - - my $exp_id = $comp->{exp_id_1}; - #print "Getting info from raw stage $exp_id\n"; - my $reg_command = "$regtool -processedimfile -exp_id $exp_id"; - $reg_command .= " -dbname $dbname" if defined $dbname; - my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) = - run(command => $reg_command, verbose => $verbose); - &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success; - my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or - &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); - my $reg_components = parse_md_list($reg_metadata) or - &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); - my $reg_comp = $$reg_components[0]; - - # Now get the difftype value from diff->{diff_mode} - my $difftype = ""; - if ($comp->{diff_mode} == 1) { - $difftype = "WW"; - } elsif ($comp->{diff_mode} == 2) { - $difftype = "WS"; - } elsif ($comp->{diff_mode} == 3) { - $difftype = "SW"; # Not used yet - } elsif ($comp->{diff_mode} == 4) { - $difftype = "SS"; - } else { - $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]"; - } + my $cam_id = $comp->{cam_id_1}; + #print "Getting info from camera stage $cam_id\n"; + my $cam_command = "$camtool -processedexp -cam_id $cam_id"; + $cam_command .= " -dbname $dbname" if defined $dbname; + my ( $cam_success, $cam_error_code, $cam_full_buf, $cam_stdout_buf, $cam_stderr_buf ) = + run(command => $cam_command, verbose => $verbose); + &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $cam_success; + my $cam_metadata = $mdcParser->parse(join "", @$cam_stdout_buf) or + &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); + my $cam_components = parse_md_list($cam_metadata) or + &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + &my_die("More than one entry for cam_id $stage_id", $pub_id, $PS_EXIT_PROG_ERROR) if scalar @$cam_components > 1; + my $cam_comp = $$cam_components[0]; + + my $exp_id = $comp->{exp_id_1}; + #print "Getting info from raw stage $exp_id\n"; + my $reg_command = "$regtool -processedimfile -exp_id $exp_id"; + $reg_command .= " -dbname $dbname" if defined $dbname; + my ( $reg_success, $reg_error_code, $reg_full_buf, $reg_stdout_buf, $reg_stderr_buf ) = + run(command => $reg_command, verbose => $verbose); + &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $reg_success; + my $reg_metadata = $mdcParser->parse(join "", @$reg_stdout_buf) or + &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); + my $reg_components = parse_md_list($reg_metadata) or + &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + my $reg_comp = $$reg_components[0]; + + # Now get the difftype value from diff->{diff_mode} + my $difftype = ""; + if ($comp->{diff_mode} == 1) { + $difftype = "WW"; + } elsif ($comp->{diff_mode} == 2) { + $difftype = "WS"; + } elsif ($comp->{diff_mode} == 3) { + $difftype = "SW"; # Not used yet + } elsif ($comp->{diff_mode} == 4) { + $difftype = "SS"; + } else { + $difftype = "Unsupported diff_mode value: [" . $comp->{diff_mode} . "]"; + } my $data = { zp => $zp, @@ -250,12 +254,12 @@ output_format => $comp->{output_format}, direction => 1, - comment => $reg_comp->{comment}, - obsmode => $reg_comp->{obs_mode}, - difftype => $difftype, - sky => $cam_comp->{bg}, - shutoutc => $reg_comp->{dateobs}, + comment => $reg_comp->{comment}, + obsmode => $reg_comp->{obs_mode}, + difftype => $difftype, + sky => $cam_comp->{bg}, + shutoutc => $reg_comp->{dateobs}, }; - #warn("Checking for positive"); + #warn("Checking for positive"); diff_check(\%positive, $data, "positive"); @@ -288,12 +292,12 @@ diff_id => $comp->{diff_id}, direction => 0, - comment => $reg_comp->{comment}, - obsmode => $reg_comp->{obs_mode}, - difftype => $difftype, - sky => $cam_comp->{bg}, - shutoutc => $reg_comp->{dateobs}, + comment => $reg_comp->{comment}, + obsmode => $reg_comp->{obs_mode}, + difftype => $difftype, + sky => $cam_comp->{bg}, + shutoutc => $reg_comp->{dateobs}, }; - #warn("Checking for negative"); + #warn("Checking for negative"); diff_check(\%negative, $data, "negative"); @@ -326,4 +330,6 @@ my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $dsType --list $dsFileName"; $command .= " --ps0 \"$comment\"" if defined $comment; + $command .= " --ps1 \"$exp_name_1\"" if defined $exp_name_1; + $command .= " --ps2 \"$exp_name_2\"" if defined $exp_name_2; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/skycalibration.pl (revision 35236) @@ -231,5 +231,6 @@ $command .= " -F PSASTRO.OUTPUT $output_sources_filerule"; $command .= " -stats $stats"; - $command .= " -recipe PPSTATS CAMSTATS"; + # $command .= " -recipe PPSTATS CAMSTATS"; + $command .= " -recipe PPSTATS SKYCALSTATS"; $command .= " -dumpconfig $configuration" if $configuration; $command .= " -tracedest $traceDest"; Index: /branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl =================================================================== --- /branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippScripts/scripts/warp_skycell.pl (revision 35236) @@ -245,6 +245,12 @@ print $weightFile "$weight\n"; my $bkg; - if ($doBackground) { - $bkg = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id}); + if (($doBackground)) { + # &&($imfile->{background_model} == 1) + if ($imfile->{cam_background_model}) { + $bkg = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{cam_path_base}, $imfile->{class_id}); + } + else { + $bkg = $ipprc->filename("PPIMAGE.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id}); + } print $bkgFile "$bkg\n"; } Index: /branches/eam_branches/ipp-20130207/ippTasks/background.pro =================================================================== --- /branches/eam_branches/ipp-20130207/ippTasks/background.pro (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTasks/background.pro (revision 35236) @@ -241,5 +241,5 @@ else # save the DB name for the exit tasks - option $DB:$chip_advance_DB + option $DB:$bg_chip_advance_DB $run = $run -dbname $DB:$bg_chip_advance_DB $bg_chip_advance_DB ++ @@ -446,5 +446,4 @@ periods -poll $LOADPOLL -# periods -exec $LOADEXEC periods -exec 30 periods -timeout 60 @@ -461,5 +460,5 @@ else # save the DB name for the exit tasks - option $DB:$warp_advance_DB + option $DB:$bg_warp_advance_DB $run = $run -dbname $DB:$bg_warp_advance_DB $bg_warp_advance_DB ++ Index: /branches/eam_branches/ipp-20130207/ippTasks/burntool.pro =================================================================== --- /branches/eam_branches/ipp-20130207/ippTasks/burntool.pro (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTasks/burntool.pro (revision 35236) @@ -1,4 +1,4 @@ ## burntool.pro : -*- sh -*- - +## This task is designed to only be used when re-burntooling data. It is not necessary for standard registration. check.globals @@ -161,4 +161,5 @@ periods -exec $LOADEXEC periods -timeout 30 + active false npending 1 @@ -216,4 +217,5 @@ periods -exec $LOADEXEC periods -timeout 300 + active false npending 1 @@ -281,4 +283,5 @@ periods -exec $LOADEXEC periods -timeout 30 + active false npending 1 @@ -336,4 +339,5 @@ periods -exec $RUNEXEC periods -timeout 10800 + active false npending 30 task.exec Index: /branches/eam_branches/ipp-20130207/ippTasks/summit.copy.pro =================================================================== --- /branches/eam_branches/ipp-20130207/ippTasks/summit.copy.pro (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTasks/summit.copy.pro (revision 35236) @@ -6,6 +6,8 @@ # pztool -adddatastore -inst isp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/skyprobe/index.txt +# pztool -adddatastore -inst ssp -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/ssp/index.txt # pztool -adddatastore -inst gpc1 -telescope ps1 -uri http://conductor/ds/gpc1/index.txt # pztool -adddatastore -inst allskycam -telescope ps1 -uri http://otis1.ifa.hawaii.edu/ds/allskycam/index.txt + # NOTE: workdir / volume mangling and nebulous. these tasks copy the Index: /branches/eam_branches/ipp-20130207/ippTools/configure.ac =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/configure.ac (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/configure.ac (revision 35236) @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([ipptools], [1.1.73], [ipp-support@ifa.hawaii.edu]) +AC_INIT([ipptools], [1.1.75], [ipp-support@ifa.hawaii.edu]) AC_CONFIG_SRCDIR([autogen.sh]) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.71]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.75]) PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) Index: /branches/eam_branches/ipp-20130207/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/Makefile.am (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/Makefile.am (revision 35236) @@ -155,4 +155,5 @@ difftool_completed_runs.sql \ difftool_coalesce_run.sql \ + difftool_definewarpstack.sql \ difftool_definewarpstack_part1.sql \ difftool_definewarpstack_part2.sql \ @@ -463,6 +464,14 @@ sctool_list.sql \ releasetool_listsurvey.sql \ + releasetool_tocalibexp.sql \ releasetool_listrelease.sql \ releasetool_listrelexp.sql \ - releasetool_definerelexp.sql + releasetool_definerelexp.sql \ + releasetool_definerelstack.sql \ + releasetool_definerelstack_with_skycal.sql \ + releasetool_definerelgroup_select_lap.sql \ + releasetool_definerelgroup_select_data_group.sql \ + releasetool_definerelgroup_select_exp_data_group.sql \ + releasetool_definerelgroup_select_exp_lap.sql \ + releasetool_pendingrelgroup.sql Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chip.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chip.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chip.sql (revision 35236) @@ -5,4 +5,6 @@ chipBackgroundRun.label, chipBackgroundRun.data_group, + chipBackgroundRun.chip_id, + chipBackgroundRun.cam_id, rawExp.exp_id, rawExp.exp_name, @@ -12,6 +14,5 @@ rawExp.ra, rawExp.decl, - rawExp.exp_time, - chipRun.chip_id + rawExp.exp_time FROM chipBackgroundRun JOIN chipBackgroundImfile USING(chip_bg_id) Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chipinputs.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chipinputs.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_chipinputs.sql (revision 35236) @@ -1,7 +1,9 @@ SELECT - chipProcessedImfile.* + chipProcessedImfile.*, + camProcessedExp.path_base AS cam_path_base FROM chipBackgroundRun JOIN chipRun USING(chip_id) JOIN chipProcessedImfile USING(chip_id) +LEFT JOIN camProcessedExp ON chipBackgroundRun.cam_id = camProcessedExp.cam_id WHERE chipRun.state = 'full' AND chipProcessedImfile.fault = 0 Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definechip.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definechip.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definechip.sql (revision 35236) @@ -1,6 +1,12 @@ SELECT - chipRun.* + chipRun.*, + camRun.cam_id, + chip_bg_id, + CURRENT_TIMESTAMP AS registered FROM chipRun JOIN rawExp USING(exp_id) -LEFT JOIN chipBackgroundRun USING(chip_id) -WHERE chipRun.state = 'full' +JOIN chipRun as altChipRun USING(exp_id) +JOIN camRun ON altChipRun.chip_id = camRun.chip_id +JOIN camProcessedExp USING(cam_id) +LEFT JOIN chipBackgroundRun ON chipBackgroundRun.chip_id = chipRun.chip_id -- labelHook %s +WHERE chipRun.state = 'full' AND camRun.state ='full' AND camProcessedExp.quality = 0 Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definewarp.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definewarp.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_definewarp.sql (revision 35236) @@ -1,12 +1,19 @@ SELECT - warpRun.*, - chipBackgroundRun.chip_bg_id + warpRun.warp_id, + chipBackgroundRun.* + -- the following items are selected for aid in debugging + , + warpsChipRun.chip_id as warpschip_id, + chipRun.chip_id AS thischip_id, + warpBackgroundRun.warp_bg_id FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN chipBackgroundRun USING(chip_id) +JOIN chipRun as warpsChipRun ON camRun.chip_id = warpsChipRun.chip_id JOIN rawExp USING(exp_id) -LEFT JOIN warpBackgroundRun USING(chip_bg_id) +JOIN chipRun USING(exp_id) +JOIN chipBackgroundRun ON chipRun.chip_id = chipBackgroundRun.chip_id + AND (chipBackgroundRun.cam_id = camRun.cam_id OR chipBackgroundRun.cam_id = 0) +LEFT JOIN warpBackgroundRun ON chipBackgroundRun.chip_bg_id = warpBackgroundRun.warp_bg_id -- label hook %s WHERE chipBackgroundRun.state = 'full' - AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed + AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- need warp to have completed so warpSkyCellMap is populated Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_tochip.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_tochip.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_tochip.sql (revision 35236) @@ -16,3 +16,3 @@ AND (Label.active OR Label.active IS NULL) -- WHERE hook %s -ORDER BY priority DESC, chip_bg_id +ORDER BY priority DESC, chip_bg_id, class_id Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_towarp.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_towarp.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_towarp.sql (revision 35236) @@ -24,3 +24,3 @@ AND (Label.active OR Label.active IS NULL) -- WHERE hook %s -ORDER BY priority DESC, chip_bg_id +ORDER BY priority DESC, warp_bg_id, skycell_id Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warp.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warp.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warp.sql (revision 35236) @@ -2,6 +2,10 @@ warpBackgroundSkyfile.*, warpBackgroundRun.state, + warpBackgroundRun.warp_id, + camRun.cam_id, + warpBackgroundRun.chip_bg_id, warpBackgroundRun.workdir, warpBackgroundRun.label, + warpRun.label as warp_label, rawExp.exp_id, rawExp.exp_name, Index: /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warpinputs.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warpinputs.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/bgtool_warpinputs.sql (revision 35236) @@ -11,5 +11,5 @@ JOIN chipBackgroundImfile USING(chip_bg_id, class_id) JOIN fakeRun USING(fake_id) -JOIN camProcessedExp USING(cam_id) +JOIN camProcessedExp ON camProcessedExp.cam_id = fakeRun.cam_id WHERE warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed AND warpSkyfile.fault = 0 Index: /branches/eam_branches/ipp-20130207/ippTools/share/difftool_definewarpstack.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/difftool_definewarpstack.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/difftool_definewarpstack.sql (revision 35236) @@ -1,58 +1,42 @@ --- Get list of warps that can be diffed, with any associated diff, --- and the best stack to use as a template --- Warps without an existing diff can be identified by a NULL diff_id +-- Get list of warp-stack pairs that can be diffed +-- and check results against existing diffs SELECT - warpsToDiff.warp_id, - warpsToDiff.skycell_id, - warpsToDiff.tess_id, - warpsToDiff.filter, - warpsToDiff.good_frac, - warpsToDiff.diff_id, - current_stack_id, - best_stack_id, - exp_id -FROM ( - -- Get list of warps that can be diffed, with any associated diff - SELECT - warpSkyfile.warp_id, - warpSkyfile.skycell_id, - warpSkyfile.tess_id, - warpSkyfile.good_frac, - warpRun.label, - filter, - warpRun.label as warp_label, - diffInputs.diff_id, - diffInputs.stack2 AS current_stack_id, - rawExp.exp_id - FROM warpSkyfile - JOIN warpRun USING(warp_id) - JOIN fakeRun USING(fake_id) - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp USING(exp_id) - -- Check if it has an associated diff - LEFT JOIN diffInputSkyfile AS diffInputs - ON diffInputs.warp1 = warpSkyfile.warp_id - AND diffInputs.skycell_id = warpSkyfile.skycell_id - AND diffInputs.stack2 IS NOT NULL - WHERE - warpSkyfile.fault = 0 - AND warpSkyfile.quality = 0 - -- warpsToDiff WHERE hook %s - ) AS warpsToDiff --- Get best stack as a function of skycell_id, filter -JOIN ( - SELECT - MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment - skycell_id, - filter - FROM stackRun - JOIN stackSumSkyfile USING(stack_id) - WHERE stackRun.state = 'full' - AND stackSumSkyfile.fault = 0 - AND stackSumSkyfile.quality = 0 - -- stacksForDiff WHERE hook %s - GROUP BY - skycell_id, - filter - ) AS stacksForDiff USING(skycell_id, filter) + exp_id, + warp_id, + rawExp.filter, + warpRun.label AS warpLabel, + warpRun.data_group AS warpDataGroup, + warpRun.tess_id, + warpSkyfile.skycell_id, + MAX(stack_id) AS stack_id, + stackRun.label AS stackLabel, + stackRun.data_group AS stackDataGroup, + diff_id, + diffRun.label AS diffLabel +FROM warpRun + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN warpSkyfile USING(warp_id) + JOIN stackRun ON + (stackRun.skycell_id = warpSkyfile.skycell_id AND + stackRun.filter = rawExp.filter AND + stackRun.tess_id = warpRun.tess_id) + JOIN stackSumSkyfile USING(stack_id) + LEFT JOIN diffInputSkyfile ON + (warp_id = diffInputSkyfile.warp1 AND + stack_id = diffInputSkyfile.stack2 AND + diffInputSkyfile.skycell_id = stackRun.skycell_id AND + diffInputSkyfile.tess_id = stackRun.tess_id) + LEFT JOIN diffRun USING(diff_id) +WHERE + warpRun.state = 'full' + AND stackRun.state = 'full' + AND warpSkyfile.fault = 0 + AND warpSkyfile.quality = 0 + AND stackSumSkyfile.fault = 0 + AND stackSumSkyfile.quality = 0 + AND exp_id IS NOT NULL +-- %s +GROUP BY exp_id,warp_id,skycell_id Index: /branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_create_tables.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_create_tables.sql (revision 35236) @@ -521,4 +521,5 @@ deteff_uq FLOAT, quality SMALLINT NOT NULL DEFAULT 0, + background_model SMALLINT, PRIMARY KEY(cam_id), KEY(fault), @@ -1791,4 +1792,5 @@ chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier chip_id BIGINT NOT NULL, -- link to chipRun + cam_id BIGINT DEFAULT 0, -- optional link to camRun state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) workdir VARCHAR(255) NOT NULL, -- working directory @@ -1843,5 +1845,5 @@ data_group VARCHAR(64), -- group for data dist_group VARCHAR(64), -- group for distribution - reduction VARCHAR(64), -- reduction class (for altering recipe) + reduction VARCHAR(64), -- reduction class (for altering recipe) note VARCHAR(255), -- note registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered @@ -2122,4 +2124,6 @@ zpt_obs FLOAT, zpt_stdev FLOAT, + fwhm_major FLOAT, + fwhm_minor FLOAT, quality SMALLINT NOT NULL, hostname VARCHAR(64) NOT NULL, @@ -2149,22 +2153,21 @@ CREATE TABLE survey ( - surveyID INT(8), + surveyID INT, surveyName VARCHAR(16) NOT NULL, description VARCHAR(255), PRIMARY KEY(surveyID), - UNIQUE KEY(name) + UNIQUE KEY(surveyName) ) ENGINE=InnoDB CHARSET=latin1; - CREATE TABLE ippRelease ( - rel_id INT(8) AUTO_INCREMENT, - surveyID INT(8), - releaseName VARCHAR(64), - relase_state VARCHAR(16), -- active, pending, archive, drop - dataRelease INT(8), -- PSPS dataRelease - priority INT(8), + rel_id INT AUTO_INCREMENT, + surveyID INT, + release_name VARCHAR(64), + release_state VARCHAR(16), -- active, pending, archive, drop + dataRelease INT, -- PSPS dataRelease + priority INT, PRIMARY KEY(rel_id), - KEY(releaseName), - KEY(state), + KEY(release_name), + KEY(release_state), KEY(priority), FOREIGN KEY(surveyID) REFERENCES survey(surveyID) @@ -2173,14 +2176,16 @@ CREATE TABLE relExp ( relexp_id BIGINT AUTO_INCREMENT, - rel_id INT(8), + rel_id INT, exp_id BIGINT, chip_id BIGINT, -- links to the runs that supplied the data cam_id BIGINT, -- for this release + group_id INT, -- id of relGroup that this exposure is contained in state VARCHAR(16), -- released, pending, archive, drop - flags INT UNSIGNED, -- flags for relphot, relastro, ?? + flags INT UNSIGNED, -- flags for relphot, relastro, ?? zpt_obs FLOAT, -- calibrated zero point for this release of zpt_stdev FLOAT, -- this exposure path_base VARCHAR(255), -- path_base of any supporting files for this -- release of this exposure. + fault SMALLINT NOT NULL, registered DATETIME, -- insertion time for this row time_stamp DATETIME, -- time of last update for this row @@ -2188,4 +2193,5 @@ FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), KEY (state), + KEY (fault), FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id), FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id), @@ -2195,9 +2201,11 @@ CREATE TABLE relStack ( relstack_id BIGINT AUTO_INCREMENT, - rel_id INT(8), + rel_id INT, + stack_id BIGINT, -- id of the stackRun skycal_id BIGINT, -- id of the sky calibration run that supplied the calibration - -- for this release. We get back to stack from here - skycell_id varchar(64), - tess_id varchar(64), + -- for this release of this skycell. + skycell_id VARCHAR(64), + tess_id VARCHAR(64), + filter VARCHAR(16), state VARCHAR(16), -- released, pending, archive, drop flags INT UNSIGNED, @@ -2205,13 +2213,51 @@ zpt_obs FLOAT, -- calibrated zero point for this release of this skycell zpt_stdev FLOAT, - fwhm_major FLOAT, -- seeing as measured by staticsky + mjd_obs INT UNSIGNED, -- for nightly stacks the mjd_obs, zero for deep and reference path_base VARCHAR(255), -- path_base of any supporting files, + fault SMALLINT NOT NULL, registered DATETIME, time_stamp DATETIME, PRIMARY KEY (relstack_id), KEY (tess_id, skycell_id), - KEY (rel_id, tess_id, skycell_id), + KEY (rel_id, tess_id, skycell_id, filter, mjd_obs), + KEY (stack_type), + KEY (mjd_obs), + KEY (state), + KEY (fault), FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), - FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE relGroup ( + group_id INT AUTO_INCREMENT, + rel_id INT, + group_type VARCHAR(16), + lap_id BIGINT, + group_name VARCHAR(16), + state VARCHAR(16), -- new full + label VARCHAR(64), + exp_list_path VARCHAR(255), + fault SMALLINT NOT NULL, + registered DATETIME, + PRIMARY KEY (group_id), + KEY (rel_id, group_name), + KEY (group_type), + KEY (lap_id), + KEY (group_name), + KEY (state), + KEY (label), + KEY (fault), + FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE lapGroup ( + seq_id BIGINT, + projection_cell VARCHAR(64), + state VARCHAR(16), + fault SMALLINT, + PRIMARY KEY(seq_id, projection_cell), + KEY(state), + KEY(fault), + FOREIGN KEY(seq_id) REFERENCES lapSequence(seq_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_drop_tables.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/pxadmin_drop_tables.sql (revision 35236) @@ -109,7 +109,10 @@ DROP TABLE IF EXISTS skycalResult; DROP TABLE IF EXISTS skycell; +DROP TABLE IF EXISTS relStack; DROP TABLE IF EXISTS relExp; +DROP TABLE IF EXISTS relGroup; DROP TABLE IF EXISTS ippRelease; DROP TABLE IF EXISTS survey; +DROP TABLE IF EXISTS lapGroup; SET FOREIGN_KEY_CHECKS=1 Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelexp.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelexp.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelexp.sql (revision 35236) @@ -1,5 +1,5 @@ SELECT ippRelease.rel_id, - ippRelease.releaseName, + ippRelease.release_name, rawExp.exp_name, rawExp.filter, Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_data_group.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_data_group.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_data_group.sql (revision 35236) @@ -0,0 +1,9 @@ +SELECT + ippRelease.rel_id, + count(exp_id) AS num_exposures +FROM ippRelease + LEFT JOIN relExp using(rel_id) + LEFT JOIN camRun using(cam_id) + WHERE relExp.group_id = 0 + AND relExp.state != 'drop' + Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_exp_data_group.sql (revision 35236) @@ -0,0 +1,6 @@ +SELECT relExp.relexp_id +FROM relExp + JOIN camRun USING(cam_id) +WHERE + relExp.group_id = 0 + AND relExp.state != 'drop' Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_exp_lap.sql (revision 35236) @@ -0,0 +1,4 @@ +SELECT relExp.relexp_id +FROM relGroup JOIN lapRun USING(lap_id) + JOIN relExp using(rel_id, exp_id) +WHERE relExp.group_id = 0 Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_lap.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_lap.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelgroup_select_lap.sql (revision 35236) @@ -0,0 +1,11 @@ +SELECT + lapRun.seq_id, + lapRun.lap_id, + ippRelease.rel_id +FROM lapRun + JOIN ippRelease + LEFT JOIN relGroup USING(lap_id) +WHERE (lapRun.state = 'done' or lapRun.state = 'full') + AND relGroup.group_id IS NULL +-- AND lapRun.seq_id = xxx + Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelstack.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelstack.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelstack.sql (revision 35236) @@ -0,0 +1,16 @@ +SELECT + ippRelease.rel_id, + stackRun.stack_id, + 0 as skycal_id, + stackRun.skycell_id, + stackRun.tess_id, + stackRun.filter, + 0 AS mjd_obs, + 0 AS zpt_obs, + 0 AS zpt_stdev + +FROM ippRelease +JOIN stackRun +LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) + +WHERE previousRelStack.relstack_id IS NULL Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelstack_with_skycal.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelstack_with_skycal.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_definerelstack_with_skycal.sql (revision 35236) @@ -0,0 +1,19 @@ +SELECT + ippRelease.rel_id, + stackRun.stack_id, + IFNULL(skycalRun.skycal_id, 0) as skycal_id, + stackRun.skycell_id, + stackRun.tess_id, + stackRun.filter, + stackSumSkyfile.mjd_obs, + skycalResult.zpt_obs, + skycalResult.zpt_stdev + +FROM ippRelease +JOIN stackRun +JOIN stackSumSkyfile USING(stack_id) +LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) +JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id +JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id + +WHERE previousRelStack.relstack_id IS NULL Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_listrelexp.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_listrelexp.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_listrelexp.sql (revision 35236) @@ -2,5 +2,6 @@ relexp_id, relExp.state, - ippRelease.releaseName, + relExp.fault, + ippRelease.release_name, ippRelease.release_state, ippRelease.rel_id, @@ -21,4 +22,5 @@ camProcessedExp.path_base as cam_path_base, camProcessedExp.fwhm_major, + camProcessedExp.fwhm_minor, warpRun.warp_id, warpRun.state as warp_state, Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_pendingrelgroup.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_pendingrelgroup.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_pendingrelgroup.sql (revision 35236) @@ -0,0 +1,4 @@ +SELECT relGroup.* +FROM relGroup + JOIN ippRelease USING(rel_id) +WHERE relGroup.state = 'new' Index: /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_tocalibexp.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_tocalibexp.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/releasetool_tocalibexp.sql (revision 35236) @@ -0,0 +1,5 @@ +SELECT relExp.*, +ippRelease.release_name, +ippRelease.release_state +FROM relExp JOIN ippRelease USING(rel_id) +WHERE relExp.state = 'goto_calib' AND relExp.fault = 0 Index: /branches/eam_branches/ipp-20130207/ippTools/share/reltool_pendinggroup.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/reltool_pendinggroup.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/ippTools/share/reltool_pendinggroup.sql (revision 35236) @@ -0,0 +1,8 @@ +-- relGroups to process +SELECT relGroup.*, + ippRelease.release_id, + ippRelease.release_name +FROM relGroup + JOIN ippRelease USING(rel_id) +WHERE relGroup.state ='new' + AND relGroup.fault = 0 Index: /branches/eam_branches/ipp-20130207/ippTools/share/warptool_scmap.sql =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/share/warptool_scmap.sql (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/share/warptool_scmap.sql (revision 35236) @@ -5,4 +5,5 @@ camProcessedExp.path_base as cam_path_base, camProcessedExp.fault as cam_fault, + camProcessedExp.background_model AS cam_background_model, chipProcessedImfile.chip_id, chipRun.state, Index: /branches/eam_branches/ipp-20130207/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/addtool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/addtool.c (revision 35236) @@ -988,13 +988,14 @@ if (strcmp (stage,"cam") == 0) { query = pxDataGet("addtool_find_processedexp_cam.sql"); - } - if (strcmp (stage,"stack") == 0) { + } else if (strcmp (stage,"stack") == 0) { query = pxDataGet("addtool_find_processedexp_stack.sql"); - } - if (strcmp (stage,"staticsky") == 0) { + } else if (strcmp (stage,"staticsky") == 0) { query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql"); - } - if (strcmp (stage,"skycal") == 0) { + } else if (strcmp (stage,"skycal") == 0) { query = pxDataGet("addtool_find_processedexp_skycal.sql"); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "can't find sql file for stage %s", stage); + return false; } Index: /branches/eam_branches/ipp-20130207/ippTools/src/addtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/addtoolConfig.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/addtoolConfig.c (revision 35236) @@ -125,5 +125,5 @@ psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); - psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-stage", 0, "set the stage", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-stage", 0, "set the stage", "cam"); pxcamSetSearchArgs(processedexpArgs); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); Index: /branches/eam_branches/ipp-20130207/ippTools/src/bgtool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/bgtool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/bgtool.c (revision 35236) @@ -285,4 +285,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); @@ -324,5 +325,6 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); - pxAddLabelSearchArgs(config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-cam_label", "camRun.label", "=="); if (!psListLength(where->list)) { @@ -343,4 +345,10 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_STR(cam_label, config->args, "-cam_label", false, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", false, false); + if (!cam_id && !cam_label) { + psError(PXTOOLS_ERR_CONFIG, true, "either cam_id or cam_label is required"); + return false; + } // Get chip runs to promote to chipBackgroundRun @@ -364,24 +372,11 @@ } + psString labelHook = psStringCopy(""); if (!rerun) { - psStringAppend(&query, "\nAND chipBackgroundRun.chip_bg_id IS NULL"); if (label) { - psStringAppend(&query, - "\nAND (chipBackgroundRun.label = '%s'" - " OR chipBackgroundRun.label IS NULL)", - label); - } - if (data_group) { - psStringAppend(&query, - "\nAND (chipBackgroundRun.data_group = '%s'" - " OR chipBackgroundRun.data_group IS NULL)", - data_group); - } - if (dist_group) { - psStringAppend(&query, - "\nAND (chipBackgroundRun.dist_group = '%s'" - " OR chipBackgroundRun.dist_group IS NULL)", - dist_group); - } + // check for run with the newly specified label + psStringAppend(&labelHook, "\nAND (chipBackgroundRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND chip_bg_id IS NULL"); } @@ -391,7 +386,8 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { psError(psErrorCodeLast(), false, "database error"); psFree(query); + psFree(labelHook); if (!psDBRollback(config->dbh)) { psError(psErrorCodeLast(), false, "database error"); @@ -400,4 +396,5 @@ } psFree(query); + psFree(labelHook); psArray *output = p_psDBFetchResult(config->dbh); // Matching rows @@ -420,5 +417,5 @@ if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -435,5 +432,5 @@ psMetadata *md = output->data[i]; - chipRunRow *row = chipRunObjectFromMetadata(md); + chipBackgroundRunRow *row = chipBackgroundRunObjectFromMetadata(md); if (!row) { psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); @@ -445,5 +442,5 @@ } - if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, "new", + if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, row->cam_id, "new", workdir ? workdir : row->workdir, label ? label : row->label, @@ -560,5 +557,5 @@ if (psArrayLength(output)) { - if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "pendingchipBackgroundRun", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -621,5 +618,5 @@ return true; } - if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "pendingchipBackgroundImfile", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -1295,5 +1292,6 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); pxAddLabelSearchArgs(config, where, "-warp_label", "warpRun.label", "=="); - pxAddLabelSearchArgs(config, where, "-chip_label", "chipBackgroundRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-chip_bg_label", "chipBackgroundRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-chip_label", "chipRun.label", "=="); if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -1335,24 +1333,11 @@ } + psString labelHook = psStringCopy(""); if (!rerun) { - psStringAppend(&query, "\nAND warpBackgroundRun.warp_bg_id IS NULL"); if (label) { - psStringAppend(&query, - "\nAND (warpBackgroundRun.label = '%s'" - " OR warpBackgroundRun.label IS NULL)", - label); - } - if (data_group) { - psStringAppend(&query, - "\nAND (warpBackgroundRun.data_group = '%s'" - " OR warpBackgroundRun.data_group IS NULL)", - data_group); - } - if (dist_group) { - psStringAppend(&query, - "\nAND (warpBackgroundRun.dist_group = '%s'" - " OR warpBackgroundRun.dist_group IS NULL)", - dist_group); - } + // check for run with the newly specified label + psStringAppend(&labelHook, "\nAND (warpBackgroundRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND warp_bg_id IS NULL"); } @@ -1362,6 +1347,7 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(psErrorCodeLast(), false, "database error"); + if (!p_psDBRunQueryF(config->dbh, query, labelHook)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(labelHook); psFree(query); if (!psDBRollback(config->dbh)) { @@ -1370,4 +1356,5 @@ return false; } + psFree(labelHook); psFree(query); @@ -1391,5 +1378,5 @@ if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); @@ -1408,7 +1395,8 @@ psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id"); - warpRunRow *row = warpRunObjectFromMetadata(md); + + warpBackgroundRunRow *row = warpBackgroundRunObjectFromMetadata(md); if (!row) { - psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psError(psErrorCodeLast(), false, "failed to convert metadata into warpRun"); psFree(output); if (!psDBRollback(config->dbh)) { @@ -1665,7 +1653,9 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); - pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpBackgroundRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "warpBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); pxAddLabelSearchArgs(config, where, "-data_group", "warpBackgroundRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, where, "-dist_group", "warpBackgroundRun.data_group", "LIKE"); Index: /branches/eam_branches/ipp-20130207/ippTools/src/bgtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/bgtoolConfig.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/bgtoolConfig.c (revision 35236) @@ -50,4 +50,5 @@ psMetadata *definechipArgs = psMetadataAlloc(); psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0); + psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-cam_id", 0, "choose cam_id", 0); psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); @@ -89,5 +90,6 @@ psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); - psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", 0, "search on chipRun label", NULL); + psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-cam_label", 0, "search on camRun label", NULL); psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-rerun", 0, "re-run data?", false); @@ -279,5 +281,6 @@ psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL); - psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL); + psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_bg_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL); psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false); @@ -352,4 +355,6 @@ psMetadata *warpArgs = psMetadataAlloc(); psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0); + psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0); + psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0); psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL); psMetadataAddS16(warpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); Index: /branches/eam_branches/ipp-20130207/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/camtool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/camtool.c (revision 35236) @@ -500,5 +500,6 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - + PXOPT_LOOKUP_S16(background_model, config->args, "-background_model", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); @@ -520,5 +521,5 @@ PXOPT_LOOKUP_F32(maskfrac_max_magic, config->args, "-maskfrac_max_magic", false, false); PXOPT_LOOKUP_F32(maskfrac_max_advisory, config->args, "-maskfrac_max_advisory", false, false); - + // we store actual detection efficiency by adding in zpt_obs PXOPT_LOOKUP_F32(deteff_inst, config->args, "-deteff_inst", false, false); @@ -542,6 +543,4 @@ } -/* psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s psastro %s ppstats %s ppImage %s streaks %s\n", */ -/* ver_pslib,ver_psmodules,ver_psphot,ver_psastro,ver_ppstats,ver_ppimage,ver_streaks); */ psString software_ver = NULL; if ((ver_pslib)&&(ver_psmodules)) { @@ -692,5 +691,6 @@ deteff_lq, deteff_uq, - quality + quality, + background_model ); @@ -817,5 +817,6 @@ pxAddLabelSearchArgs (config, where, "-data_group", "camRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - + PXOPT_COPY_S16(config->args, where, "-background_model", "camProcessedExp.background_model", "=="); + psString where2 = NULL; if (!pxspaceAddWhere(config, &where2, "rawExp")) { @@ -987,18 +988,32 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); - PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - - if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault, quality)) { + PXOPT_LOOKUP_S16(background_model, config->args, "-set_background_model", false, false); + if (background_model) { + // CZW Copied from warptool.c + psMetadata *where = psMetadataAlloc(); + psMetadata *values = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S16(config->args, values, "-set_background_model", "background_model", "=="); + long rows = psDBUpdateRows(config->dbh, "camProcessedExp", where, values); + psFree(values); + psFree(where); + if (!rows) { + psError(PS_ERR_UNKNOWN, true, "no rows changed"); + } + } + else { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + + if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault, quality)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); psFree (where); return false; - } - psFree (where); - + } + psFree (where); + } return true; } Index: /branches/eam_branches/ipp-20130207/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/camtoolConfig.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/camtoolConfig.c (revision 35236) @@ -204,5 +204,5 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_lq", 0, "define deteff_lq", NAN); psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-deteff_inst_uq", 0, "define deteff_uq", NAN); - + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-background_model", 0, "set background_model value", 0); // -processedexp @@ -222,4 +222,5 @@ psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); + psMetadataAddS16(processedexpArgs, PS_LIST_TAIL, "-background_model", 0, "search by background_model value", 0); // -revertprocessedexp @@ -246,4 +247,5 @@ psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", INT16_MAX); psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-set_background_model", 0, "set background_model value", 0); // -block Index: /branches/eam_branches/ipp-20130207/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/difftool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/difftool.c (revision 35236) @@ -1226,5 +1226,282 @@ } - +static bool definewarpstackMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "comment", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_label", "warpRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "warpRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">="); + PXOPT_COPY_STR(config->args, where, "-stack_label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-stack_data_group", "stackRun.data_group", "=="); + + PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); + PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(available, config->args, "-available", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + + // Get query file + psString query = pxDataGet("difftool_definewarpstack.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where,NULL); + + // Don't queue things that already have diffs. + if (! (reRun || newTemplates) ) { + psStringAppend(&whereClause, "\nAND diff_id IS NULL\n"); + } + // Append diff qualifiers, if we have them + if (label) { + psStringAppend(&whereClause, "\nAND ((diffRun.label = '%s') OR (diffRun.label IS NULL))",label); + } + if (data_group) { + psStringAppend(&whereClause, "\nAND ((diffRun.data_group = '%s') OR (diffRun.data_group IS NULL))",data_group); + } + if (reduction) { + psStringAppend(&whereClause, "\nAND ((diffRun.reduction = '%s') OR (diffRun.reduction IS NULL))",reduction); + } + + psString whereClauseString = psStringCopy(""); + psStringAppend(&whereClauseString, " \n AND %s ", whereClause); + // fprintf(stderr,query,whereClauseString); + + // This is just a simple query, so we don't need to do a transaction + if (!p_psDBRunQueryF(config->dbh, query, whereClauseString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereClause); + return(false); + } + psFree(query); + psFree(whereClause); + psFree(whereClauseString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + break; + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + break; + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + break; + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + psArray *list = psArrayAllocEmpty(16); // List of runs, to print + long numGood = 0; + psS64 last_exp_id = 0; + psS64 diff_id = 0; + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; + bool mdok; + + psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "exp_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (exp_id != last_exp_id) { + if (diff_id != 0) { // We've added a run already, and are now switching to a new one. + // Set state to new + if (!setdiffRunState(config, diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, diff_id); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + // Commit results + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + } + + // Begin transaction + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + // Add a new diffRun row + psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + diffRunRow *run = diffRunRowAlloc( + 0, // ID + "reg", // state + workdir, + label, + data_group ? data_group : label, + dist_group, + reduction, + NULL, // dvodb + registered, + tess_id, + bothways, // bothways (default is false) + true, // exposure + 0, // magicked + NULL, // software_version + 0, // mask stat npix + NAN, // static + NAN, // dynamic + NAN, // magic + NAN, // advisory + IPP_DIFF_MODE_WARP_STACK, + note + ); + if (!diffRunInsertObject(config->dbh, run)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(run); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + } + diff_id = psDBLastInsertID(config->dbh); + run->diff_id = diff_id; + + psArrayAdd(list, list->n, run); + numGood++; + + last_exp_id = exp_id; + } // End Adding diffRun + + if (exp_id == last_exp_id) { + psString skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "skycell_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "tess_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "warp_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psS64 stack_id = psMetadataLookupS64(&mdok, row, "stack_id"); + if (!mdok) { + psError(PXTOOLS_ERR_PROG, false, "stack_id not found"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // Add a new skyfile row + diffInputSkyfileRow *skyfile = diffInputSkyfileRowAlloc( + diff_id, // ID + skycell_id, + warp_id, // warp1_id + PS_MAX_S64, // stack1 -> NULL + PS_MAX_S64, // warp2_id -> NULL + stack_id, // stack2 + tess_id, + 0 // diff_skyfile_id + ); + // fprintf(stderr,"%"PRId64 " %"PRId64 " %"PRId64 " %s %s\n",diff_id,warp_id,stack_id,skycell_id, tess_id); + if (!diffInputSkyfileInsertObject(config->dbh, skyfile)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } // End Adding skyfile + } // End parsing result set + + // Finish the last run's update and close the connection + if (diff_id != 0) { + if (!setdiffRunState(config, diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, diff_id); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + // Commit results + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(list); + return false; + } + } + + if (numGood && !diffRunPrintObjects(stdout, list, !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print object"); + psFree(list); + return(false); + } + psFree(list); + // Free things + return true; +} + +#if (0) static bool definewarpstackMode(pxConfig *config) { @@ -1614,4 +1891,5 @@ return true; } +#endif static bool definewarpwarpMode(pxConfig *config) Index: /branches/eam_branches/ipp-20130207/ippTools/src/disttool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/disttool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/disttool.c (revision 35236) @@ -143,5 +143,5 @@ // optional PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_BOOL(magic, config->args, "-magic", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); @@ -175,5 +175,5 @@ return false; } - if (no_magic) { + if (!magic) { psError(PXTOOLS_ERR_SYS, true, "no_magic forbidden with alternate inputs"); return false; @@ -187,5 +187,5 @@ // an interest) // and exp_id to be supplied. We could add -exp_type and dateobs cuts to make it easier - if (!strcmp(stage, "raw") && no_magic && !(exp_id && dist_group && exp_type)) { + if (!strcmp(stage, "raw") && !magic && !(exp_id && dist_group && exp_type)) { psError(PXTOOLS_ERR_SYS, true, "exp_id, exp_type, and dist_group are required for raw stage if no_magic"); return false; @@ -204,5 +204,5 @@ runJoinStr = "camRun.exp_id"; } - if (!no_magic) { + if (magic) { query = pxDataGet("disttool_definebyquery_raw.sql"); } else { @@ -215,5 +215,5 @@ } - if (!no_magic) { + if (magic) { if (label) { psStringAppend(&query, " AND (magicDSRun.label = '%s')", label); @@ -289,5 +289,5 @@ } // fake stage doesn't require magic - no_magic = true; + magic = false; } else if (!strcmp(stage, "warp")) { magicRunType = "warpRun"; @@ -357,5 +357,5 @@ } // stack stage doesn't require magic - no_magic = true; + magic = false; } else if (!strcmp(stage, "sky")) { magicRunType = "staticskyRun"; @@ -378,5 +378,5 @@ } // (static)sky stage doesn't require magic - no_magic = true; + magic = false; } else if (!strcmp(stage, "SSdiff")) { magicRunType = "diffRun"; @@ -396,5 +396,5 @@ } - no_magic = true; + magic = false; } else { psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage); @@ -408,5 +408,5 @@ if (!strcmp(stage, "raw")) { - if (!no_magic) { + if (magic) { psStringAppend(&query, " AND (magicDSRun.re_place = %d)", !use_alternate); psStringAppend(&query, " AND (camRun.state = 'full')"); @@ -426,5 +426,5 @@ psString joinHook = NULL; - if (!no_magic) { + if (magic) { psStringAppend(&query, " AND (%s.magicked > 0)", magicRunType); } @@ -517,5 +517,5 @@ NULL, // outdir clean, - no_magic, + !magic, use_alternate, "new", Index: /branches/eam_branches/ipp-20130207/ippTools/src/disttoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/disttoolConfig.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/disttoolConfig.c (revision 35236) @@ -48,5 +48,6 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed (ignored)", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-magic", 0, "magic is needed", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); Index: /branches/eam_branches/ipp-20130207/ippTools/src/releasetool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/releasetool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/releasetool.c (revision 35236) @@ -41,6 +41,14 @@ static bool listreleaseMode(pxConfig *config); static bool definerelexpMode(pxConfig *config); +static bool tocalibexpMode(pxConfig *config); static bool updaterelexpMode(pxConfig *config); static bool listrelexpMode(pxConfig *config); +static bool definerelstackMode(pxConfig *config); +static bool updaterelstackMode(pxConfig *config); +static bool listrelstackMode(pxConfig *config); +static bool definerelgroupMode(pxConfig *config); +static bool pendingrelgroupMode(pxConfig *config); +static bool updaterelgroupMode(pxConfig *config); +static bool listrelgroupMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -66,4 +74,5 @@ MODECASE(RELEASETOOL_MODE_DEFINERELEASE, definereleaseMode); MODECASE(RELEASETOOL_MODE_UPDATERELEASE, updatereleaseMode); + MODECASE(RELEASETOOL_MODE_TOCALIBEXP, tocalibexpMode); MODECASE(RELEASETOOL_MODE_LISTRELEASE, listreleaseMode); @@ -71,4 +80,13 @@ MODECASE(RELEASETOOL_MODE_UPDATERELEXP, updaterelexpMode); MODECASE(RELEASETOOL_MODE_LISTRELEXP, listrelexpMode); + + MODECASE(RELEASETOOL_MODE_DEFINERELSTACK, definerelstackMode); + MODECASE(RELEASETOOL_MODE_UPDATERELSTACK, updaterelstackMode); + MODECASE(RELEASETOOL_MODE_LISTRELSTACK, listrelstackMode); + + MODECASE(RELEASETOOL_MODE_DEFINERELGROUP, definerelgroupMode); + MODECASE(RELEASETOOL_MODE_UPDATERELGROUP, updaterelgroupMode); + MODECASE(RELEASETOOL_MODE_PENDINGRELGROUP, pendingrelgroupMode); + MODECASE(RELEASETOOL_MODE_LISTRELGROUP, listrelgroupMode); default: psAbort("invalid option (this should not happen)"); @@ -163,5 +181,5 @@ PXOPT_LOOKUP_S32(surveyID, config->args, "-set_surveyID", true, false); - PXOPT_LOOKUP_STR(releaseName, config->args, "-set_releaseName", true, false); + PXOPT_LOOKUP_STR(release_name, config->args, "-set_release_name", true, false); PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state", true, false); PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false); @@ -171,5 +189,5 @@ 0, // rel_id (auto_increment) surveyID, - releaseName, + release_name, release_state, dataRelease, @@ -187,20 +205,39 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(releaseName, config->args, "-releaseName", false, false); + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); PXOPT_LOOKUP_S32(rel_id, config->args, "-rel_id", false, false); - if (!releaseName && !rel_id) { - psError(PXTOOLS_ERR_CONFIG, true, "either -releaseName or -rel_id is required\n"); - return false; - } - PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state", true, false); + if (!release_name && !rel_id) { + psError(PXTOOLS_ERR_CONFIG, true, "either -release_name or -rel_id is required\n"); + return false; + } + PXOPT_LOOKUP_STR(release_state, config->args, "-set_release_state", false, false); + PXOPT_LOOKUP_S32(priority, config->args, "-set_priority", false, false); + PXOPT_LOOKUP_S32(dataRelease, config->args, "-set_dataRelease", false, false); + if (!release_state && priority == 0 && dataRelease == -1) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -set_release_state -set_priority and -set_dataRelease is required\n"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S32(config->args, where, "-rel_id", "rel_id", "=="); - PXOPT_COPY_STR(config->args, where, "-releaseName", "releaseName", "=="); - - psString query = NULL; - psStringAppend(&query, "UPDATE ippRelease SET release_state = '%s'", release_state); + PXOPT_COPY_STR(config->args, where, "-release_name", "release_name", "=="); + + psString sep = ""; + psString comma = ", "; + psString query = psStringCopy("UPDATE ippRelease SET"); + if (release_state) { + psStringAppend(&query, "%s release_state = '%s'", sep, release_state); + sep = comma; + } + if (priority != 0) { + psStringAppend(&query, "%s priority = %d", sep, priority); + sep = comma; + } + if (dataRelease != -1) { + psStringAppend(&query, "%s dataRelease = %d", sep, dataRelease); + sep = comma; + } psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -232,5 +269,5 @@ PXOPT_COPY_STR(config->args, where, "-surveyName", "surveyName", "LIKE"); - PXOPT_COPY_STR(config->args, where, "-releaseName", "releaseName", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-release_name", "release_name", "LIKE"); PXOPT_COPY_STR(config->args, where, "-release_state", "release_state", "=="); PXOPT_COPY_S32(config->args, where, "-rel_id", "rel_id", "=="); @@ -313,13 +350,13 @@ // insure that at least one of these is supplied to select the release PXOPT_LOOKUP_S64(rel_id, config->args, "-rel_id", false, false); - PXOPT_LOOKUP_STR(releaseName, config->args, "-releaseName", false, false); - if (!rel_id && !releaseName) { - psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -releaseName is required"); - return false; - } - - // limit query to the target release. (Note: this will select nothing of releaseName and rel_id don't match) + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + if (!rel_id && !release_name) { + psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required"); + return false; + } + + // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match) // note: we add these after above to insure that search args are supplied - PXOPT_COPY_STR(config->args, where, "-releaseName", "ippRelease.releaseName", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "=="); PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "=="); @@ -327,5 +364,8 @@ PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); PXOPT_LOOKUP_U32(flags, config->args, "-set_flags", false, false); + PXOPT_LOOKUP_F32(user_zpt_obs, config->args, "-set_zpt_obs", false, false); + PXOPT_LOOKUP_F32(user_zpt_stdev, config->args, "-set_zpt_stdev", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -391,6 +431,17 @@ psS64 chip_id = psMetadataLookupS64(&status, md, "chip_id"); psS64 cam_id = psMetadataLookupS64(&status, md, "cam_id"); - psF32 zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs"); - psF32 zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev"); + + // if zpt_obs or zpt_stdev were not supplied use the results from the camRun + psF32 zpt_obs, zpt_stdev; + if (!isfinite(user_zpt_obs)) { + zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs"); + } else { + zpt_obs = user_zpt_obs; + } + if (!isfinite(user_zpt_stdev)) { + zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev"); + } else { + zpt_stdev = user_zpt_stdev; + } if (!relExpInsert(config->dbh, @@ -400,4 +451,5 @@ chip_id, cam_id, + 0, // group_id state, flags, @@ -405,4 +457,5 @@ zpt_stdev, path_base, + fault, now, // registered now // time_stamp @@ -430,9 +483,99 @@ static bool updaterelexpMode(pxConfig *config) { - psError(PS_ERR_UNKNOWN, true, "not yet implemented"); - return false; -} - -static bool listrelexpMode(pxConfig *config) + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + PXOPT_LOOKUP_S32(rel_id, config->args, "-rel_id", false, false); + PXOPT_LOOKUP_S32(relexp_id, config->args, "-relexp_id", false, false); + + if (!relexp_id && !release_name && !rel_id) { + psError(PXTOOLS_ERR_CONFIG, true, "at least one of -relexp_id, -release_name, and -rel_id is required\n"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_U32(flags, config->args, "-set_flags", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false); + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-set_zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_stdev,config->args, "-set_zpt_stdev", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + if (!state && !flags && !path_base && !isfinite(zpt_obs) && !isfinite(zpt_stdev) && !fault && !clearfault) { + psError(PXTOOLS_ERR_CONFIG, true, "must set at least one column\n"); + return false; + } + + if (fault && clearfault) { + psError(PXTOOLS_ERR_CONFIG, true, "cannot set and clear fault at same time\n"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); + PXOPT_COPY_S32(config->args, where, "-relexp_id", "relExp.relexp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + // make sure that we have enough parameters to identify the relExp to change + if (!psListLength(where->list)){ + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + PXOPT_COPY_S32(config->args, where, "-rel_id", "rel_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "release_name", "=="); + + psString sep = ""; + psString comma = ", "; + psString query = psStringCopy("UPDATE relExp JOIN ippRelease USING(rel_id) JOIN rawExp USING(exp_id) SET"); + if (state) { + psStringAppend(&query, "%s relExp.state = '%s'", sep, state); + sep = comma; + } + if (path_base) { + psStringAppend(&query, "%s relExp.path_base = '%s'", sep, path_base); + sep = comma; + } + if (flags) { + psStringAppend(&query, "%s relExp.flags = %d", sep, flags); + sep = comma; + } + if (isfinite(zpt_obs)) { + psStringAppend(&query, "%s relExp.zpt_obs = %f", sep, zpt_obs); + sep = comma; + } + if (isfinite(zpt_stdev)) { + psStringAppend(&query, "%s relExp.zpt_stdev = %f", sep, zpt_stdev); + sep = comma; + } + if (fault) { + psStringAppend(&query, "%s relExp.fault = %d", sep, fault); + sep = comma; + } + if (clearfault) { + psStringAppend(&query, "%s relExp.fault = 0", sep); + sep = comma; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("releasetool", PS_LOG_INFO, "Updated %" PRIu64 " ippReleases", affected); + + + psFree(query); + + return true; +} + +static bool tocalibexpMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); @@ -440,5 +583,8 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-releaseName", "ippRelease.releaseName", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-relexp_id", "relExp.relexp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); +#ifdef notdef + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); PXOPT_COPY_STR(config->args, where, "-state", "relExp.state", "=="); @@ -458,4 +604,104 @@ PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); + + PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); +#endif + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("releasetool_tocalibexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString where2 = NULL; +#ifdef notdef + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); + return false; + } +#endif + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); +#ifdef notdef + } else if (where2) { + psStringAppend(&query, "\nAND "); +#endif + } else { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + if (where2) { + psStringAppend(&query, "\n%s", where2); + psFree(where2); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool listrelexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-relexp_id", "relExp.relexp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); + pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "relExp.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-fwhm_min", "camProcessedExp.fwhm_major", ">="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "camProcessedExp.fwhm_major", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "relExp.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); @@ -533,3 +779,721 @@ return true; } - +static bool definerelstackMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + bool includeSkycal = false; + + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "=="); + if (psListLength(where->list)) { + includeSkycal = true; + } + + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + + // Insure that at least one of the skycalRun selectors is supplied + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "at least one of -label, -data_group, or -skycal_id is required"); + return false; + } + + // PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); + // PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + + // insure that at least one of these is supplied to select the release + PXOPT_LOOKUP_S64(rel_id, config->args, "-rel_id", false, false); + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + if (!rel_id && !release_name) { + psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required"); + return false; + } + + // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match) + // note: we add these after above to insure that search args are supplied + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "=="); + + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + PXOPT_LOOKUP_STR(stack_type, config->args,"-set_stack_type", true, false); + PXOPT_LOOKUP_U32(flags, config->args, "-set_flags", false, false); + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-set_zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-set_zpt_stdev", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-set_path_base", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the parameters of all the exposures that we want to add to the release + psString query; + + if (includeSkycal) { + query = pxDataGet("releasetool_definerelstack_with_skycal.sql"); + } else { + query = pxDataGet("releasetool_definerelstack.sql"); + } + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "relstack", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + // loop over our list of exposures + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status = true; + psS32 rel_id = psMetadataLookupS32(&status, md, "rel_id"); + psS64 stack_id = psMetadataLookupS64(&status, md, "stack_id"); + psS64 skycal_id = psMetadataLookupS64(&status, md, "skycal_id"); + psString skycell_id = psMetadataLookupStr(&status, md, "skycell_id"); + psString tess_id = psMetadataLookupStr(&status, md, "tess_id"); + psString filter = psMetadataLookupStr(&status, md, "filter"); + + // use skycal zero point measurement if not supplied with arguments + psF32 sc_zpt_obs = psMetadataLookupF32(&status, md, "zpt_obs"); + psF32 sc_zpt_stdev = psMetadataLookupF32(&status, md, "zpt_stdev"); + psF32 stack_mjd_obs = psMetadataLookupF32(&status, md, "mjd_obs"); + + psU32 mjd_obs = 0; + if (!strcmp(stack_type, "nightly")) { + // only for nightly stacks do we set mjd_obs to a non zero value + // We use integer day because this is part of the key that is used + // to make sure entries are unique + // KEY (rel_id, tess_id, skycell_id, filter, mjd_obs), + // 1 entry per release, skycell, filter and mjd_obs. + // mjd_obs is zero for deep and reference stacks + mjd_obs = (psU32) stack_mjd_obs; + } + + if (!relStackInsert(config->dbh, + 0, // relstack_id (auto increment) + rel_id, + stack_id, + skycal_id, + skycell_id, + tess_id, + filter, + state, + flags, + stack_type, + isfinite(zpt_obs) ? zpt_obs : sc_zpt_obs, + isfinite(zpt_stdev) ? zpt_stdev : sc_zpt_stdev, + mjd_obs, + path_base, + fault, + now, // registered + now // time_stamp + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to insert relStack for stack_id %" PRId64, stack_id); + psFree(output); + psFree(now); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updaterelstackMode(pxConfig *config) +{ + psError(PS_ERR_UNKNOWN, true, "not yet implemented"); + return false; +} + +static bool listrelstackMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); + pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "relExp.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-fwhm_min", "skycalResult.fwhm_major", ">="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalresult.fwhm_major", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_ame", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "relExp.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + + PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("releasetool_listrelexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString where2 = NULL; + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, "\nWHERE "); + } else { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + if (where2) { + psStringAppend(&query, "\n%s", where2); + psFree(where2); + } + + if (priority_order) { + psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority"); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool definerelgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(group_type, config->args, "-set_group_type", true, false); + PXOPT_LOOKUP_STR(group_name, config->args, "-set_group_name", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(exp_list_path, config->args, "-set_exp_list_path", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + psString query = NULL; + + psMetadata *where = psMetadataAlloc(); + + // select sql and paramters based on group type + bool type_lap = false; + if (!strcmp(group_type, "lap")) { + type_lap = true; + if (group_name) { + psError(PXTOOLS_ERR_CONFIG, true, "group_name is not allowed with group_type lap"); + psFree(where); + return false; + } + PXOPT_COPY_S64(config->args, where, "-select_seq_id", "lapRun.seq_id", "=="); + PXOPT_COPY_S64(config->args, where, "-select_lap_id", "lapRun.lap_id", "=="); + query = pxDataGet("releasetool_definerelgroup_select_lap.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + } else { + if (!strcmp(group_type, "date")) { + // group_type ok + } else if (!strcmp(group_type, "data_group")) { + // group_type ok + } else { + psError(PXTOOLS_ERR_CONFIG, true, "%s is not valid group_type", group_type); + return false; + } + if (!group_name) { + psError(PXTOOLS_ERR_CONFIG, true, "group_name is required allowed with group_type %s", group_type); + psFree(where); + return false; + } + pxAddLabelSearchArgs (config, where, "-select_data_group", "camRun.data_group", "=="); + query = pxDataGet("releasetool_definerelgroup_select_data_group.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + } + + // Insure that at least one of the exposure selectors is supplied + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search arguments are required"); + return false; + } + + // insure that at least one of these is supplied to select the release + PXOPT_LOOKUP_S64(rel_id, config->args, "-rel_id", false, false); + PXOPT_LOOKUP_STR(release_name, config->args, "-release_name", false, false); + if (!rel_id && !release_name) { + psError(PXTOOLS_ERR_CONFIG, true, "either -rel_id or -release_name is required"); + return false; + } + + // limit query to the target release. (Note: this will select nothing if release_name and rel_id don't match) + // note: we add these after above to insure that search args are supplied + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "ippRelease.rel_id", "=="); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + + psFree(where); + + if (!type_lap) { + psStringAppend(&query, "\nHAVING COUNT(exp_id) > 0"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "relGroupToQueue", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + psTime *now = psTimeGetNow(PS_TIME_TAI); + // loop over our list of exposures + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status = true; + psS32 rel_id = psMetadataLookupS32(&status, md, "rel_id"); + psS64 lap_id = 0; + if (type_lap) { + lap_id = psMetadataLookupS64(&status, md, "lap_id"); + } + + if (!relGroupInsert(config->dbh, + 0, // group_id (auto increment) + rel_id, + group_type, + lap_id, + group_name, + "reg", // state + label, + exp_list_path, + fault, + now // registered + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to trying to insert relGroup"); + psFree(output); + psFree(now); + return false; + } + // Here's our new group_id + psS32 group_id = psDBLastInsertID(config->dbh); + + psString query; + if (type_lap) { + query = pxDataGet("releasetool_definerelgroup_select_exp_lap.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + psStringAppend(&query, " AND relExp.rel_id = %d AND lap_id = %" PRId64, rel_id, lap_id); + } else { + query = pxDataGet("releasetool_definerelgroup_select_exp_data_group.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + psStringAppend(&query, " AND relExp.rel_id = %d", rel_id); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-select_data_group", "camRun.data_group", "=="); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(output); + psFree(now); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *exposures = p_psDBFetchResult(config->dbh); + if (!exposures) { + psFree(now); + psFree(output); + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(exposures)) { + psFree(now); + psFree(output); + psFree(exposures); + psError(PS_ERR_UNKNOWN, false, "no exposures found error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + // assign the relExps to this new group + char *updateQuery = "UPDATE relExp SET group_id = %d WHERE relexp_id = %" PRId64; + for (int j = 0; j < psArrayLength(exposures); j++) { + psMetadata *md = exposures->data[j]; + + bool status = true; + psS32 relexp_id = psMetadataLookupS32(&status, md, "relexp_id"); + if (!p_psDBRunQueryF(config->dbh, updateQuery, group_id, relexp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(now); + psFree(output); + psFree(exposures); + + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(exposures); + + // set state of relGroup to 'new' + if (!p_psDBRunQueryF(config->dbh, "UPDATE relGroup set state = 'new' WHERE group_id = %d", group_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(now); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(output); + psFree(now); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updaterelgroupMode(pxConfig *config) +{ + psError(PS_ERR_UNKNOWN, true, "not yet implemented"); + return false; +} + +static bool pendingrelgroupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + pxAddLabelSearchArgs (config, where, "-label", "relGroup.label", "=="); + PXOPT_COPY_S32(config->args, where, "-group_id", "relGroup.group_id", "=="); + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("releasetool_pendingrelgroup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "pending_relGroup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool listrelgroupMode(pxConfig *config) +{ + return false; +#ifdef notdef + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-release_name", "ippRelease.release_name", "LIKE"); + pxAddLabelSearchArgs(config, where, "-release_state","ippRelease.state", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "relExp.state", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin","rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_F32(config->args, where, "-fwhm_min", "skycalResult.fwhm_major", ">="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalresult.fwhm_major", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_ame", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "relExp.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "relExp.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-chip_data_group", "chipRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + + PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); + PXOPT_COPY_S32(config->args, where, "-rel_id", "relExp.rel_id", "=="); + + PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("releasetool_listrelexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString where2 = NULL; + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxspaceAddWhere failed"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + } else if (where2) { + psStringAppend(&query, "\nWHERE "); + } else { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required\n"); + psFree(where); + return false; + } + + if (where2) { + psStringAppend(&query, "\n%s", where2); + psFree(where2); + } + + if (priority_order) { + psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority"); + } + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psArrayLength(output)) { + psTrace("releasetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "relExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +#endif //notdef +} + Index: /branches/eam_branches/ipp-20130207/ippTools/src/releasetool.h =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/releasetool.h (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/releasetool.h (revision 35236) @@ -32,5 +32,14 @@ RELEASETOOL_MODE_DEFINERELEXP, RELEASETOOL_MODE_UPDATERELEXP, + RELEASETOOL_MODE_TOCALIBEXP, RELEASETOOL_MODE_LISTRELEXP, + RELEASETOOL_MODE_DEFINERELSTACK, + RELEASETOOL_MODE_UPDATERELSTACK, + RELEASETOOL_MODE_TOCALIBSTACK, + RELEASETOOL_MODE_LISTRELSTACK, + RELEASETOOL_MODE_DEFINERELGROUP, + RELEASETOOL_MODE_PENDINGRELGROUP, + RELEASETOOL_MODE_UPDATERELGROUP, + RELEASETOOL_MODE_LISTRELGROUP, } releasetoolMode; Index: /branches/eam_branches/ipp-20130207/ippTools/src/releasetoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/releasetoolConfig.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/releasetoolConfig.c (revision 35236) @@ -60,23 +60,19 @@ psMetadata *definereleaseArgs = psMetadataAlloc(); psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_surveyID", 0, "define survey ID (required)", 0); - psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_releaseName", 0, "define release name (required)", NULL); + psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_release_name", 0, "define release name (required)", NULL); psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0, "define dataRelease", -1); psMetadataAddStr(definereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define release state", NULL); - psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define release priority", -1); + psMetadataAddS32(definereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define release priority", 0); // -uptdaterelease psMetadata *updatereleaseArgs = psMetadataAlloc(); - // releaseName and/or rel_id is required. This is handled in updatereleaseMode - psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name", NULL); + // release_name and/or rel_id is required. This is handled in updatereleaseMode + psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name", NULL); psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); - // Only support changing state and priority for now - psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define new release state(required)", NULL); + psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_release_state", 0, "define new release state", NULL); psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_priority", 0, "define new release priority", 0); - -// psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_surveyID", 0, "define surveyID (required)", 0); -// psMetadataAddStr(updatereleaseArgs, PS_LIST_TAIL, "-set_releaseName", 0, "define release name (required)", NULL); -// psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0, "define dataRelease", -1); + psMetadataAddS32(updatereleaseArgs, PS_LIST_TAIL, "-set_dataRelease", 0, "define data release", -1); @@ -84,5 +80,5 @@ psMetadata *listreleaseArgs = psMetadataAlloc(); psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); - psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); psMetadataAddStr(listreleaseArgs, PS_LIST_TAIL, "-release_state", 0, "select by release state", NULL); psMetadataAddS32(listreleaseArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); @@ -95,5 +91,5 @@ // set the target release - psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-releaseName", 0, "define release name", NULL); + psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-release_name", 0, "define release name", NULL); psMetadataAddS32(definerelexpArgs, PS_LIST_TAIL, "-rel_id", 0, "define release ID", 0); @@ -109,19 +105,41 @@ // parameters of the relExp psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + psMetadataAddU32(definerelexpArgs, PS_LIST_TAIL, "-set_flags", 0, "define flags", 0); + psMetadataAddF32(definerelexpArgs, PS_LIST_TAIL, "-set_zpt_obs", 0, "define zero point", NAN); + psMetadataAddF32(definerelexpArgs, PS_LIST_TAIL, "-set_zpt_stdev", 0, "define zero point stdev", NAN); psMetadataAddStr(definerelexpArgs, PS_LIST_TAIL, "-set_path_base", 0, "define state", NULL); - psMetadataAddU32(definerelexpArgs, PS_LIST_TAIL, "-set_flags", 0, "define flags", 0); - - psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-pretend", 0, "do not actual modify the database", false); - psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simpel output format", false); - - + psMetadataAddS16(definerelexpArgs, PS_LIST_TAIL, "-set_fault", 0, "define fault", 0); + + psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definerelexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterelexp psMetadata *updaterelexpArgs = psMetadataAlloc(); - psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddS64(updaterelexpArgs, PS_LIST_TAIL, "-relexp_id", 0, "select by released exposure ID", 0); + psMetadataAddS64(updaterelexpArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID", 0); + psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-exp_name", 0, "select by exposure name", NULL); + psMetadataAddS32(updaterelexpArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); + + psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state)", NULL); + psMetadataAddU32(updaterelexpArgs, PS_LIST_TAIL, "-set_flags", 0, "define new flags", 0); + psMetadataAddF32(updaterelexpArgs, PS_LIST_TAIL, "-set_zpt_obs", 0, "define new zero point", NAN); + psMetadataAddF32(updaterelexpArgs, PS_LIST_TAIL, "-set_zpt_stdev", 0, "define new zero point stdev", NAN); + psMetadataAddStr(updaterelexpArgs, PS_LIST_TAIL, "-set_path_base", 0, "define path_base", NULL); + psMetadataAddS16(updaterelexpArgs, PS_LIST_TAIL, "-set_fault", 0, "define new fault", 0); + psMetadataAddBool(updaterelexpArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); + + // -tocalibexp + psMetadata *tocalibexpArgs = psMetadataAlloc(); + psMetadataAddS64(tocalibexpArgs, PS_LIST_TAIL, "-relexp_id", 0, "select by released exposure ID", 0); + psMetadataAddStr(tocalibexpArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); + + psMetadataAddU64(tocalibexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocalibexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -listrelexp psMetadata *listrelexpArgs = psMetadataAlloc(); - psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-releaseName", 0, "select by release name (LIKE comparision)", NULL); + psMetadataAddS64(listrelexpArgs, PS_LIST_TAIL, "-relexp_id", 0, "select by released exposure ID", 0); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name (LIKE comparision)", NULL); psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-release_state", PS_META_DUPLICATE_OK, "select by release state", NULL); psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-state", 0, "select by released exposure state", NULL); @@ -155,4 +173,88 @@ psMetadataAddBool(listrelexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -definerelstack + psMetadata *definerelstackArgs = psMetadataAlloc(); + + // set the target release + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-release_name", 0, "define release name", NULL); + psMetadataAddS32(definerelstackArgs, PS_LIST_TAIL, "-rel_id", 0, "define release ID", 0); + + // select the processing + psMetadataAddS64(definerelstackArgs, PS_LIST_TAIL, "-stack_id", 0, "select by stack ID", 0); + psMetadataAddS64(definerelstackArgs, PS_LIST_TAIL, "-skycal_id", 0, "select by skycal ID", 0); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-label", 0, "select by stackRun.label", NULL); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-data_group", 0, "select by stackRun.data_group", NULL); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-skycal_label", 0, "select by skycalRun.label", NULL); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-skycal_data_group", 0, "select by skycalRun.data_group", NULL); + + // parameters of the relStack + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + psMetadataAddU32(definerelstackArgs, PS_LIST_TAIL, "-set_flags", 0, "define flags", 0); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-set_stack_type", 0, "define stack type (required)", NULL); + psMetadataAddF32(definerelstackArgs, PS_LIST_TAIL, "-set_zpt_obs", 0, "define zero point", NAN); + psMetadataAddF32(definerelstackArgs, PS_LIST_TAIL, "-set_zpt_stdev", 0, "define zero point stdev", NAN); + psMetadataAddF32(definerelstackArgs, PS_LIST_TAIL, "-set_fwhm_major", 0, "define fwhm_major", NAN); + psMetadataAddStr(definerelstackArgs, PS_LIST_TAIL, "-set_path_base", 0, "define state (required)", NULL); + psMetadataAddS16(definerelstackArgs, PS_LIST_TAIL, "-set_fault", 0, "define fault", 0); + + psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definerelstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -definerelgroup + psMetadata *definerelgroupArgs = psMetadataAlloc(); + + // set the target release + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-release_name", 0, "define release name", NULL); + psMetadataAddS32(definerelgroupArgs, PS_LIST_TAIL, "-rel_id", 0, "define release ID", 0); + + // select the members of the group + psMetadataAddS64(definerelgroupArgs, PS_LIST_TAIL, "-select_seq_id", 0, "select by LAP sequence ID", 0); + psMetadataAddS64(definerelgroupArgs, PS_LIST_TAIL, "-select_lap_id", 0, "select by LAP ID", 0); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "select by camRun.data_group", NULL); + // do we want this? + // psMetadataAddS64(definerelgroupArgs, PS_LIST_TAIL, "-lap_id", 0, "select by LAP ID", NULL); + + // parameters of the relGroup + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_type", 0, "define group_type (required)", NULL); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_group_name", 0, "define group_name (required)", NULL); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label (required)", NULL); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definerelgroupArgs, PS_LIST_TAIL, "-set_exp_list_path", 0, "define path in exposure list", NULL); + psMetadataAddS16(definerelgroupArgs, PS_LIST_TAIL, "-set_fault", 0, "define fault", 0); + + // options + psMetadataAddBool(definerelgroupArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definerelgroupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -pendingrelgroup + psMetadata *pendingrelgroupArgs = psMetadataAlloc(); + + psMetadataAddStr(pendingrelgroupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by relGroup.label", NULL); + psMetadataAddS32(pendingrelgroupArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + psMetadataAddS32(pendingrelgroupArgs, PS_LIST_TAIL, "-group_id", 0, "select by relGroup ID", 0); + psMetadataAddStr(pendingrelgroupArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name", 0); + psMetadataAddBool(pendingrelgroupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingrelgroupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -updaterelgroup + psMetadata *updaterelgroupArgs = psMetadataAlloc(); + + // set the target release + psMetadataAddS32(updaterelgroupArgs, PS_LIST_TAIL, "-group_id", 0, "select by group ID (required)", 0); + + // parameters of the relGroup + psMetadataAddStr(updaterelgroupArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(updaterelgroupArgs, PS_LIST_TAIL, "-set_exp_list_path", 0, "define path in exposure list", NULL); + psMetadataAddStr(updaterelgroupArgs, PS_LIST_TAIL, "-set_label", 0, "define relgroup label", NULL); + psMetadataAddS16(updaterelgroupArgs, PS_LIST_TAIL, "-set_fault", 0, "define fault", 0); + psMetadataAddBool(updaterelgroupArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); + + psMetadata *listrelgroupArgs = psMetadataAlloc(); + psMetadataAddS32(listrelgroupArgs, PS_LIST_TAIL, "-group_id", 0, "select by group ID", 0); + psMetadataAddS32(listrelgroupArgs, PS_LIST_TAIL, "-rel_id", 0, "select by release ID", 0); + psMetadataAddStr(listrelgroupArgs, PS_LIST_TAIL, "-release_name", 0, "select by release name", 0); + + // ****************************************************************** + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -160,10 +262,22 @@ PXOPT_ADD_MODE("-definesurvey", "define a survey", RELEASETOOL_MODE_DEFINESURVEY, definesurveyArgs); PXOPT_ADD_MODE("-listsurvey", "list skycell parameters", RELEASETOOL_MODE_LISTSURVEY, listsurveyArgs); + PXOPT_ADD_MODE("-definerelease", "define a release", RELEASETOOL_MODE_DEFINERELEASE, definereleaseArgs); PXOPT_ADD_MODE("-updaterelease", "update a release", RELEASETOOL_MODE_UPDATERELEASE, updatereleaseArgs); PXOPT_ADD_MODE("-listrelease", "list releases", RELEASETOOL_MODE_LISTRELEASE, listreleaseArgs); + PXOPT_ADD_MODE("-definerelexp", "define a released exposure", RELEASETOOL_MODE_DEFINERELEXP, definerelexpArgs); PXOPT_ADD_MODE("-updaterelexp", "update a released exposure", RELEASETOOL_MODE_UPDATERELEXP, updaterelexpArgs); + PXOPT_ADD_MODE("-tocalibexp", "list relExps pending calibration", RELEASETOOL_MODE_TOCALIBEXP, tocalibexpArgs); PXOPT_ADD_MODE("-listrelexp", "list released exposures", RELEASETOOL_MODE_LISTRELEXP, listrelexpArgs); + + PXOPT_ADD_MODE("-definerelstack", "define a released stack", RELEASETOOL_MODE_DEFINERELSTACK, definerelstackArgs); + + PXOPT_ADD_MODE("-definerelgroup", "define a group of exposures", RELEASETOOL_MODE_DEFINERELGROUP, definerelgroupArgs); + PXOPT_ADD_MODE("-pendingrelgroup", "list relGroups pending processing", RELEASETOOL_MODE_PENDINGRELGROUP, pendingrelgroupArgs); + PXOPT_ADD_MODE("-updaterelgroup", "update a relGroup", RELEASETOOL_MODE_UPDATERELGROUP, updaterelgroupArgs); + PXOPT_ADD_MODE("-listrelgroup", "define a group of exposures", RELEASETOOL_MODE_LISTRELGROUP, listrelgroupArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/ipp-20130207/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/staticskytool.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/staticskytool.c (revision 35236) @@ -1281,4 +1281,6 @@ PXOPT_LOOKUP_F32(zpt_obs, config->args, "-zpt_obs", false, false); PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false); + PXOPT_LOOKUP_F32(fwhm_major, config->args, "-fwhm_major", false, false); + PXOPT_LOOKUP_F32(fwhm_minor, config->args, "-fwhm_minor", false, false); @@ -1334,4 +1336,6 @@ zpt_obs, zpt_stdev, + fwhm_major, + fwhm_minor, quality, software_ver, Index: /branches/eam_branches/ipp-20130207/ippTools/src/staticskytoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20130207/ippTools/src/staticskytoolConfig.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippTools/src/staticskytoolConfig.c (revision 35236) @@ -211,6 +211,8 @@ psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); - psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", 0); - psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0, "define observed zero point stdandard deviation", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0, "define observed zero point stdandard deviation", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-fwhm_major", 0, "define fwhm (major axis; pixels)", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-fwhm_minor", 0, "define fwhm (minor axis; pixels)", NAN); psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); Index: /branches/eam_branches/ipp-20130207/ippconfig/dvo.photcodes =================================================================== --- /branches/eam_branches/ipp-20130207/ippconfig/dvo.photcodes (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippconfig/dvo.photcodes (revision 35236) @@ -20,5 +20,5 @@ # PS1 ISP: g: 3200, r: 3300, i: 3400, z: 3500, y: 3600 # PS1 TC3: g: 3280-3296, r: 3380-3396, i: 3480-3496, z: 3580-3596, y: 3680-3696, -# PS1 GPC1: g: 3201-3264, r: 3301-3364, i: 3401-3464, z: 3501-3564, y: 3601-3664 +# PS1 GPC1: g: 3201-3264, r: 3301-3364, i: 3401-3464, z: 3501-3564, y: 3601-3664, w: 3201-3464 # NOTE 2012.03.30 EAM : I have added 'poor' and 'bad' bitsets for GPC1 data using psphot @@ -37,4 +37,5 @@ 7 H sec 0.000 0.000 0.000 - - 0.0000 0 2012 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 8 K sec 0.000 0.000 0.000 - - 0.0000 0 2013 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 9 w sec 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 1010 U_L92 ref 0.000 0.000 0.000 - - 0.0000 0 - 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 @@ -835,4 +836,64 @@ 10475 GPC1.y.XY75 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 10476 GPC1.y.XY76 dep 23.320 -0.073 0.000 - - 0.0000 0 5 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10501 GPC1.w.XY01 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10502 GPC1.w.XY02 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10503 GPC1.w.XY03 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10504 GPC1.w.XY04 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10505 GPC1.w.XY05 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10506 GPC1.w.XY06 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10510 GPC1.w.XY10 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10511 GPC1.w.XY11 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10512 GPC1.w.XY12 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10513 GPC1.w.XY13 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10514 GPC1.w.XY14 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10515 GPC1.w.XY15 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10516 GPC1.w.XY16 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10517 GPC1.w.XY17 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10520 GPC1.w.XY20 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10521 GPC1.w.XY21 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10522 GPC1.w.XY22 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10523 GPC1.w.XY23 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10524 GPC1.w.XY24 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10525 GPC1.w.XY25 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10526 GPC1.w.XY26 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10527 GPC1.w.XY27 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10530 GPC1.w.XY30 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10531 GPC1.w.XY31 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10532 GPC1.w.XY32 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10533 GPC1.w.XY33 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10534 GPC1.w.XY34 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10535 GPC1.w.XY35 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10536 GPC1.w.XY36 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10537 GPC1.w.XY37 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10540 GPC1.w.XY40 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10541 GPC1.w.XY41 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10542 GPC1.w.XY42 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10543 GPC1.w.XY43 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10544 GPC1.w.XY44 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10545 GPC1.w.XY45 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10546 GPC1.w.XY46 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10547 GPC1.w.XY47 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10550 GPC1.w.XY50 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10551 GPC1.w.XY51 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10552 GPC1.w.XY52 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10553 GPC1.w.XY53 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10554 GPC1.w.XY54 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10555 GPC1.w.XY55 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10556 GPC1.w.XY56 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10557 GPC1.w.XY57 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10560 GPC1.w.XY60 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10561 GPC1.w.XY61 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10562 GPC1.w.XY62 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10563 GPC1.w.XY63 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10564 GPC1.w.XY64 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10565 GPC1.w.XY65 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10566 GPC1.w.XY66 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10567 GPC1.w.XY67 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10571 GPC1.w.XY71 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10572 GPC1.w.XY72 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10573 GPC1.w.XY73 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10574 GPC1.w.XY74 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10575 GPC1.w.XY75 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 + 10576 GPC1.w.XY76 dep 25.000 0.000 0.000 - - 0.0000 0 9 0.015 0.000 1.000 0.000 0x0000 0x0000 0xe0440130 0x1003bc88 11000 GPC1.g.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 1 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 11100 GPC1.r.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 2 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 @@ -840,4 +901,5 @@ 11300 GPC1.z.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 4 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 11400 GPC1.y.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 5 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 + 11500 GPC1.w.SkyChip dep 25.000 0.000 0.000 - - 0.0000 0 9 0.000 0.000 0.000 0.000 0x0000 0x0000 0x0000 0x0000 160 MOSAIC2.B.00 dep 25.107 0.000 0.000 - - 0.0000 0 1 0.000 0.000 1.000 0.000 0x0000 0x3888 0x0000 0x0000 161 MOSAIC2.B.01 dep 25.120 0.000 0.000 - - 0.0000 0 1 0.000 0.000 1.000 0.000 0x0000 0x3888 0x0000 0x0000 Index: /branches/eam_branches/ipp-20130207/ippconfig/gpc1/psphot.config =================================================================== --- /branches/eam_branches/ipp-20130207/ippconfig/gpc1/psphot.config (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippconfig/gpc1/psphot.config (revision 35236) @@ -48,5 +48,5 @@ BACKGROUND.YBIN S32 32 # size of background superpixels DIFF_STATS BOOL TRUE # Calculate diff-specific stats - OUTPUT.FORMAT STR PS1_DV2 # Format of output file + OUTPUT.FORMAT STR PS1_DV3 # Format of output file PSPHOT.EXT.NSIGMA.LIMIT F32 0.0 @@ -56,4 +56,9 @@ PSPHOT.EXT.FIT.ALL.SOURCES BOOL TRUE PSPHOT.EXT.FIT.ALL.THRESH F32 1000 +# Lower threshold as requested by MOPS + EXTENDED_SOURCE_MODELS METADATA + TYPE EXTENDED_SOURCE_MODEL MODEL SNLIM PSF_CONVOLVED + TRAIL_RAW EXTENDED_SOURCE_MODEL PS_MODEL_TRAIL 4.75 FALSE + END END @@ -65,4 +70,9 @@ PSPHOT.EXT.FIT.ALL.SOURCES BOOL TRUE PSPHOT.EXT.FIT.ALL.THRESH F32 1000 +# Lower threshold as requested by MOPS + EXTENDED_SOURCE_MODELS METADATA + TYPE EXTENDED_SOURCE_MODEL MODEL SNLIM PSF_CONVOLVED + TRAIL_RAW EXTENDED_SOURCE_MODEL PS_MODEL_TRAIL 4.75 FALSE + END END Index: /branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStats.config =================================================================== --- /branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStats.config (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStats.config (revision 35236) @@ -358,2 +358,43 @@ ANALYSIS STR MASKFRAC_NPIX END + +# stats for sky calibration (astrometry, summary stats) +SKYCALSTATS METADATA + HEADER MULTI + HEADER STR CERROR + HEADER STR DT_ASTR # elapsed time in astrometry processing + HEADER STR NASTRO + + HEADER STR ZPT_OBS + HEADER STR ZPT_ERR + HEADER STR FWHM_MAJ + HEADER STR FWHM_MIN + + HEADER STR PSLIB_V + HEADER STR MODULE_V + HEADER STR PHOT_V + HEADER STR ASTRO_V + HEADER STR STATS_V + HEADER STR IMAGE_V + HEADER STR STREAK_V + + HEADER STR NDET # sources->n + HEADER STR NDET_EXT + HEADER STR NDET_FRC + + ANALYSIS MULTI # metadata blocks to search in chip/cell/readout->analysis + ANALYSIS STR PSPHOT.HEADER + ANALYSIS STR PSASTRO.HEADER + + ANALYSIS STR MASKFRAC_REF_STATIC + ANALYSIS STR MASKFRAC_REF_MAGIC + ANALYSIS STR MASKFRAC_REF_ADVISORY + ANALYSIS STR MASKFRAC_REF_DYNAMIC + ANALYSIS STR MASKFRAC_REF_NPIX + + ANALYSIS STR MASKFRAC_MAX_STATIC + ANALYSIS STR MASKFRAC_MAX_MAGIC + ANALYSIS STR MASKFRAC_MAX_ADVISORY + ANALYSIS STR MASKFRAC_MAX_DYNAMIC + ANALYSIS STR MASKFRAC_MAX_NPIX +END Index: /branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- /branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStatsFromMetadata.config (revision 35235) +++ /branches/eam_branches/ipp-20130207/ippconfig/recipes/ppStatsFromMetadata.config (revision 35236) @@ -337,4 +337,16 @@ +BACKGROUND_CHIP METADATA + ENTRY MULTI + TYPE VAL KEYWORD TYPE STATISTIC FLAG + ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats + +END + BACKGROUND_WARP METADATA ENTRY MULTI @@ -550,4 +562,7 @@ ENTRY VAL ZPT_OBS F32 CONSTANT -zpt_obs ENTRY VAL ZPT_ERR F32 CONSTANT -zpt_stdev + # don't know why these come out as F64 instead of F32 + ENTRY VAL FWHM_MAJ F64 CONSTANT -fhwm_major + ENTRY VAL FWHM_MIN F64 CONSTANT -fhwm_minor ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag Index: /branches/eam_branches/ipp-20130207/ppStack/src/ppStackCamera.c =================================================================== --- /branches/eam_branches/ipp-20130207/ppStack/src/ppStackCamera.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ppStack/src/ppStackCamera.c (revision 35236) @@ -48,5 +48,5 @@ bool convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images before stack? bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); - + // fprintf(stderr,"ppSC: %d\n",doBackground); psArray *runImages = pmFPAfileDefineMultipleFromRun(&status, NULL, config, "PPSTACK.INPUT"); // Input images from previous run @@ -218,7 +218,13 @@ // Grab bkgmodel information here - if (doBackground) { - if (!bkgmodel || strlen(bkgmodel) == 0) { + + // if (doBackground) { + if ((!bkgmodel) || (strlen(bkgmodel) == 0)) { // We have no background models. + // fprintf(stderr,"No backgrounds specified, resettting recipe value.\n"); + doBackground = false; + psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE, + "Do photometry on stacked image?", false); + } else { @@ -227,4 +233,5 @@ if (!inputBKG) { // We failed to generate an pmFPAfile, so disable the background construction and continue. + // fprintf(stderr,"No backgrounds specified, resettting recipe value.\n"); doBackground = false; psMetadataAddBool(recipe, PS_LIST_TAIL, "BACKGROUND.MODEL", PS_META_REPLACE, @@ -238,5 +245,5 @@ } }// End bkgmodel - } + // } Index: /branches/eam_branches/ipp-20130207/ppStack/src/ppStackCombineAlternate.c =================================================================== --- /branches/eam_branches/ipp-20130207/ppStack/src/ppStackCombineAlternate.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ppStack/src/ppStackCombineAlternate.c (revision 35236) @@ -88,4 +88,8 @@ } + if (!ppStackLinearScale(inputs, config)) { + psFree(inputs); + return(false); + } // Do combination Index: /branches/eam_branches/ipp-20130207/ppStack/src/ppStackLoop.c =================================================================== --- /branches/eam_branches/ipp-20130207/ppStack/src/ppStackLoop.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ppStack/src/ppStackLoop.c (revision 35236) @@ -63,8 +63,8 @@ } - bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); + // bool doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); // Prepare for combination - if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", doBackground ? "PPSTACK.OUTPUT.BKGMODEL" : NULL, PPSTACK_FILES_STACK, stack, options, config)) { + if (!ppStackCombinePrepare("PPSTACK.OUTPUT", "PPSTACK.OUTPUT.EXP", options->doBackground ? "PPSTACK.OUTPUT.BKGMODEL" : NULL, PPSTACK_FILES_STACK, stack, options, config)) { psError(psErrorCodeLast(), false, "Unable to prepare for combination."); psFree(stack); @@ -146,5 +146,5 @@ // Generate median background stack here. - if (doBackground && !ppStackCombineBackground(stack, options, config)) { + if (options->doBackground && !ppStackCombineBackground(stack, options, config)) { psError(psErrorCodeLast(), false, "Unable to generate median of background images."); psFree(stack); Index: /branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.c =================================================================== --- /branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.c (revision 35236) @@ -43,4 +43,5 @@ options->matchZPs = true; options->photometry = false; + options->doBackground = false; options->stats = NULL; options->statsFile = NULL; Index: /branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.h =================================================================== --- /branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.h (revision 35235) +++ /branches/eam_branches/ipp-20130207/ppStack/src/ppStackOptions.h (revision 35236) @@ -8,4 +8,5 @@ bool matchZPs; // Adjust relative fluxes based on transparency analysis? bool photometry; // Perform photometry? + bool doBackground; // Do background model combination? psMetadata *stats; // Statistics for output FILE *statsFile; // File to which to write statistics Index: /branches/eam_branches/ipp-20130207/ppStack/src/ppStackSetup.c =================================================================== --- /branches/eam_branches/ipp-20130207/ppStack/src/ppStackSetup.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/ppStack/src/ppStackSetup.c (revision 35236) @@ -17,4 +17,6 @@ options->convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images? + options->doBackground = psMetadataLookupBool(NULL, recipe, "BACKGROUND.MODEL"); + if (!psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) { psWarning("No PSFs provided --- unable to convolve to common PSF."); Index: /branches/eam_branches/ipp-20130207/psLib/src/imageops/psImageInterpolate.c =================================================================== --- /branches/eam_branches/ipp-20130207/psLib/src/imageops/psImageInterpolate.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/psLib/src/imageops/psImageInterpolate.c (revision 35236) @@ -47,5 +47,6 @@ 4, // LANCZOS2 6, // LANCZOS3 - 8 // LANCZOS4 + 8, // LANCZOS4 + 2 // BILINEAR_SIMPLE }; Index: /branches/eam_branches/ipp-20130207/psModules/src/imcombine/pmStack.c =================================================================== --- /branches/eam_branches/ipp-20130207/psModules/src/imcombine/pmStack.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/psModules/src/imcombine/pmStack.c (revision 35236) @@ -1418,5 +1418,7 @@ } combined->image->data.F32[y][x] = stats->robustMedian; - combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0; + if (combined->mask) { + combined->mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0; + } #if (0) if ((x == 59)&&(y > 40)&&(y < 50)) { Index: /branches/eam_branches/ipp-20130207/psModules/src/objects/models/pmModel_TRAIL.c =================================================================== --- /branches/eam_branches/ipp-20130207/psModules/src/objects/models/pmModel_TRAIL.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/psModules/src/objects/models/pmModel_TRAIL.c (revision 35236) @@ -256,5 +256,7 @@ if (!isfinite(axes.theta)) return false; - float size = (axes.major > source->moments->Mrf) ? axes.major : source->moments->Mrf; + float size = (axes.major > sqrt(source->moments->Mrf)) ? axes.major : sqrt(source->moments->Mrf); + // if (size > psfAxes.major) { size -= psfAxes.major; } + //else { size = psfAxes.major; } // axes.major is a sigma in the major direction; scale to Index: /branches/eam_branches/ipp-20130207/psphot/src/psphotSourceSize.c =================================================================== --- /branches/eam_branches/ipp-20130207/psphot/src/psphotSourceSize.c (revision 35235) +++ /branches/eam_branches/ipp-20130207/psphot/src/psphotSourceSize.c (revision 35236) @@ -423,4 +423,5 @@ psLogMsg("psModules.objects", PS_LOG_INFO, "Source Size classifications: %4s %4s %4s %4s %4s", "Npsf", "Next", "Nsat", "Ncr", "Nskip"); // Determine if this readout is above the threshold to ext fit all sources + options->extFitAllReadout = false; if (options->extFitAll) { float maskFrac = psMetadataLookupF32(&status,readout->analysis,"READOUT.MASK.FRAC"); @@ -584,4 +585,5 @@ // the EXT_LIMIT flag if (options->extFitAllReadout) { + psTrace("psphotSourceClassRegion.EXTALT",10,"In extFitAll: %d %d\n",options->extFitAll,options->extFitAllReadout); source->mode |= PM_SOURCE_MODE_EXT_LIMIT; } Index: /branches/eam_branches/ipp-20130207/tools/fixburntool =================================================================== --- /branches/eam_branches/ipp-20130207/tools/fixburntool (revision 35235) +++ /branches/eam_branches/ipp-20130207/tools/fixburntool (revision 35236) @@ -106,5 +106,5 @@ { - my $command = "$burntool --continue 10 --exp_id $exp_id --class_id $class_id --this_uri $uri --previous_uri $prev_uri"; + my $command = "$burntool --exp_id $exp_id --class_id $class_id --this_uri $uri --previous_uri $prev_uri"; print "$command\n"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = Index: /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/README =================================================================== --- /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/README (revision 35236) +++ /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/README (revision 35236) @@ -0,0 +1,30 @@ +this is how I make my plots... + +Step 1: on ippc18 get whatever has been processed in LAP label for addstar etc: + +mysql -h ippdb01 -u ipp -pipp gpc1 < grab_cam_LAP.sql > cam.20130215.txt +mysql -h ippdb01 -u ipp -pipp gpc1 < grab_skycal_LAP.sql > skycal.20130215.txt + +Step 2: rsync those files back home: + +rsync -rvua skycal.20130215.txt heather@canoes: +rsync -rvua cam.20130215.txt heather@canoes: + +Step 3: parse the .txt files to something for my script: + +perl mangle.LAP.cam.pl cam.20130215.txt > cam.20130215.mng +perl mangle.LAP.cam.pl skycal.20130215.txt > skycal.20130215.mng + +(yes, same perl script for both) + +Step 4: edit plot.cam.py and plot.sky.py to use the correct input files and make the correct output files + +Step 5: run plot.cam.py and plot.sky.py + +python plot.cam.py +python plot.sky.py + +Step 6: look at plots :) + +(see http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/AddStarLap20120706) + Index: /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/grab_cam_LAP.sql =================================================================== --- /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/grab_cam_LAP.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/grab_cam_LAP.sql (revision 35236) @@ -0,0 +1,1 @@ +select ra,decl, addRun.state, addProcessedExp.fault from addRun left join addProcessedExp using (add_id) join camRun on stage_id = cam_id join chipRun using (chip_id) join rawExp using (exp_id) where stage = 'cam' and addRun.dvodb = 'LAP.20120706'; Index: /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/grab_skycal_LAP.sql =================================================================== --- /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/grab_skycal_LAP.sql (revision 35236) +++ /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/grab_skycal_LAP.sql (revision 35236) @@ -0,0 +1,1 @@ +select radeg*3.1415927/180., decdeg*3.1415927/180., addRun.state, addProcessedExp.fault FROM addRun left join addProcessedExp using (add_id) join skycalRun on skycal_id = stage_id JOIN skycalResult using (skycal_id) JOIN staticskyInput using (sky_id, stack_id) JOIN staticskyRun using (sky_id) JOIN stackRun using (stack_id) JOIN skycell using (tess_id, skycell_id) where stage = 'skycal' and addRun.dvodb = 'LAP.20120706' Index: /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/mangle.LAP.cam.pl =================================================================== --- /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/mangle.LAP.cam.pl (revision 35236) +++ /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/mangle.LAP.cam.pl (revision 35236) @@ -0,0 +1,38 @@ +#!/usr/bin/env perl + +#this script expects ra dec state fault + +#state 0 - full and no fault +#state 1 - new and pending +#state 2 - faulted somehow + + + + +while (<>) { + if ($_ =~ 'ra') { next;} + my @split = split /\s+/, $_; + if ($split[2] =~ 'full') { + if ($split[3] == 0) { + $state = 0; + } else { + $state = 2; + } + } else { + #assume it is 'new' + if ($split[3] =~ 'NULL') { + $state = 1; + } else { + $state = 2; + } + + } + + $ra = $split[0]*180/3.1415927; + $dec = $split[1]*180/3.1415927; + + + print "$ra $dec $state\n"; + + +} Index: /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/plot.cam.py =================================================================== --- /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/plot.cam.py (revision 35236) +++ /branches/eam_branches/ipp-20130207/tools/heathers_plots/alpha/plot.cam.py (revision 35236) @@ -0,0 +1,36 @@ +import numpy as np +#np.seterr("raise") +import matplotlib.pyplot as plt +import matplotlib as mpl +from pylab import * +filename = 'cam.20130219.mng' + +stuff = np.loadtxt(filename,usecols=(0,1,2),dtype = [('f0','