Index: /branches/czw_branch/20110406/Nebulous-Server/lib/Nebulous/Server.pm =================================================================== --- /branches/czw_branch/20110406/Nebulous-Server/lib/Nebulous/Server.pm (revision 31433) +++ /branches/czw_branch/20110406/Nebulous-Server/lib/Nebulous/Server.pm (revision 31434) @@ -2546,9 +2546,9 @@ } - if ($xattr) { - my $path = $uri->file; - $log->logdie("can not set xattr on $path: $!") - unless (setfattr($path, 'user.nebulous_key', $key->path)); - } +# if ($xattr) { +# my $path = $uri->file; +# $log->logdie("can not set xattr on $path: $!") +# unless (setfattr($path, 'user.nebulous_key', $key->path)); +# } return $uri; Index: /branches/czw_branch/20110406/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- /branches/czw_branch/20110406/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 31433) +++ /branches/czw_branch/20110406/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 31434) @@ -336,4 +336,5 @@ AND m.available = ? AND m.allocate = ? + AND m.xattr = 0 AND ( (v.cab_id IS NULL) || (v.cab_id != ?) ) @@ -344,4 +345,5 @@ }, # volume handler + # This has a hack to get around the lack of location awareness by using xattr. get_storage_volume => qq{ SELECT * from ( @@ -357,4 +359,5 @@ AND available = ? AND allocate = ? + AND xattr = 0 ORDER BY free DESC LIMIT ?) as topfew Index: /branches/czw_branch/20110406/Nebulous/bin/neb-df =================================================================== --- /branches/czw_branch/20110406/Nebulous/bin/neb-df (revision 31433) +++ /branches/czw_branch/20110406/Nebulous/bin/neb-df (revision 31434) @@ -16,5 +16,5 @@ use Pod::Usage qw( pod2usage ); -my ($server); +my ($server,$names); $server = $ENV{'NEB_SERVER'} unless $server; @@ -22,4 +22,5 @@ GetOptions( 'server|s=s' => \$server, + 'names' => \$names, ) || pod2usage( 2 ); @@ -51,4 +52,8 @@ = @$row; my $path = $vol{host}; + if ($names) { + $path = $vol{name}; + } + my $total = $vol{total}; $total_total += $total; Index: /branches/czw_branch/20110406/Ohana/src/addstar/src/GetFileMode.c =================================================================== --- /branches/czw_branch/20110406/Ohana/src/addstar/src/GetFileMode.c (revision 31433) +++ /branches/czw_branch/20110406/Ohana/src/addstar/src/GetFileMode.c (revision 31434) @@ -4,5 +4,5 @@ int GetFileMode (Header *header) { - char ctype[80], ctmp; + char ctype[81], ctmp[80]; int Naxis; int simple, extend, haveNaxis, haveCTYPE; @@ -17,6 +17,6 @@ // SDSS tsObj files have a version number for the PHOTO and // TS (target selection) pipelines present as header keywords - havePHOT_VER = gfits_scan (header, "PHOT_VER", "%s", 1, &ctmp); - haveTARG_VER = gfits_scan (header, "TARG_VER", "%s", 1, &ctmp); + havePHOT_VER = gfits_scan (header, "PHOT_VER", "%s", 1, ctmp); + haveTARG_VER = gfits_scan (header, "TARG_VER", "%s", 1, ctmp); if (havePHOT_VER && haveTARG_VER) return SDSS_OBJ; Index: /branches/czw_branch/20110406/Ohana/src/addstar/src/ReadImageHeader.c =================================================================== --- /branches/czw_branch/20110406/Ohana/src/addstar/src/ReadImageHeader.c (revision 31433) +++ /branches/czw_branch/20110406/Ohana/src/addstar/src/ReadImageHeader.c (revision 31434) @@ -6,5 +6,5 @@ int Nastro, hour, min, Nx, Ny, haveNx, haveNy, sourceID; - double tmp, sec, Cerror, ZeroPt, FWHM_X, FWHM_Y; + double tmp, sec, Cerror, FWHM_X, FWHM_Y; char *c, photname[64], line[80], ccdnum[64]; PhotCode *photcodeData = NULL; @@ -205,12 +205,15 @@ } +#ifdef notdef // XXX this is archaic: we used to set a fixed zero point of 25 to shift data into the range // 0 - 32 so it would fit in an unsigned int. This is also needed because some programs like // sextractor will put in an arbitrary zero point that we need to understand to get back to // instrumental mags. + double ZeroPt; gfits_scan (header, "ZERO_PT", "%lf", 1, &ZeroPt); if (ZeroPt != GetZeroPoint()) { fprintf (stderr, "WARNING: inconsistent zero point values: image: %f, config: %f\n", ZeroPt, GetZeroPoint()); } +#endif // in this case, lookup and apply the zero point measured for this chip Index: /branches/czw_branch/20110406/Ohana/src/libfits/table/F_get_column.c =================================================================== --- /branches/czw_branch/20110406/Ohana/src/libfits/table/F_get_column.c (revision 31433) +++ /branches/czw_branch/20110406/Ohana/src/libfits/table/F_get_column.c (revision 31434) @@ -30,9 +30,9 @@ tlabel[0] = 0; if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return (NULL); - for (i = 1; strcmp (label, tlabel) && (i < Nfields + 1); i++) { + for (i = 1; strcasecmp (label, tlabel) && (i < Nfields + 1); i++) { sprintf (field, "TTYPE%d", i); gfits_scan (header, field, "%s", 1, tlabel); } - if (strcmp (label, tlabel)) return (NULL); + if (strcasecmp (label, tlabel)) return (NULL); N = i - 1; @@ -137,9 +137,9 @@ tlabel[0] = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - for (i = 1; strcmp (label, tlabel) && (i < Nfields + 1); i++) { + for (i = 1; strcasecmp (label, tlabel) && (i < Nfields + 1); i++) { sprintf (field, "TTYPE%d", i); gfits_scan (header, field, "%s", 1, tlabel); } - if (strcmp (label, tlabel)) return (FALSE); + if (strcasecmp (label, tlabel)) return (FALSE); N = i - 1; @@ -166,9 +166,9 @@ tlabel[0] = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - for (i = 1; strcmp (label, tlabel) && (i < Nfields + 1); i++) { + for (i = 1; strcasecmp (label, tlabel) && (i < Nfields + 1); i++) { sprintf (field, "TTYPE%d", i); gfits_scan (header, field, "%s", 1, tlabel); } - if (strcmp (label, tlabel)) return (FALSE); + if (strcasecmp (label, tlabel)) return (FALSE); N = i - 1; @@ -291,9 +291,9 @@ tlabel[0] = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - for (i = 1; strcmp (label, tlabel) && (i < Nfields + 1); i++) { + for (i = 1; strcasecmp (label, tlabel) && (i < Nfields + 1); i++) { sprintf (field, "TTYPE%d", i); gfits_scan (header, field, "%s", 1, tlabel); } - if (strcmp (label, tlabel)) return (FALSE); + if (strcasecmp (label, tlabel)) return (FALSE); N = i - 1; @@ -321,9 +321,9 @@ tlabel[0] = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - for (i = 1; strcmp (label, tlabel) && (i < Nfields + 1); i++) { + for (i = 1; strcasecmp (label, tlabel) && (i < Nfields + 1); i++) { sprintf (field, "TTYPE%d", i); gfits_scan (header, field, "%s", 1, tlabel); } - if (strcmp (label, tlabel)) return (FALSE); + if (strcasecmp (label, tlabel)) return (FALSE); N = i - 1; Index: /branches/czw_branch/20110406/Ohana/src/libfits/table/F_set_column.c =================================================================== --- /branches/czw_branch/20110406/Ohana/src/libfits/table/F_set_column.c (revision 31433) +++ /branches/czw_branch/20110406/Ohana/src/libfits/table/F_set_column.c (revision 31434) @@ -31,9 +31,9 @@ tlabel[0] = 0; gfits_scan (header, "TFIELDS", "%d", 1, &Nfields); - for (i = 1; strcmp (label, tlabel) && (i < Nfields + 1); i++) { + for (i = 1; strcasecmp (label, tlabel) && (i < Nfields + 1); i++) { sprintf (field, "TTYPE%d", i); gfits_scan (header, field, "%s", 1, tlabel); } - if (strcmp (label, tlabel)) return (FALSE); + if (strcasecmp (label, tlabel)) return (FALSE); N = i - 1; Index: /branches/czw_branch/20110406/dbconfig/add.md =================================================================== --- /branches/czw_branch/20110406/dbconfig/add.md (revision 31433) +++ /branches/czw_branch/20110406/dbconfig/add.md (revision 31434) @@ -2,5 +2,6 @@ addRun METADATA add_id S64 0 # Primary Key AUTO_INCREMENT - cam_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + stage STR 64 # what the stage is (warp, cam, diff, + stage_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) state STR 64 # Key workdir STR 255 Index: /branches/czw_branch/20110406/dbconfig/changes.txt =================================================================== --- /branches/czw_branch/20110406/dbconfig/changes.txt (revision 31433) +++ /branches/czw_branch/20110406/dbconfig/changes.txt (revision 31434) @@ -2057,4 +2057,20 @@ UPDATE dbversion set schema_version = '1.1.69', updated= CURRENT_TIMESTAMP(); +ALTER TABLE addRun ADD COLUMN stage VARCHAR(64) AFTER add_id; +ALTER TABLE addRun ADD COLUMN stage_id bigint(20) after stage; +UPDATE addRun set stage_id = cam_id; + +-- these next steps I don't know if it's generic or not. This is how to drop + the cam_id table in addRun for gpc1 + +show create table addRun; +alter table `addRun` drop foreign key `addRun_ibfk_1`, drop column cam_id; + +-- this sets the addRun stage for all the old addRuns to 'cam' + + +--do this update only once, so I'm commenting it out (it's been done) +-- update addRun set stage = 'cam'; + -- Version 1.1.70 @@ -2115,2 +2131,3 @@ UPDATE dbversion set schema_version = '1.1.70', updated= CURRENT_TIMESTAMP(); + Index: /branches/czw_branch/20110406/ippMonitor/def/addRunProcessed.d =================================================================== --- /branches/czw_branch/20110406/ippMonitor/def/addRunProcessed.d (revision 31433) +++ /branches/czw_branch/20110406/ippMonitor/def/addRunProcessed.d (revision 31434) @@ -11,5 +11,6 @@ FIELD distinct(addRun.add_id), 5, %d, add ID -FIELD addRun.cam_id, 5, %d, cam ID +FIELD addRun.stage, 5, %s, stage +FIELD addRun.stage_id, 5, %d, cam ID FIELD addRun.state, 5, %s, state FIELD addProcessedExp.fault, 5, %d, fault Index: /branches/czw_branch/20110406/ippMonitor/def/addRunRun.d =================================================================== --- /branches/czw_branch/20110406/ippMonitor/def/addRunRun.d (revision 31433) +++ /branches/czw_branch/20110406/ippMonitor/def/addRunRun.d (revision 31434) @@ -10,6 +10,6 @@ # field size format name show link to extras FIELD distinct(addRun.add_id), 5, %d, add ID - -FIELD addRun.cam_id, 5, %d, cam ID +FIELD addRun.stage, 5, %s, stage +FIELD addRun.stage_id, 5, %d, stage ID FIELD addRun.add_id, 5, %d, add ID FIELD addRun.state, 5, %s, state Index: /branches/czw_branch/20110406/ippScripts/Build.PL =================================================================== --- /branches/czw_branch/20110406/ippScripts/Build.PL (revision 31433) +++ /branches/czw_branch/20110406/ippScripts/Build.PL (revision 31434) @@ -117,4 +117,5 @@ scripts/skycell_jpeg.pl scripts/diffphot.pl + scripts/lap_science.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: /branches/czw_branch/20110406/ippScripts/MANIFEST =================================================================== --- /branches/czw_branch/20110406/ippScripts/MANIFEST (revision 31433) +++ /branches/czw_branch/20110406/ippScripts/MANIFEST (revision 31434) @@ -45,3 +45,4 @@ scripts/ipp_cluster_load_monitor.pl scripts/skycell_jpeg.pl +scripts/lap_science.pl t/00_distribution.t Index: /branches/czw_branch/20110406/ippScripts/scripts/addstar_run.pl =================================================================== --- /branches/czw_branch/20110406/ippScripts/scripts/addstar_run.pl (revision 31433) +++ /branches/czw_branch/20110406/ippScripts/scripts/addstar_run.pl (revision 31434) @@ -37,13 +37,14 @@ } my $minidvodb_path; -my ( $exp_tag, $add_id, $camera, $outroot, $camroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my ( $add_id, $camera, $stage, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( - 'exp_tag=s' => \$exp_tag, # Exposure identifier 'add_id=s' => \$add_id, # Camtool identifier 'camera|c=s' => \$camera, # Camera + 'stage|s=s' => \$stage, # Camera + 'dbname|d=s' => \$dbname, # Database name 'outroot|w=s' => \$outroot, # output file base name - 'camroot|w=s' => \$camroot, # camera stage root name. + 'stageroot|w=s' => \$stageroot, # stage root name. 'reduction=s' => \$reduction, # Reduction class 'dvodb|w=s' => \$dvodb, # output DVO database @@ -61,11 +62,11 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --exp_tag --add_id --camera --outroot --dvodb --camroot", + -msg => "Required options: --add_id --camera --outroot --dvodb --stageroot --stage", -exitval => 3, ) unless - defined $exp_tag and + defined $stage and defined $add_id and defined $outroot and - defined $camroot and + defined $stageroot and defined $dvodb and defined $camera; @@ -86,7 +87,15 @@ # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; - +my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # XXX This is now not used: do we still need it? -my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +if ($stage =~/cam/) { + $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use +} +if ($stage =~/stack/) { + $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STACK'); # Recipe to use +} +#if ($stage =~/staticsky/) { +# $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR_STATICSKY'); # Recipe to use +#} &my_die("Unrecognised ADDSTAR recipe", $add_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; @@ -97,5 +106,12 @@ # the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $camroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +# it was PSASTRO.OUTPUT +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if (($stage =~/staticsky/) || ($stage =~/stack/)) { + $fpaObjects =~ s/smf$/cmf/; + +} my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); @@ -173,6 +189,8 @@ $command .= " $realFile"; $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference - $command .= " -image" if $image_only; - + $command .= " -image" if $image_only; + if ($stage = ~/staticsky/) { + $command .= " -accept-astrom "; + } my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script Index: /branches/czw_branch/20110406/ippScripts/scripts/nightly_science.pl =================================================================== --- /branches/czw_branch/20110406/ippScripts/scripts/nightly_science.pl (revision 31433) +++ /branches/czw_branch/20110406/ippScripts/scripts/nightly_science.pl (revision 31434) @@ -996,4 +996,5 @@ $cmd .= " -min_num 4"; $cmd .= " -select_good_frac_min 0.05"; + $cmd .= " -set_reduction NIGHTLY_STACK "; $cmd .= " $select "; if ($debug == 1) { Index: /branches/czw_branch/20110406/ippScripts/scripts/stack_skycell.pl =================================================================== --- /branches/czw_branch/20110406/ippScripts/scripts/stack_skycell.pl (revision 31433) +++ /branches/czw_branch/20110406/ippScripts/scripts/stack_skycell.pl (revision 31434) @@ -181,6 +181,7 @@ my $recipe_psphot = $ipprc->reduction($reduction, 'STACK_PSPHOT'); # Recipe to use for psphot my $recipe_ppstats = 'STACKSTATS'; -if ($run_state eq 'update') { - $recipe_ppstats = 'WARPSTATS'; +if ($ +run_state eq 'update') { + $recipe_ppstats = 'WARPSTATS'; } unless ($recipe_ppStack and $recipe_ppSub and $recipe_psphot) { @@ -208,4 +209,6 @@ &my_die("Unable to not compress and logflux compress simultaneously. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR); } +my $stack_type = metadataLookupStr($recipe, 'STACK.TYPE'); +&my_die("STACK.TYPE not found in recipe. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR) unless $stack_type; # Generate MDC file with the inputs @@ -281,4 +284,5 @@ $command .= " -recipe PSPHOT $recipe_psphot"; $command .= " -recipe PPSTATS $recipe_ppstats" if $do_stats;; + $command .= " -stack-type $stack_type"; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE"; $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF"; Index: /branches/czw_branch/20110406/ippScripts/scripts/staticsky.pl =================================================================== --- /branches/czw_branch/20110406/ippScripts/scripts/staticsky.pl (revision 31433) +++ /branches/czw_branch/20110406/ippScripts/scripts/staticsky.pl (revision 31434) @@ -100,15 +100,4 @@ # Recipes to use based on reduction class $reduction = 'DEFAULT' unless defined $reduction; -my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); # Recipe to use for psphot -my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); # Recipe to use for ppsub -my $recipe_ppstack = $ipprc->reduction($reduction, 'STACKPHOT_PPSTACK'); # Recipe to use for ppstack -unless ($recipe_psphot) { - &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); -} - -print "reduction: $reduction\n"; -print "recipe_psphot: $recipe_psphot\n"; -print "recipe_ppsub: $recipe_ppsub\n"; -print "recipe_ppstack: $recipe_ppstack\n"; @@ -118,4 +107,15 @@ if ($nInputs > 1) { + my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_PSPHOT'); # Recipe to use for psphot + my $recipe_ppsub = $ipprc->reduction($reduction, 'STACKPHOT_PPSUB'); # Recipe to use for ppsub + my $recipe_ppstack = $ipprc->reduction($reduction, 'STACKPHOT_PPSTACK'); # Recipe to use for ppstack + unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); + } + + print "reduction: $reduction\n"; + print "recipe_psphot: $recipe_psphot\n"; + print "recipe_ppsub: $recipe_ppsub\n"; + print "recipe_ppstack: $recipe_ppstack\n"; my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); @@ -177,6 +177,4 @@ $command .= " -tracedest $traceDest -log $logDest"; # $command .= " -dbname $dbname" if defined $dbname; - # $command .= " -image_id $diff_skyfile_id" if defined $diff_skyfile_id; - # $command .= " -source_id $source_id" if defined $source_id; unless ($no_op) { @@ -242,5 +240,14 @@ } else { # single input. Run psphot - my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot); + # find the recipe + my $recipe_psphot = $ipprc->reduction($reduction, 'STACKPHOT_SINGLE_PSPHOT'); # Recipe to use for psphot + unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STACKPHOT: $reduction\n", $sky_id, $PS_EXIT_CONFIG_ERROR); + } + + print "reduction: $reduction\n"; + print "recipe_psphot: $recipe_psphot\n"; + + my $configuration = $ipprc->filename("PSPHOT.SKY.CONFIG", $outroot); my $file = $files->[0]; @@ -274,8 +281,6 @@ } - my $sources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base ); # Sources name - - # XXX is this the correct PSF file? - my $psfCnv = $ipprc->filename("PPSTACK.TARGET.PSF", $path_base ); # PSF name + my $output_sources_filerule = "PSPHOT.OUT.CMF.MEF"; + my $output_psf_filerule = "PSPHOT.PSF.SKY.SAVE"; # XXX we could make some different choices if some inputs do not exist... @@ -283,8 +288,4 @@ &my_die("Couldn't find input: $mask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$mask"); &my_die("Couldn't find input: $variance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$variance"); - &my_die("Couldn't find input: $psfCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv"); - &my_die("Couldn't find input: $sources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources"); - - # my $cmdflags; # Perform stack photometry analysis @@ -298,4 +299,6 @@ $command .= " -dumpconfig $configuration" if $configuration; $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -F PSPHOT.OUTPUT $output_sources_filerule"; + $command .= " -F PSPHOT.PSF.SAVE $output_psf_filerule"; unless ($no_op) { @@ -303,56 +306,17 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform psphotStack: $error_code", $sky_id, $error_code); + &my_die("Unable to perform psphot: $error_code", $sky_id, $error_code); } - # my $outputStatsReal = $ipprc->file_resolve($outputStats); - # &my_die("Couldn't find expected output file: $outputStats", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal); - - # measure chip stats - # $command = "$ppStatsFromMetadata $outputStatsReal - DIFF_SKYCELL"; - # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - # run(command => $command, verbose => $verbose); - # unless ($success) { - # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - # &my_die("Unable to perform ppStatsFromMetadata: $error_code", $sky_id, $error_code); - # } - # foreach my $line (@$stdout_buf) { - # $cmdflags .= " $line"; - # } - # chomp $cmdflags; - - # my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag - my $quality = 0; - if (0 and !$quality) { - - # Get the output filenames - # we have one set of output files per input file set - for (my $i = 0; $i < @$files; $i++) { - my $outputName = $ipprc->filename("PSPHOT.STACK.OUTPUT.IMAGE", $outroot, $i); - my $outputMask = $ipprc->filename("PSPHOT.STACK.OUTPUT.MASK", $outroot, $i); - my $outputVariance = $ipprc->filename("PSPHOT.STACK.OUTPUT.VARIANCE", $outroot, $i); - my $outputSources = $ipprc->filename("PSPHOT.STACK.OUTPUT", $outroot, $i); - - # XXX these are optional and not generated by default - # &my_die("Couldn't find expected output file: $outputName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName); - # &my_die("Couldn't find expected output file: $outputMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask); - # &my_die("Couldn't find expected output file: $outputVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance); - &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); - } - - #my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); - #my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot); - #my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); - - my $chisqName = $ipprc->filename("PSPHOT.CHISQ.IMAGE", $outroot); - my $chisqMask = $ipprc->filename("PSPHOT.CHISQ.MASK", $outroot); - my $chisqVariance = $ipprc->filename("PSPHOT.CHISQ.VARIANCE", $outroot); - - # XXX check the recipe -- should we expect these to exist? - # &my_die("Couldn't find expected output file: $chisqName", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqName); - # &my_die("Couldn't find expected output file: $chisqMask", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqMask); - # &my_die("Couldn't find expected output file: $chisqVariance", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($chisqVariance); + if (!$quality) { + my $outputSources = $ipprc->filename($output_sources_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputSources", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources); + my $outputPSF = $ipprc->filename($output_psf_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputPSF", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputPSF); } + + &my_die("Couldn't find expected output file: $configuration", $sky_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($configuration); } else { print "Not executing: $command\n"; Index: /branches/czw_branch/20110406/ippTasks/addstar.pro =================================================================== --- /branches/czw_branch/20110406/ippTasks/addstar.pro (revision 31433) +++ /branches/czw_branch/20110406/ippTasks/addstar.pro (revision 31434) @@ -17,5 +17,11 @@ macro addstar.on - task addstar.exp.load + task addstar.exp.load.stack + active true + end + task addstar.exp.load.cam + active true + end + task addstar.exp.load.staticsky active true end @@ -26,5 +32,11 @@ macro addstar.off - task addstar.exp.load + task addstar.exp.load.stack + active false + end + task addstar.exp.load.cam + active false + end + task addstar.exp.load.staticsky active false end @@ -54,5 +66,5 @@ # new entries are added to addPendingExp # skip already-present entries -task addstar.exp.load +task addstar.exp.load.stack host local @@ -67,5 +79,5 @@ task.exec # if ($LABEL:n == 0) break - $run = addtool -pendingexp + $run = addtool -pendingexp -stage stack if ($DB:n == 0) option DEFAULT @@ -109,4 +121,112 @@ end +task addstar.exp.load.cam + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + # if ($LABEL:n == 0) break + $run = addtool -pendingexp -stage cam + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task addstar.exp.load.staticsky + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + # if ($LABEL:n == 0) break + $run = addtool -pendingexp -stage staticsky + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end # run the addstar script on pending exposures task addstar.exp.run @@ -132,5 +252,6 @@ book getword addPendingExp $pageName exp_tag -var EXP_TAG book getword addPendingExp $pageName add_id -var ADD_ID - book getword addPendingExp $pageName camroot -var CAMROOT + book getword addPendingExp $pageName stageroot -var STAGEROOT + book getword addPendingExp $pageName stage -var STAGE book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE book getword addPendingExp $pageName reduction -var REDUCTION @@ -157,10 +278,20 @@ ## generate outroot specific to this exposure (& chip) - sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + + if ("$STAGE" == "cam") + sprintf outroot "%s/%s/%s.add.%s" $WORKDIR $EXP_TAG $EXP_TAG $ADD_ID + end + if ("$STAGE" == "staticsky") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + if ("$STAGE" == "stack") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end + stdout $LOGDIR/addstar.exp.log stderr $LOGDIR/addstar.exp.log - $run = addstar_run.pl --exp_tag $EXP_TAG --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --camroot $CAMROOT --outroot $outroot --redirect-output + $run = addstar_run.pl --add_id $ADD_ID --camera $CAMERA --dvodb $DVODB --stage $STAGE --stageroot $STAGEROOT --outroot $outroot --redirect-output if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION Index: /branches/czw_branch/20110406/ippTasks/survey.pro =================================================================== --- /branches/czw_branch/20110406/ippTasks/survey.pro (revision 31433) +++ /branches/czw_branch/20110406/ippTasks/survey.pro (revision 31434) @@ -231,6 +231,6 @@ macro survey.add.addstar - if ($0 != 4) - echo "USAGE: survey.add.addstar (label) (dvodb) (minidvodb_group)" + if ($0 != 5) + echo "USAGE: survey.add.addstar (label) (dvodb) (minidvodb_group) (stage)" break end @@ -238,4 +238,5 @@ book setword SURVEY_ADDSTAR $1 DVODB $2 book setword SURVEY_ADDSTAR $1 MINIDVODB_GROUP $3 + book setword SURVEY_ADDSTAR $1 STAGE $4 book setword SURVEY_ADDSTAR $1 STATE PENDING end @@ -753,4 +754,6 @@ book getword SURVEY_ADDSTAR $label DVODB -var dvodb book getword SURVEY_ADDSTAR $label MINIDVODB_GROUP -var minidvodb_group + book getword SURVEY_ADDSTAR $label STAGE -var stage + # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb @@ -763,6 +766,12 @@ end - $run = $run -definebyquery -destreaked -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $minidvodb_group + $run = $run -definebyquery -label $label -set_dvodb $dvodb -set_minidvodb_group $minidvodb_group -set_minidvodb -set_label $minidvodb_group -stage $stage # echo $run + if ($stage == 'cam') + #only queue destreaked cams. stacks and staticsky don't need this + $run = $run -destreaked + end + + command $run end Index: /branches/czw_branch/20110406/ippToPsps/config/detection/map.xml =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/detection/map.xml (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/detection/map.xml (revision 31434) @@ -146,4 +146,10 @@ --> --> + + + + + + Index: /branches/czw_branch/20110406/ippToPsps/config/detection/tables.xml =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/detection/tables.xml (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/detection/tables.xml (revision 31434) @@ -1,5 +1,5 @@ - + @@ -146,4 +146,10 @@ + + + + + + Index: /branches/czw_branch/20110406/ippToPsps/config/init/data.xml =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/init/data.xml (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/init/data.xml (revision 31434) @@ -4,6 +4,6 @@
- - + +
Index: /branches/czw_branch/20110406/ippToPsps/config/init/tables.vot =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/init/tables.vot (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/init/tables.vot (revision 31434) @@ -6,7 +6,13 @@ VOTable description of PSPS table Filter - - - + + filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ... + + + filter name: g,r,i,z,y, etc. + + + No comment + @@ -23,53 +29,137 @@ VOTable description of PSPS table FitModel - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + indentifier of alternate model fit + + + model name + + + string describing the model + + + description of model parameter 1 + + + description of model parameter 2 + + + description of model parameter 3 + + + description of model parameter 4 + + + description of model parameter 5 + + + description of model parameter 6 + + + description of model parameter 7 + + + description of model parameter 8 + + + description of model parameter 9 + + + description of model parameter 10 + + + description of model parameter 11 + + + description of model parameter 12 + + + description of model parameter 13 + + + description of model parameter 14 + + + description of model parameter 15 + + + description of model parameter 16 + + + description of model parameter 17 + + + description of model parameter 18 + + + description of model parameter 19 + + + description of model parameter 20 + + + description of model parameter 21 + + + description of model parameter 22 + + + description of model parameter 23 + + + description of model parameter 24 + + + description of model parameter 25 + + + description of model parameter 26 + + + description of model parameter 27 + + + description of model parameter 28 + + + description of model parameter 29 + + + description of model parameter 30 + + + description of model parameter 31 + + + description of model parameter 32 + + + description of model parameter 33 + + + description of model parameter 34 + + + description of model parameter 35 + + + description of model parameter 36 + + + description of model parameter 37 + + + description of model parameter 38 + + + description of model parameter 39 + + 0PS_MODEL_PSF_AVGPsf avg fit 1PS_MODEL_PSFPsf fit 2PS_MODEL_DEVdeVaucouleur Fit 3PS_MODEL_EXPExponential Fit 4PS_MODEL_SERSICSersic Fit - 5PS_MODEL_LINLinear Streak Fitx1x2x3x4 @@ -78,6 +168,10 @@ VOTable description of PSPS table PhotozRecipe - - + + recipe index + + + string describing the recipe + @@ -87,38 +181,44 @@ VOTable description of PSPS table Survey - - - + + index of survey type + + + survey name + + + survey description + - 0PS1 3PI Survey3PI - 1PS1 MD01 XMM-LSS-DXS 022224-043000MD01 - 2PS1 MD02 CDFS/GOODS/GEMS 033224-274800MD02 - 3PS1 MD03 IFA/Lynx 084300+444000MD03 - 4PS1 MD04 COSMOS 100000+021200MD04 - 5PS1 MD05 Lockman-DXS 110000+570000MD05 - 6PS1 MD06 NGC 4258 121857+471814MD06 - 7PS1 MD07 VISTA-Video1 140000+050000MD07 - 8PS1 MD08 EliasN1-DXS 160000+570000MD08 - 9PS1 MD09 Vimos4-DXS-SSA 220000+003000MD09 - 10PS1 MD10 Deep2 233000+000000MD10 - 11PS1 M31 Andromeda 004242+411600M31 - 12PS1 STS1 Outskirts of Bulge 195048+170339STS1 - 26PS1 STS2 Hyades 004000+150000STS2 - 27PS1 STS3 Praesepe 083000+200000STS3 - 13PS1 Cal North Pole 000000+900000CAL01 - 14PS1 Cal SA92 Dense Landolt Std Field 005808+004232CAL02 - 15PS1 Cal Perseus Cluster / Abell 426 031900+413300CAL03 - 16PS1 Cal SA95 Dense Landolt Std Field 065207+001710CAL04 - 17PS1 Cal Taurus dark cloud (diverse extinction) 043000+250000CAL05 - 18PS1 Cal SA98 Densle Landolt Std Field 065207+001710CAL06 - 19PS1 Cal M81-M82-HolmIX (Local Group) 095640+692200CAL07 - 20PS1 Cal Virgo Cluster (Center) 123000+130000CAL08 - 21PS1 Cal Hercules Cluster (Center) 160508+1745+28CAL09 - 22PS1 Cal M17-M24 Region 181400+000544CAL10 - 23PS1 Cal SA110 Dense Landolt Std Field 184207+000544CAL11 - 24PS1 Cal Kepler Mission Astrometric field 192240+443000CAL12 - 25PS1 Cal 3C454.1 (Northern field) 225033+712919CAL13 - 28PS1 Solar System Sweet Spot Survey SSS + 03PIPS1 3PI Survey + 1MD01PS1 MD01 XMM-LSS-DXS 022224-043000 + 2MD02PS1 MD02 CDFS/GOODS/GEMS 033224-274800 + 3MD03PS1 MD03 IFA/Lynx 084300+444000 + 4MD04PS1 MD04 COSMOS 100000+021200 + 5MD05PS1 MD05 Lockman-DXS 110000+570000 + 6MD06PS1 MD06 NGC 4258 121857+471814 + 7MD07PS1 MD07 VISTA-Video1 140000+050000 + 8MD08PS1 MD08 EliasN1-DXS 160000+570000 + 9MD09PS1 MD09 Vimos4-DXS-SSA 220000+003000 + 10MD10PS1 MD10 Deep2 233000+000000 + 11M31PS1 M31 Andromeda 004242+411600 + 12STS1PS1 STS1 Outskirts of Bulge 195048+170339 + 26STS2PS1 STS2 Hyades 004000+150000 + 27STS3PS1 STS3 Praesepe 083000+200000 + 13CAL01PS1 Cal North Pole 000000+900000 + 14CAL02PS1 Cal SA92 Dense Landolt Std Field 005808+004232 + 15CAL03PS1 Cal Perseus Cluster / Abell 426 031900+413300 + 16CAL04PS1 Cal SA95 Dense Landolt Std Field 065207+001710 + 17CAL05PS1 Cal Taurus dark cloud (diverse extinction) 043000+250000 + 18CAL06PS1 Cal SA98 Densle Landolt Std Field 065207+001710 + 19CAL07PS1 Cal M81-M82-HolmIX (Local Group) 095640+692200 + 20CAL08PS1 Cal Virgo Cluster (Center) 123000+130000 + 21CAL09PS1 Cal Hercules Cluster (Center) 160508+1745+28 + 22CAL10PS1 Cal M17-M24 Region 181400+000544 + 23CAL11PS1 Cal SA110 Dense Landolt Std Field 184207+000544 + 24CAL12PS1 Cal Kepler Mission Astrometric field 192240+443000 + 25CAL13PS1 Cal 3C454.1 (Northern field) 225033+712919 + 28SSSPS1 Solar System Sweet Spot Survey @@ -127,8 +227,16 @@ VOTable description of PSPS table CameraConfig - - - - + + camera identification number + + + configuration identification number + + + ending date of this configuration in MJD (unit = day) + + + text/table giving the details of the camera configuration + @@ -138,15 +246,37 @@ VOTable description of PSPS table PhotoCal - - - - - - - - - - - + + numerical code for this reduction + + + filter id + + + Photometry reduction code name + + + AB magnitude + + + photometric zero point (excluding long term variation) + + + photometric extinction term (excluding long term variation) + + + photometric color term (excluding long term variation) + + + photometric color dependent extinction (excluding long term variation) + + + color adopted for magnitude calculation + + + error in calibrating color + + + starting date of this configuration in MJD (unit = day) + @@ -156,8 +286,16 @@ VOTable description of PSPS table SkyCell - - - - + + indentifier of the sky cell + + + indentifier of the projection cell + + + fgetPANobjID(skyCell_center_ra, skyCell_center_dec, zHeight) + + + region identifier in the Region table + @@ -167,18 +305,46 @@ VOTable description of PSPS table ProjectionCell - - - - - - - - - - - - - - + + projection cell unique identifier + + + fGetPanObjID(crval1,crval2) + + + reference to the primary cell region definition in the Region table + + + reference to the projectio cell region definition in the Region table) + + + name of astrometric projection in RA + + + name of astrometric projection in DEC + + + RA corresponding to reference pixel (unit = deg) + + + DEC corresponding to reference pixel (unit = deg) + + + reference pixel value for RA + + + reference pixel value for DEC + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + + + elements of rotation/Dcale matrix + @@ -188,10 +354,22 @@ VOTable description of PSPS table Region - - - - - - + + Unique Region identifier + + + Custom Identifier + + + Region type + + + Comment + + + Area covered by the region + + + Binary object managed by the spatial library + @@ -201,12 +379,17 @@ VOTable description of PSPS table StackType - - - + + stack type identifier + + + stack type name + + + stack type description + 0NIGHTLY_STACKNightly stack 1DEEP_STACKDeep stack - @@ -215,7 +398,13 @@ VOTable description of PSPS table ImageFlags - - - + + Name of flag + + + Flag value + + + Description of the flag + @@ -225,7 +414,13 @@ VOTable description of PSPS table DetectionFlags - - - + + Name of flag + + + Flag value + + + Description of the flag + Index: /branches/czw_branch/20110406/ippToPsps/config/init/tables.xml =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/init/tables.xml (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/init/tables.xml (revision 31434) @@ -9,4 +9,5 @@ + @@ -56,6 +57,6 @@
+ -
@@ -110,5 +111,6 @@
- + +
Index: /branches/czw_branch/20110406/ippToPsps/config/stack/map.xml =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/stack/map.xml (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/stack/map.xml (revision 31434) @@ -9,5 +9,5 @@ - + @@ -20,5 +20,5 @@ - + @@ -44,12 +44,12 @@ - + - + - + @@ -74,5 +74,5 @@
- +
@@ -81,8 +81,8 @@ - + - + @@ -237,15 +237,15 @@ - + - - + + - - - + + + - + @@ -255,32 +255,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -297,32 +297,32 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -341,45 +341,45 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- +
Index: /branches/czw_branch/20110406/ippToPsps/config/stack/tables.vot =================================================================== --- /branches/czw_branch/20110406/ippToPsps/config/stack/tables.vot (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/config/stack/tables.vot (revision 31434) @@ -24,7 +24,4 @@ stack type identifier - - stack group identifier - saturation magnitude level @@ -33,4 +30,7 @@ analysis version index + + exposure time (unit = s) + number of P2 images contributing to this cell @@ -51,8 +51,11 @@ number of photometric reference sources - + + PSF model identifier + + mean PSF FWHM in the stack - + maximum PSF FWHM in the stack @@ -130,7 +133,4 @@ stack type identifier - - stack group id - survey flag identifier @@ -148,7 +148,4 @@ projection cell identifier - - version number of this stack - Time of mid observation (unit = day) @@ -199,5 +196,50 @@ PSF coverage factor - + + moment XX (unit = arcsec) + + + moment XY (unit = arcsec) + + + moment YY (unit = arcsec) + + + moment M3C (unit = arcsec) + + + moment M3S (unit = arcsec) + + + moment M4C (unit = arcsec) + + + moment M4S (unit = arcsec) + + + moment R1 (unit = arcsec) + + + moment RH (unit = arcsec) + + + Aperture magnitude (unit = magnitude) + + + Aperture magnitude error (unit = magnitude) + + + Kron flux (unit = magnitude) + + + Kron flux error (unit = magnitude) + + + Kron radius (unit = arcsecs) + + + Kron radius error (unit = arcsecs) + + indicator of strange propeties @@ -242,7 +284,4 @@ stack type identifier - - stack group id - survey flag identifier @@ -711,7 +750,4 @@ stack type identifier - - stack group id - survey flag identifier @@ -1154,4 +1190,54 @@ + + VOTable description of PSPS table SkinnyObject + + + ODM object identifier index + + + IPP object number + + + projection cell identifier at discovery time + + + Data release to propagate to the object + + + surveyID </coluumn> + + + + +
+ + VOTable description of PSPS table ObjectCalColor + + + ODM object identifier + + + ipp object identifier + + + filter identifier + + + color adopted for magnitude calculation (unit = mag) + + + error in calibrating color (unit = mag) + + + calibration modification number + + + Data release when this color calibration was established + + + + +
Index: /branches/czw_branch/20110406/ippToPsps/jython/batch.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/batch.py (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/batch.py (revision 31434) @@ -12,4 +12,5 @@ from datastore import Datastore +from scratchdb import ScratchDb from gpc1db import Gpc1Db from ipptopspsdb import IppToPspsDb @@ -24,6 +25,4 @@ class Batch(object): - driverName="com.mysql.jdbc.Driver" - ''' Constructor @@ -33,8 +32,9 @@ "../config/2/tables.vot" ''' - def __init__(self, logger, batchType, inputFitsPath="", survey=""): + def __init__(self, logger, batchType, inputFitsPath="", survey="", useFullTables=False): # set up logging self.logger = logger + self.logger.info("-------------------------------------------------------------------------------") self.logger.debug("Batch class constructor") @@ -44,46 +44,35 @@ self.inputFitsPath = inputFitsPath self.survey = survey + self.useFullTables = useFullTables + + # TODO + self.tablesToExport = [] # open config doc = ElementTree(file="config.xml") - # set up JDBC connection to local Db - dbName = doc.find("localdatabase/name").text - dbHost = doc.find("localdatabase/host").text - dbUser = doc.find("localdatabase/user").text - dbPass = doc.find("localdatabase/password").text - self.localUrl = "jdbc:mysql://"+dbHost+"/"+dbName+"?user="+dbUser+"&password="+dbPass - self.localCon = DriverManager.getConnection(self.localUrl) - self.localStmt = self.localCon.createStatement() - # create Gpc1Db object self.gpc1Db = Gpc1Db(self.logger) - - # get survey ID from init table - sql = "SELECT surveyID from Survey WHERE name = '" + survey + "'" - try: - rs = self.localStmt.executeQuery(sql) - rs.first() - self.surveyID = rs.getInt(1) - except: - self.logger.exception("No survey ID found for this survey: '" + survey + "'") - self.surveyID = -1; + self.ippToPspsDb = IppToPspsDb(logger) + self.scratchDb = ScratchDb(logger, self.useFullTables) + + if self.survey != "": + self.surveyID = self.scratchDb.getSurveyID(self.survey) - # get dvo info from config - if survey != "": - dvoName = doc.find(survey+"dvo/name").text - dvoLocation = doc.find(survey+"dvo/location").text + # get dvo info from config + dvoName = doc.find("dvo_" + self.survey + "/name").text + self.dvoLocation = doc.find("dvo_" + self.survey + "/location").text else: dvoName = "" - dvoLocation = "" + self.dvoLocation = "" + self.surveyID = -1; # get datastore info from config self.datastore = Datastore(self.logger) - # create IppToPspsDb object and create a new batch - self.ippToPspsDb = IppToPspsDb(logger) - self.batchID = self.ippToPspsDb.createNewBatch(66, - survey, + # create a new batch + self.batchID = self.ippToPspsDb.createNewBatch( self.getPspsBatchType(), + survey, dvoName, self.datastore.product) @@ -95,13 +84,16 @@ if not os.path.exists(self.localOutPath): os.makedirs(self.localOutPath) - # store today's date now = datetime.datetime.now(); self.dateStr = now.strftime("%Y-%m-%d") - if self.inputFitsPath != "": self.parseFitsHeader() - - # create DVO table - self.createDvoTable() + if self.inputFitsPath != "": + file = open(self.inputFitsPath) + self.header = self.parseFitsHeader(file) + self.logger.info("Read primary and found " + str(len(self.header)) + " header cards") + # TODO close file? + + # create DVO tables if accessing DVO directly + if not self.useFullTables: self.scratchDb.createDvoTables() ''' @@ -111,6 +103,44 @@ self.logger.debug("Batch destructor") - self.localStmt.close() - self.localCon.close() + + + ''' + Returns the value from this dictinary or else NULL + ''' + def safeDictionaryAccess(self, header, key): + + if key in header: return header[key] + else: return "NULL" + + ''' + Finds and reads a header extension + ''' + def findAndReadFITSHeader(self, name, file): + + found = False + + while True: + + index = file.tell() + + record = file.read(80) + if not record: break; + + # quit when we reach 'END' + if record.startswith("XTENSION= 'IMAGE"): + + header = self.parseFitsHeader(file) + if header['EXTNAME'] == name: + found = True + file.seek(index + 2880, 0) + break + + file.seek(index + 2880, 0) + + if found != True: self.logger.error("...could not find extension '" + name + "'") + else: self.logger.info("...read header at '" + name + "' and found " + str(len(header)) + " header cards") + + return header + ''' @@ -127,8 +157,13 @@ root.attrib['name'] = self.batchName root.attrib['type'] = self.getPspsBatchType() - root.attrib['survey'] = self.getPspsSurveyType() root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - root.attrib['minObjId'] = str(self.minObjID) - root.attrib['maxObjId'] = str(self.maxObjID) + if self.survey != "": + root.attrib['survey'] = self.getBatchFriendlySurveyType() + try: self.minObjID + except: pass + else: root.attrib['minObjId'] = str(self.minObjID) + try: self.maxObjID + except: pass + else: root.attrib['maxObjId'] = str(self.maxObjID) # get md5sum @@ -190,15 +225,24 @@ def publishToDatastore(self): - self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz") - # TODO update ippToPsps Db here + if self.datastore.publish(self.batchName, self.subDir, self.tarballFile, "tgz"): + self.ippToPspsDb.updateLoadedToDatastore(self.batchID, 1) ''' Gets PSPS-friendly survey type ''' - def getPspsSurveyType(self): - - if self.survey == "ThreePi": return "3PI" - elif self.survey == "MD04": return "MD04" - else: self.logger.error("Don't know this survey: " + self.survey) + def getBatchFriendlySurveyType(self): + + return "SCR" # TODO + + try: + self.survey + except: + return "NA" + + if self.survey == "3PI": return "3PI" + elif self.survey == "MD04": return "MD4" + else: + self.logger.error("Don't know this survey: '" + self.survey + "'") + return "NA" ''' @@ -212,81 +256,53 @@ else: self.logger.error("Don't know this batch type: " + self.survey) - - ''' - Prints a log message with the current time - ''' - def log(self, msg): - - print datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " | " + msg - - ''' - Sets min and max obj ID using the provided table - ''' - def setMinMaxObjID(self, table): - - sql = "SELECT MIN(objID), MAX(objID) FROM " + table - print - rs = self.localStmt.executeQuery(sql) - rs.first() - self.minObjID = rs.getLong(1) - self.maxObjID = rs.getLong(2) - - print "MIN mAX = %d %d" % (self.minObjID, self.maxObjID) - - ''' - Updates a table with surveyID - ''' - def updateSurveyID(self, table): - - sql = "UPDATE " + table + " SET surveyID=%d" % self.surveyID - self.localStmt.execute(sql) - - ''' - Updates a table with filterID grabbed from Filter init table - ''' - def updateFilterID(self, table): - - sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + self.filter + "'" - self.localStmt.execute(sql) + ''' + Sets min and max obj ID using the provided table, or list of tables + ''' + def setMinMaxObjID(self, tables): + + first = True + for table in tables: + + sql = "SELECT MIN(objID), MAX(objID) FROM " + table + rs = self.scratchDb.stmt.executeQuery(sql) + rs.first() + + if first: + self.minObjID = rs.getLong(1) + self.maxObjID = rs.getLong(2) + else: + if rs.getLong(1) < self.minObjID: self.minObjID = rs.getLong(1) + if rs.getLong(2) > self.maxObjID: self.maxObjID = rs.getLong(2) + + first = False + + self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID) ''' Reads FITS header and stores all fields in a dictionary object ''' - def parseFitsHeader(self): - - fitsFile = open(self.inputFitsPath) - - self.header = {} + def parseFitsHeader(self, fitsFile): + + header = {} while (True): + record = fitsFile.read(80) # quit when we reach 'END' - if record.startswith("END"): break - - # ignore comments - if record.startswith("COMMENT"): continue - match = re.match('(.*)=(.*)', record) + if re.match('END\s+', record): break + + # this regex will get param/value pairs for all header cards, ignoring comments and parsing out 'HIERACH' prefixes + match = re.match('^(HIERARCH )*([a-zA-Z0-9-_\.]+)\s*=\s+\'*([a-zA-Z0-9-_\.:\s@#]+)\'*\\/*', record) if match: - # remove HIERARCH prefix - param = match.group(1).replace("HIERARCH", "") - param = param.strip() - - value = match.group(2) - # remove trailing comment after / char, if there is one - index = value.find("/") - if index != -1: value = value[0:index] - - # remove ' chars around content - value = value.replace("'", "") - - # remove leading and trailing whitespace - value = value.strip() - - # store in out dictionary object - self.header[param] = value + param = match.group(2) + value = match.group(3).strip() + if value == "NaN": value = "NULL" + header[param] = value #print param + "|" + value + "|" + + return header ''' @@ -298,7 +314,8 @@ for table in self.pspsTables: self.logger.info("Creating PSPS table: " + table.name) - table.write(self.localUrl + '#' + table.name) - - self.indexPspsTables() + table.write(self.scratchDb.url + '#' + table.name) + self.tablesToExport.append(table.name) + + self.alterPspsTables(); ''' @@ -310,30 +327,17 @@ ''' - Adds an index to the supplied table and column - ''' - def createIndex(self, table, column): - - self.logger.info("Creating index on column '"+column+"' for table '"+table+"'") - - sql = "CREATE INDEX "+table+"_index ON "+table+" ("+column+")" - try: - self.localStmt.execute(sql) - except: - self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") - - - ''' - Subclass should implement this to index PSPS tables + Alter PSPS tables ''' - def indexPspsTables(self): - self.logger.warn("indexPspsTables not implemented") + def alterPspsTables(self): + self.logger.warn("alterPspsTables not implemented") ''' Imports IPP tables from FITS file - Accepts a regular expression filter so not all tabls need to be imported + Accepts a regular expression filter so not all tables need to be imported ''' def importIppTables(self, filter): + self.logger.info("Attempting to import tables from input FITS file") tables = stilts.treads(self.inputFitsPath) @@ -343,29 +347,50 @@ match = re.match(filter, table.name) if not match: continue - self.logger.info("Creating IPP table " + table.name) + self.logger.info(" Reading IPP table " + table.name + " from FITS file") table = stilts.tpipe(table, cmd='explodeall') + + # drop any previous tables before import + self.scratchDb.dropTable(table.name) + + # IPP FITS files are littered with infinities, so remove these + self.logger.info(" Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + try: - table.write(self.localUrl + '#' + table.name) + table.write(self.scratchDb.url + '#' + table.name) except: - self.logger.exception("Problem writing table '" + table.name + "' to the database") - + self.logger.exception(" Problem writing table '" + table.name + "' to the database") count = count + 1 - self.logger.info("Imported %d tables from '%s' " % (count, self.inputFitsPath)) + self.logger.info("Done. Imported %d tables" % count) self.indexIppTables() ''' - Exports PSPS tables from the database to FITS format + Exports PSPS tables from the database to FITS format. Optional regex if you want to alter table names prior to export ''' - def exportPspsTablesToFits(self): - - self.logger.info("Exporting all PSPS tables to FITS") + def exportPspsTablesToFits(self, regex="(.*)"): + + self.logger.info("Replacing NULLs with -999 then exporting all PSPS tables to FITS") _tables = [] self.logger.info(" Selecting database tables") - for table in self.pspsTables: - _table = stilts.tread(self.localUrl + '#SELECT * FROM ' + table.name) - _table = stilts.tpipe(_table, cmd='tablename ' + table.name) + for table in self.tablesToExport: + + # check for an empty table + if self.scratchDb.getRowCount(table) < 1: continue + + # get everything from table + _table = stilts.tread(self.scratchDb.url + '#SELECT * FROM ' + table) + + # replace nulls and empty fields with weird PSPS -999 pseudo-null + _table = stilts.tpipe(_table, cmd='replaceval "" -999 *') + + match = re.match(regex, table) + newTableName = match.group(1) + + # change table names + _table = stilts.tpipe(_table, cmd='tablename ' + newTableName) _tables.append(_table) @@ -373,64 +398,5 @@ stilts.twrites(_tables, self.outputFitsPath, fmt='fits') self.logger.info(" ...done") - - ''' - Returns a list of column names for this table - ''' - def getColumnNames(self, tableName): - - sql = "SHOW COLUMNS FROM " + tableName - rs = self.localStmt.executeQuery(sql) - columns = [] - while (rs.next()): columns.append(rs.getString(1)) - rs.close() - - return columns - - ''' - Replaces all NULL values in the provided table with the prvoded substitute - ''' - def replaceNulls(self, tableName, sub): - - # get list of columns - columns = self.getColumnNames(tableName) - - # now loop through all columns and replace all NULLs with sub - for column in columns: - - sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL" - self.localStmt.execute(sql) - - - ''' - Searches a table and reports the columns that are either partially or completely populated with NULLs - ''' - def reportNulls(self, tableName, showPartials): - - # first, count rows - sql = "SELECT COUNT(*) FROM " + tableName - rs = self.localStmt.executeQuery(sql) - rs.first() - numRows = rs.getInt(1) - - # get list of columns - columns = self.getColumnNames(tableName) - - print "+----------------------+---------------+" - print "| %25s |" % tableName - print "+----------------------+---------------+" - - # now see which columns are full of NULLS, with are partially NULL - for column in columns: - - sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + column + " IS NULL" - rs = self.localStmt.executeQuery(sql) - rs.first() - if rs.getInt(1) == numRows: - print "| %20s | all NULL |" % column - elif showPartials and rs.getInt(1) > 0: - print "| %20s | partial NULL |" % column - rs.close() - print "+----------------------+---------------+" - + self.ippToPspsDb.updateProcessed(self.batchID, 1) ''' @@ -440,5 +406,5 @@ for table in self.pspsTables: - self.reportNulls(table.name, showPartials) + self.scratchDb.reportNulls(table.name, showPartials) ''' @@ -449,5 +415,5 @@ self.logger.info("Replacing all NULL values in PSPS tables with '" + sub + "'...") for table in self.pspsTables: - self.replaceNulls(table.name, sub) + self.scratchDb.replaceNulls(table.name, sub) self.logger.info("...done") @@ -459,26 +425,28 @@ ''' - Updates provided table with DVO IDs from DVO table - ''' - def updateDvoIDs(self, table): - - self.logger.info("Updating table '" + table + "' with DVO IDs...") - sql = "UPDATE " + table + " AS a, dvo AS b SET \ - a.ippObjID = b.ippObjID, \ - a.objID = b.objID \ - WHERE a.ippDetectID = b.ippDetectID" - self.localStmt.execute(sql) + Calls DVO program to 'query' DVO database and populate results to local MySQL Db table + ''' + def getIDsFromDVO(self): + + # TODO path to DVO prog hardcoded temporarily + cmd = "../src/dvograbber " + self.dvoLocation + self.logger.info("Running: '" + cmd + "'...") + p = Popen(cmd, shell=True, stdout=PIPE) + p.wait() + # out = p.stdout.read() self.logger.info("...done") - - ''' - Creates a table for for ID matching - ''' - def createDvoTable(self): - - self.logger.info("Creating DVO table for ID matching") - sql = "DROP TABLE dvo" - self.localStmt.execute(sql) - sql = "CREATE TABLE dvo (ippDetectID BIGINT PRIMARY KEY, ippObjID BIGINT, objID BIGINT)" - self.localStmt.execute(sql) - + if self.scratchDb.getRowCount("dvoDetection") < 1: + self.logger.error("No DVO IDs found") + return False + + return True + + ''' + Checks whether this batch has already been processed and published. To be implemented by all subclasses + ''' + def alreadyProcessed(self): + self.logger.info("Not implemented") + + + Index: /branches/czw_branch/20110406/ippToPsps/jython/config.xml =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/config.xml (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/config.xml (revision 31434) @@ -16,4 +16,5 @@ gpc1 + ippdb01 ipp @@ -23,5 +24,5 @@ - ippToPsps + ipptopsps ippdb01 ipp @@ -31,22 +32,28 @@ - PSPS_JHU + + PSPS_test IPP_PSPS - + ThreePi.V1 /data/ipp004.0/gpc1/catdirs/ThreePi.V1 - + - - MD04.V1 - /data/ipp004.0/gpc1/catdirs/MD04.V1 - + + MD04.merge + + /data/ipp005.0/gpc1/catdirs/MD04.merges/MD04.merge + + + + + - - ./ + /data/ipp005.0/rhenders + Index: /branches/czw_branch/20110406/ippToPsps/jython/detectionbatch.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/detectionbatch.py (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/detectionbatch.py (revision 31434) @@ -1,9 +1,17 @@ #!/usr/bin/env jython +import os.path +import sys import stilts from java.lang import * from java.sql import * from batch import Batch - +from gpc1db import Gpc1Db + +import logging.config + +''' +DetectionBatch class +''' class DetectionBatch(Batch): @@ -11,61 +19,124 @@ Constructor ''' - def __init__(self): + def __init__(self, logger, camID, inputFile, test=False, useFullTables=False): super(DetectionBatch, self).__init__( + logger, "detection", - "detdemo.fits", - "detection.fits", - "localhost", - "ipptopsps_scratch", - "ipp", - "ipp", - "3PI") # TODO - - - - ''' - Updates a table with stackMetaID - ''' - def updateStackMetaID(self, table): - - sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] - self.stmt.execute(sql) - - ''' - Updates a table with stackTypeID grabbed from FitModel init table - ''' - def updateStackTypeID(self, table): - - sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" - self.stmt.execute(sql) + inputFile, + "MD04", + useFullTables) # TODO + #"3PI") # TODO + + self.logger.info("DetectionBatch constructor. Creating batch from: '" + inputFile + "'") + + meta = self.gpc1Db.getCameraStageMeta(camID) + + self.expID = meta[0]; + self.expName = meta[1]; + self.distGroup = meta[2]; + + self.logger.info("Processing exposure with ID: %d, name: %s and distribution group: %s" % (self.expID, self.expName, self.distGroup)) + + # create an output filename, which is {expID}.FITS + self.outputFitsFile = "%08d.FITS" % self.expID + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) + + # if test mode + if test: + self.startX = 3 + self.endX = 4 + self.startY = 3 + self.endY = 4 + else: + self.startX = 0 + self.endX = 8 + self.startY = 0 + self.endY = 8 + + #self.startX = 1 + #self.endX = 2 + #self.startY = 7 + #self.endY = 8 + + self.obsTime = float(self.header['MJD-OBS']) + (float(self.header['EXPTIME']) / 172800.0) + + # set up some defauts + self.calibModNum = 0 + self.dataRelease = 0 + + self.totalNumPhotoRef = 0 + + # get filterID using init table + self.filter = self.header['FILTERID'][0:1] + + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertDetectionMeta(self.batchID, self.expID, self.filter) + ''' Populates the FrameMeta table, mainly from dictionary values found in IPP FITS header ''' - def populateImageMeta(self): - self.log("Procesing ImageMeta table") - ''' - sql = "INSERT INTO ImageMeta (\ - stackMetaID \ - ,skycellID \ - ,photoZero \ - ,nP2Images \ - ,ctype1 \ - ,ctype2 \ - ,crval1 \ - ,crval2 \ - ,crpix1 \ - ,crpix2 \ - ,cdelt1 \ - ,cdelt2 \ - ,pc001001 \ - ,pc001002 \ - ,pc002001 \ - ,pc002002 \ + def populateFrameMeta(self): + self.logger.info("Procesing FrameMeta table") + + sql = "INSERT INTO FrameMeta (\ + frameID \ + ,frameName \ + ,cameraID \ + ,cameraConfigID \ + ,telescopeID \ + ,analysisVer \ + ,p1Recip \ + ,p2Recip \ + ,p3Recip \ + ,photoScat \ + ,expStart \ + ,expTime \ + ,airmass \ + ,raBore \ + ,decBore \ + ,ctype1 \ + ,ctype2 \ + ,crval1 \ + ,crval2 \ + ,crpix1 \ + ,crpix2 \ + ,cdelt1 \ + ,cdelt2 \ + ,pc001001 \ + ,pc001002 \ + ,pc002001 \ + ,pc002002 \ + ,polyOrder \ + ,pca1x3y0 \ + ,pca1x2y1 \ + ,pca1x1y2 \ + ,pca1x0y3 \ + ,pca1x2y0 \ + ,pca1x1y1 \ + ,pca1x0y2 \ + ,pca2x3y0 \ + ,pca2x2y1 \ + ,pca2x1y2 \ + ,pca2x0y3 \ + ,pca2x2y0 \ + ,pca2x1y1 \ + ,pca2x0y2 \ ) VALUES ( \ - " + self.header['STK_ID'] + " \ - ," + self.skycell + " \ - ," + self.header['FPA.ZP'] + " \ - ," + self.header['NINPUTS'] + " \ + " + str(self.expID) + " \ + ,'" + self.expName + "' \ + ,1 \ + ,1 \ + ,1 \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ," + self.header['ZPT_ERR'] + " \ + ," + self.header['MJD-OBS'] + " \ + ," + self.header['EXPREQ'] + " \ + ," + self.header['AIRMASS'] + " \ + ," + self.header['RA'] + " \ + ," + self.header['DEC'] + " \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -80,75 +151,519 @@ ," + self.header['PC002001'] + " \ ," + self.header['PC002002'] + " \ + ," + self.header['NPLYTERM'] + " \ + ," + self.header['PCA1X3Y0'] + " \ + ," + self.header['PCA1X2Y1'] + " \ + ," + self.header['PCA1X1Y2'] + " \ + ," + self.header['PCA1X0Y3'] + " \ + ," + self.header['PCA1X2Y0'] + " \ + ," + self.header['PCA1X1Y1'] + " \ + ," + self.header['PCA1X0Y2'] + " \ + ," + self.header['PCA2X3Y0'] + " \ + ," + self.header['PCA2X2Y1'] + " \ + ," + self.header['PCA2X1Y2'] + " \ + ," + self.header['PCA2X0Y3'] + " \ + ," + self.header['PCA2X2Y0'] + " \ + ," + self.header['PCA2X1Y1'] + " \ + ," + self.header['PCA2X0Y2'] + " \ )" - self.stmt.execute(sql) - - self.updateSurveyID("ImageMeta") - self.updateFilterID("ImageMeta") - self.updateStackTypeID("ImageMeta") - ''' - - ''' - Populates the Detection tables - ''' - def populateDetectionTables(self): - - self.log("Procesing Detection tables") - for x in range(3,4): - for y in range(3,4): - - self.log("Populating detections for OTA %d%d" % (x, y)) + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("FrameMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("FrameMeta", self.filter) + self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.dataRelease)) + + ''' + Populates the ImageMeta table for this OTA + ''' + def populateImageMetaTable(self, ota, header): + + tableName = "ImageMeta_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE ImageMeta" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + frameID \ + ,ccdID \ + ,sky \ + ,skyScat \ + ,magSat \ + ,completMag \ + ,astroScat \ + ,photoScat \ + ,numAstroRef \ + ,numPhotoRef \ + ,nx \ + ,ny \ + ,psfFwhm \ + ,psfWidMajor \ + ,psfWidMinor \ + ,psfTheta \ + ,momentFwhm \ + ,momentWidMajor \ + ,momentWidMinor \ + ,apResid \ + ,dapResid \ + ,detectorID \ + ,qaFlags \ + ,detrend1 \ + ,detrend2 \ + ,detrend3 \ + ,detrend4 \ + ,detrend5 \ + ,detrend6 \ + ,detrend7 \ + ,detrend8 \ + ,photoZero \ + ,ctype1 \ + ,ctype2 \ + ,crval1 \ + ,crval2 \ + ,crpix1 \ + ,crpix2 \ + ,cdelt1 \ + ,cdelt2 \ + ,pc001001 \ + ,pc001002 \ + ,pc002001 \ + ,pc002002 \ + ,polyOrder \ + ,pca1x3y0 \ + ,pca1x2y1 \ + ,pca1x1y2 \ + ,pca1x0y3 \ + ,pca1x2y0 \ + ,pca1x1y1 \ + ,pca1x0y2 \ + ,pca2x3y0 \ + ,pca2x2y1 \ + ,pca2x1y2 \ + ,pca2x0y3 \ + ,pca2x2y0 \ + ,pca2x1y1 \ + ,pca2x0y2 \ + ) VALUES ( \ + " + str(self.expID) + " \ + ," + ota[2:4] + " \ + ," + self.safeDictionaryAccess(header, 'MSKY_MN') + " \ + ," + self.safeDictionaryAccess(header, 'MSKY_SIG') + " \ + ," + self.safeDictionaryAccess(header, 'FSATUR') + " \ + ," + self.safeDictionaryAccess(header, 'FLIMIT') + " \ + ," + self.safeDictionaryAccess(header, 'CERROR') + " \ + ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ + ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ + ," + self.safeDictionaryAccess(header, 'NASTRO') + " \ + ," + self.safeDictionaryAccess(header, 'CNAXIS1') + " \ + ," + self.safeDictionaryAccess(header, 'CNAXIS2') + " \ + ," + str((float(self.safeDictionaryAccess(header, 'FWHM_MAJ')) + float(self.safeDictionaryAccess(header, 'FWHM_MIN')))/2.0) + " \ + ," + self.safeDictionaryAccess(header, 'FWHM_MAJ') + " \ + ," + self.safeDictionaryAccess(header, 'FWHM_MIN') + " \ + ," + self.safeDictionaryAccess(header, 'ANGLE') + " \ + ," + str((float(self.safeDictionaryAccess(header, 'IQ_FW1')) + float(self.safeDictionaryAccess(header, 'IQ_FW2')))/2.0) + " \ + ," + self.safeDictionaryAccess(header, 'IQ_FW1') + " \ + ," + self.safeDictionaryAccess(header, 'IQ_FW2') + " \ + ," + self.safeDictionaryAccess(header, 'APMIFIT') + " \ + ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ + ,'" + self.safeDictionaryAccess(header, 'DETECTOR') + "' \ + ," + str(self.scratchDb.getDvoImageFlags(header['SOURCEID'], header['IMAGEID'])) + " \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.MASK') + "' \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.DARK') + "' \ + ,'" + self.safeDictionaryAccess(header, 'DETREND.FLAT') + "' \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ,' ' \ + ," + self.safeDictionaryAccess(self.header, 'ZPT_OBS') + " \ + ,'" + self.safeDictionaryAccess(header, 'CTYPE1') + "' \ + ,'" + self.safeDictionaryAccess(header, 'CTYPE2') + "' \ + ," + self.safeDictionaryAccess(header, 'CRVAL1') + " \ + ," + self.safeDictionaryAccess(header, 'CRVAL2') + " \ + ," + self.safeDictionaryAccess(header, 'CRPIX1') + " \ + ," + self.safeDictionaryAccess(header, 'CRPIX2') + " \ + ," + self.safeDictionaryAccess(header, 'CDELT1') + " \ + ," + self.safeDictionaryAccess(header, 'CDELT2') + " \ + ," + self.safeDictionaryAccess(header, 'PC001001') + " \ + ," + self.safeDictionaryAccess(header, 'PC001002') + " \ + ," + self.safeDictionaryAccess(header, 'PC002001') + " \ + ," + self.safeDictionaryAccess(header, 'PC002002') + " \ + ," + self.safeDictionaryAccess(header, 'NPLYTERM') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X3Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X2Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X1Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X0Y3') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X2Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X1Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA1X0Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X3Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X2Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X1Y2') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X0Y3') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X2Y0') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X1Y1') + " \ + ," + self.safeDictionaryAccess(header, 'PCA2X0Y2') + " \ + )" + + self.scratchDb.stmt.execute(sql) + self.scratchDb.updateFilterID(tableName, self.filter) + self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) + self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") + + ''' + Populates the Detection table for this OTA + ''' + def populateDetectionTable(self, ota): + + tableName = "Detection_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE Detection" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + ippDetectID \ + ,xPos \ + ,yPos \ + ,xPosErr \ + ,yPosErr \ + ,instFlux \ + ,instFluxErr \ + ,peakADU \ + ,psfWidMajor \ + ,psfWidMinor \ + ,psfTheta \ + ,psfCf \ + ,momentXX \ + ,momentXY \ + ,momentYY \ + ,apMag \ + ,infoFlag \ + ,sky \ + ,skyErr \ + ,sgSep \ + ) \ + SELECT \ + IPP_IDET \ + ,X_PSF \ + ,Y_PSF \ + ,X_PSF_SIG \ + ,Y_PSF_SIG \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ + ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ + ,PSF_MAJOR \ + ,PSF_MINOR \ + ,PSF_THETA \ + ,PSF_QF \ + ,MOMENTS_XX \ + ,MOMENTS_XY \ + ,MOMENTS_YY \ + ,AP_MAG \ + ,FLAGS\ + ,SKY \ + ,SKY_SIGMA \ + ,EXT_NSIGMA \ + FROM " + ota + "_psf" + + self.scratchDb.stmt.execute(sql) + + # set obsTime + sql = "UPDATE " + tableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr) + self.scratchDb.stmt.execute(sql) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "historyModNum", "0") + + self.scratchDb.updateAllRows(tableName, "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID(tableName, self.filter) + + # now delete bad flux + self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "instFlux") + self.scratchDb.reportAndDeleteRowsWithNULLS(tableName, "peakADU") + + ''' + Populates the SkinnyObject table for this OTA + ''' + def populateSkinnyObjectTable(self, ota): + + tableName = "SkinnyObject_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE SkinnyObject" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,ippObjID \ + ,surveyID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + ,surveyID \ + FROM Detection_" + ota + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + + ''' + Populates the Detection table for this OTA + ''' + def populateObjectCalColorTable(self, ota): + + tableName = "ObjectCalColor_" + ota + + # drop then re-create table + self.scratchDb.dropTable(tableName) + sql = "CREATE TABLE " + tableName + " LIKE ObjectCalColor" + try: self.scratchDb.stmt.execute(sql) + except: pass + + # insert all detections into table + sql = "INSERT INTO " + tableName + " ( \ + objID \ + ,ippObjID \ + ,filterID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + ,filterID \ + FROM Detection_" + ota + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + + + ''' + Applies indexes and other constraints to the PSPS tables + ''' + def alterPspsTables(self): + + self.logger.info("Altering PSPS tables") + self.logger.info("Creating indexes on PSPS tables") + self.scratchDb.makeColumnUnique("Detection", "objID") + self.scratchDb.createIndex("Detection", "ippDetectID") + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + self.logger.info("Creating indexes on IPP tables") + + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): + + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + extension = "XY%d%d_psf" % (x, y) + self.scratchDb.createIndex(extension, "IPP_IDET") + + ''' + Updates provided table with DVO IDs from DVO table + ''' + def updateDvoIDs(self, table, sourceID, externID): + + imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID) + self.logger.info("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) + sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetection + " AS b SET \ + a.ippObjID = b.ippObjID, \ + a.detectID = b.detectID, \ + a.objID = b.objID, \ + a.infoFlag = b.flags << 32 | a.infoFlag \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.sourceID = " + str(sourceID) + " \ + AND b.imageID = " + str(imageID) + + self.scratchDb.stmt.execute(sql) + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): + + self.populateFrameMeta() + + # dictionary objects to hold sourceIDs and imageIDs for later + sourceIDs = {} + imageIDs = {} + + file = open(self.inputFitsPath, 'r') + + # loop through all OTAs and populate ImageMeta extensions + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): - sql = "INSERT INTO Detection ( \ - ippDetectID \ - ) \ - SELECT \ - IPP_IDET \ - FROM XY%d%d_psf" % (x, y) - - self.stmt.execute(sql) - - - ''' - Applies indexes to the PSPS tables - ''' - def indexPspsTables(self): - - self.log("Creating indexes on PSPS tables") -# self.createIndex("StackDetection", "ippDetectID") - # self.createIndex("StackApFlx", "ippDetectID") - # self.createIndex("StackModelFit", "ippDetectID") - - ''' - Applies indexes to the IPP tables - ''' - def indexIppTables(self): - - self.log("Creating indexes on IPP tables") - - for x in range(0,7): - for y in range(0,7): - - extension = "XY%d%d_psf" % (x, y) - self.createIndex(extension, "IPP_IDET") - - ''' - Does the processing, i.e. pulling stuff from IPP tables into PSPS tables - ''' - def populatePspsTables(self): - - # get filterID using init table - self.filter = self.header['FILTERID'] - self.filter = self.filter[0:1] - print "filter = ", self.filter - - #self.populateStackMeta() - self.populateDetectionTables() - #self.populateStackModelFit() - #self.populateStackApFlx() - + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + ota = "XY%d%d" % (x, y) + + # load corresponding header into memory + header = self.findAndReadFITSHeader(ota + ".hdr", file) + + # store sourceID/imageID combo in Db so DVO can look up later + if not self.useFullTables: + self.scratchDb.insertNewDvoImage(header['SOURCEID'], header['IMAGEID']) + + # check we have valid sourceID/imageID pair from the header + if 'SOURCEID' not in header or 'IMAGEID' not in header: + self.logger.error("Can't read SOURCEID/IMAGEID pair from " + ota + " header") + continue + + # store these for later + sourceIDs[ota] = header['SOURCEID'] + imageIDs[ota] = header['IMAGEID'] + + # populate ImageMeta + self.populateImageMetaTable(ota, header) + self.updateImageID("ImageMeta_" + ota, x, y) + + # now run DVO code to get all IDs + if not self.useFullTables: self.getIDsFromDVO() + + # loop through all OTAs again to update with DVO IDs + self.tablesToExport = [] + self.tablesToExport.append("FrameMeta") + tables = [] + otaCount = 0 + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): + + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + ota = "XY%d%d" % (x, y) + if ota not in sourceIDs: continue + + self.logger.info("******************* Dealing with OTA " + ota + " *******************") + + # populate remainder of tables + self.populateDetectionTable(ota) + + # now add DVO IDs + self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota]) + self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") + self.updateImageID("Detection_" + ota, x, y) + + # check we have something in this Detection table + if self.scratchDb.getRowCount("Detection_" + ota) < 1: + self.logger.info("Empty tables for ota " + ota + ". Skipping.") + continue; + + # update ImageMeta with count of detections for this OTA and photoCodeID + sql = "UPDATE ImageMeta_" + ota + " SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota])) + self.scratchDb.stmt.execute(sql) + + self.populateSkinnyObjectTable(ota) + self.populateObjectCalColorTable(ota) + + # add these to list of tables to export later + self.tablesToExport.append("ImageMeta_" + ota) + self.tablesToExport.append("Detection_" + ota) + self.tablesToExport.append("SkinnyObject_" + ota) + self.tablesToExport.append("ObjectCalColor_" + ota) + tables.append("Detection_" + ota) + + otaCount = otaCount + 1 + + # if we only have one table export, i.e. FrameMeta, then get out of here + if len(self.tablesToExport) == 1: + + self.logger.error("No tables to export") + return False + + self.setMinMaxObjID(tables) + + # update FrameMeta with count OTAs in this file and total number of photometric reference sources + sql = "UPDATE FrameMeta SET nOTA = %d, numPhotoRef = %d" % (otaCount, self.totalNumPhotoRef) + self.scratchDb.stmt.execute(sql) + + return True + + ''' + Updates imageID {EXP_ID}{OTA} in the provided table + ''' + def updateImageID(self, tableName, x, y): + + sql = "UPDATE " + tableName + " SET imageID = %d%d%d" % (self.expID, x, y) + self.scratchDb.stmt.execute(sql) + + ''' + Checks whether this batch has already been processed and published + ''' + def alreadyProcessed(self): + + return self.ippToPspsDb.alreadyProcessed("detection", "exp_id", self.expID) + + + ''' + Overriding this method + ''' + def reportNullsInAllPspsTables(self, showPartials): + + # loops round all imported tables, but we want to check one OTA's worth + for table in self.pspsTables: + if table.name == "FrameMeta": self.scratchDb.reportNulls(table.name, showPartials) + else: self.scratchDb.reportNulls(table.name + "_XY33", showPartials) + + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("detectionbatch") +logger.info("Starting") + +gpc1Db = Gpc1Db(logger) +camIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.V2", "cam") +logger.info("Found %d exposures" % len(camIDs)) + +i = 0 +for camID in camIDs: + + logger.info("-------------------------------------------------- cam ID: %d" % camID) + + file = gpc1Db.getCameraStageSmf(camID) + if not os.path.isfile(file): + logger.error("Cannot read file at '" + file) + continue + + detectionBatch = DetectionBatch(logger, camID, file, False, True) + + if not detectionBatch.alreadyProcessed(): + + detectionBatch.createEmptyPspsTables() + detectionBatch.importIppTables(".*.psf") + if detectionBatch.populatePspsTables(): + detectionBatch.exportPspsTablesToFits("([a-zA-Z]+)") + detectionBatch.writeBatchManifest() + #detectionBatch.reportNullsInAllPspsTables(False) + #detectionBatch.createTarball() + #detectionBatch.publishToDatastore() -detectionBatch = DetectionBatch() -detectionBatch.createEmptyPspsTables() -detectionBatch.importIppTables("XY33\.psf") -detectionBatch.populatePspsTables() -detectionBatch.exportPspsTablesToFits() + i = i+1 + # if i > 0: sys.exit() + Index: /branches/czw_branch/20110406/ippToPsps/jython/dvoToMySQL.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/dvoToMySQL.py (revision 31434) +++ /branches/czw_branch/20110406/ippToPsps/jython/dvoToMySQL.py (revision 31434) @@ -0,0 +1,209 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from gpc1db import Gpc1Db +from scratchdb import ScratchDb + +from java.lang import * +from java.sql import * +from xml.etree.ElementTree import ElementTree, Element, tostring + + +''' +Class for pulling IDs from a DVO database and shoving in a MySQL database +''' +class DvoToMySql(object): + + ''' + Constructor + + ''' + def __init__(self, logger, pathToDvo): + + # set up logging + self.logger = logger + self.pathToDvo = pathToDvo + self.logger.debug("DvoToMySql class constructor") + + self.logger.debug("Important DVO database at " + self.pathToDvo) + + # open config + doc = ElementTree(file="config.xml") + + # create database objects + self.scratchDb = ScratchDb(logger) + self.gpc1Db = Gpc1Db(self.logger) + + # create DVO tables + #self.scratchDb.createDvoTables() + + # import Images.dat table + sql = "DELETE FROM dvoMetaFull" + self.scratchDb.stmt.execute(sql) + + imagesTableName = self.importFits(self.pathToDvo, + "", + "Images.dat", + "IMAGE_ID SOURCE_ID CCDNUM EXTERN_ID FLAGS PHOTCODE NSTAR") + self.scratchDb.createIndex(imagesTableName, "IMAGE_ID") + + # insert into dvoMetaFull + self.logger.info("Inserting all image meta data into database") + sql = "INSERT INTO dvoMetaNew ( \ + sourceID, \ + imageID, \ + externID, \ + flags, \ + photcode \ + ) SELECT \ + SOURCE_ID, \ + IMAGE_ID, \ + EXTERN_ID, \ + FLAGS, \ + PHOTCODE \ + FROM " + imagesTableName + self.scratchDb.stmt.execute(sql) + + subdirs = ['n0000'] + + for subdir in subdirs: + + files = glob.glob(pathToDvo + "/" + subdir + "/*.cpm") + + #files = ['0247.06', '0244.06', '0244.10'] + + for file in files: + + # get just filename, without extension + file = os.path.basename(os.path.splitext(file)[0]) + self.logger.info("---------------------------------------------: " + file) + + if self.scratchDb.alreadyImportedThisDvoTable(file): continue + + # import cpm table and index + cpmTableName = self.importFits(self.pathToDvo, + subdir, + file + ".cpm", + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # import cpt table and index + cptTableName = self.importFits(self.pathToDvo, + subdir, + file + ".cpt", + "OBJ_ID CAT_ID EXT_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + # shove SOURCE_IDs into measurement table + self.logger.info("Adding SOURCE_IDs into measurements table") + sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (SOURCE_ID SMALLINT)" + self.scratchDb.stmt.execute(sql) + sql = "UPDATE "+cpmTableName+" AS a, "+imagesTableName+" AS b \ + SET a.SOURCE_ID = b.SOURCE_ID \ + WHERE a.IMAGE_ID = b.IMAGE_ID" + self.scratchDb.stmt.execute(sql) + + # shove PSPS objID in measurement table + self.logger.info("Adding PSPS objID into measurements table") + sql = "ALTER TABLE "+cpmTableName+" ADD COLUMN (PSPS_OBJ_ID BIGINT)" + self.scratchDb.stmt.execute(sql) + sql = "UPDATE "+cpmTableName+" AS a, "+cptTableName+" AS b \ + SET a.PSPS_OBJ_ID = b.EXT_ID \ + WHERE a.CAT_ID = b.CAT_ID \ + AND a.OBJ_ID = b.OBJ_ID" + self.scratchDb.stmt.execute(sql) + + self.logger.info("Putting everything into dvoDetectionFull table") + sql = "INSERT IGNORE INTO dvoDetectionFull (\ + sourceID \ + ,imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ) SELECT \ + SOURCE_ID \ + ,IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + FROM " + cpmTableName + self.scratchDb.stmt.execute(sql) + + # now drop what we don't need + self.logger.info("Dropping tables") + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(file) + + self.scratchDb.dropTable(imagesTableName) + + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("DvoToMySql destructor") + + ''' + Imports a FITS file. A regex filter lets you choose which tables to pull from the file + ''' + def importFits(self, path, subdir, file, columns): + + fullPath = path + "/" + subdir + "/" + file + + if len(subdir) < 1: tableName = file + else: tableName = subdir + "_" + file + + tableName = tableName.replace('.', '_') + + self.logger.info("Attempting to import tables from '" + fullPath + "' to '" + tableName + "'") + + tables = stilts.treads(fullPath) + + count = 0 + for table in tables: + + self.logger.info("Reading IPP table " + table.name + " from FITS file") + table = stilts.tpipe(table, cmd='explodeall') + + # IPP FITS files are littered with infinities, so remove these + self.logger.info("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + #try: + self.logger.info("Writing FITS table to database") + table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) + #except: + # self.logger.exception(" Problem writing table '" + table.name + "' to the database") + count = count + 1 + + self.logger.info("Done. Imported %d tables" % count) + + return tableName + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("dvotomysql") +logger.info("Starting") + +dvoToMySql = DvoToMySql(logger, "/data/ipp005.0/gpc1/catdirs/MD04.merges/MD04.merge") +#dvoToMySql = DvoToMySql(logger, "/export/ippc00.1/rhenders/MD04.merge") + +logger.info("Program complete") + Index: /branches/czw_branch/20110406/ippToPsps/jython/gpc1db.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/gpc1db.py (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/gpc1db.py (revision 31434) @@ -7,6 +7,6 @@ import logging +from mysql import MySql from java.sql import * -from xml.etree.ElementTree import ElementTree @@ -14,29 +14,11 @@ Class for GPC1 database connectivity ''' -class Gpc1Db(object): - - driverName="com.mysql.jdbc.Driver" +class Gpc1Db(MySql): ''' Constructor - ''' def __init__(self, logger): - - # setup logging - self.logger = logger - self.logger.debug("Gpc1Db constructor") - - # open config - doc = ElementTree(file="config.xml") - - # set up JDBC connection - dbName = doc.find("gpc1database/name").text - dbHost = doc.find("gpc1database/host").text - dbUser = doc.find("gpc1database/user").text - dbPass = doc.find("gpc1database/password").text - self.url = "jdbc:mysql://"+dbHost+"/"+dbName+"?user="+dbUser+"&password="+dbPass - self.con = DriverManager.getConnection(self.url) - self.stmt = self.con.createStatement() + super(Gpc1Db, self).__init__(logger,"gpc1database") ''' @@ -46,23 +28,130 @@ self.logger.debug("Gpc1Db destructor") - self.stmt.close() - self.con.close() - - - ''' - Gets all cmf files for this sky_id. handles both absolute paths and neb paths - ''' - def getStackStageCmfs(self, skyID): - - self.logger.debug("Querying for stack cmf files") - - sql = "SELECT path_base, num_inputs \ - FROM staticskyResult \ - WHERE sky_id = %d" % skyID - try: - rs = self.stmt.executeQuery(sql) - rs.first() - except: - self.logger.exception("Can't query for stack cmfs") + + ''' + Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far) + ''' + def getIDsInThisDVODbForThisStage(self, dvoDb, stage): + + sql = "SELECT DISTINCT stage_id \ + FROM addRun \ + WHERE stage = '" + stage + "' \ + AND dvodb = '" + dvoDb + "'" + + try: + rs = self.stmt.executeQuery(sql) + except: + self.logger.exception("Can't query for ids in DVO") + + ids = [] + while (rs.next()): + ids.append(rs.getInt(1)) + + rs.close() + + self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage)) + + return ids + + ''' + Gets a list of PSPS image IDs for this stack ID + ''' + def getImageIDsForThisStackID(self, stackID): + + self.logger.debug("Querying GPC1 for image IDs for stack ID: " + str(stackID)) + + sql = "SELECT DISTINCT CONCAT(exp_id, SUBSTR(class_id, 3, 4)) FROM ( \ + SELECT DISTINCT exp_id,class_id \ + FROM warpSkyCellMap \ + JOIN warpRun USING(warp_id) \ + JOIN stackInputSkyfile USING(warp_id) \ + JOIN stackRun USING(stack_id,skycell_id) \ + JOIN fakeRun USING(fake_id) \ + JOIN camRun USING(cam_id) \ + JOIN chipRun USING(chip_id) \ + WHERE stackRun.stack_id = " + str(stackID) + ") AS a" + + try: + rs = self.stmt.executeQuery(sql) + except: + self.logger.exception("Can't query for imageIDs") + + imageIDs = [] + while (rs.next()): + imageIDs.append(rs.getString(1)) + rs.close() + + return imageIDs + + ''' + Gets some stack-stage meta data for this sky_id # TODO this SQL could surely be improved + ''' + def getStackStageMeta(self, skyID, filter): + + self.logger.debug("Querying GPC1 for stack meta data") + + meta = [] + sql = "SELECT \ + stackRun.stack_id,\ + stackRun.skycell_id \ + FROM \ + staticskyInput, staticskyRun, stackRun, staticskyResult \ + WHERE staticskyRun.sky_id = staticskyInput.sky_id \ + AND staticskyInput.stack_id = stackRun.stack_id \ + AND staticskyInput.sky_id = staticskyResult.sky_id \ + and staticskyInput.sky_id = %d \ + and filter = '%s'" % (skyID, filter) + + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + meta.append(rs.getInt(1)) + meta.append(rs.getString(2)) + except: + self.logger.exception("Can't query for stack meta") + + return meta + ''' + Gets some camera-stage meta data for this cam_id + ''' + def getCameraStageMeta(self, camID): + + self.logger.debug("Querying GPC1 for camera meta data") + + meta = [] + sql = "SELECT exp_id, exp_name, camRun.dist_group \ + FROM camRun \ + JOIN chipRun USING(chip_id) \ + JOIN rawExp USING(exp_id) WHERE camRun.cam_id = %d" % camID + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + meta.append(rs.getInt(1)) + meta.append(rs.getString(2)) + meta.append(rs.getString(3)) + except: + self.logger.exception("Can't query for camera meta") + + return meta + + ''' + Gets a camera-stage smf for this cam_id + ''' + def getCameraStageSmf(self, camID): + + self.logger.debug("Querying GPC1 for camera smf files with cam_id = " + str(camID)) + + sql = "SELECT path_base \ + FROM camProcessedExp \ + JOIN camRun USING(cam_id) \ + WHERE camRun.cam_id = %d" % camID + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + except: + self.logger.exception("Can't query for camera smfs") # get path to base dir of cmf files @@ -74,5 +163,5 @@ if path.startswith("neb"): - f=os.popen("neb-ls -p "+path+"/%cmf") + f=os.popen("neb-ls -p "+path+"/%smf") for i in f.readlines(): files.append(i.rstrip()) @@ -82,4 +171,42 @@ files = glob.glob(path + "/*.cmf") + return files[0] # TODO just returning first file - check + + + ''' + Gets all cmf files for this sky_id. handles both absolute paths and neb paths + ''' + def getStackStageCmfs(self, skyID): + + self.logger.debug("Querying GPC1 for stack cmf files") + + sql = "SELECT path_base, num_inputs \ + FROM staticskyResult \ + WHERE sky_id = %d" % skyID + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + except: + self.logger.exception("Can't query for stack cmfs") + + # get path to base dir of cmf files + path = rs.getString(1) + #path = path[0:path.rfind("/")] + + # list all cmf files if a neb path + files = [] + if path.startswith("neb"): + + f=os.popen("neb-ls -p "+path+"%cmf") + print "neb-ls -p "+path+"%cmf" + for i in f.readlines(): + files.append(i.rstrip()) + print i.rstrip() + + # or not a neb path + else: + files = glob.glob(path + "*.cmf") + return files Index: /branches/czw_branch/20110406/ippToPsps/jython/initbatch.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/initbatch.py (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/initbatch.py (revision 31434) @@ -27,2 +27,3 @@ initBatch.createEmptyPspsTables() initBatch.exportPspsTablesToFits() +initBatch.writeBatchManifest() Index: /branches/czw_branch/20110406/ippToPsps/jython/ipptopspsdb.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/ipptopspsdb.py (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/ipptopspsdb.py (revision 31434) @@ -6,41 +6,38 @@ import logging +from mysql import MySql from java.sql import * -from xml.etree.ElementTree import ElementTree ''' Class for ippToPsps database connectivity ''' -class IppToPspsDb(object): - - driverName="com.mysql.jdbc.Driver" +class IppToPspsDb(MySql): ''' Constructor - ''' def __init__(self, logger): - - # setup logging - self.logger = logger - self.logger.debug("IppToPspsDb Constructor") - # open config - doc = ElementTree(file="config.xml") - - # set up JDBC connection - dbName = doc.find("ipptopspsdatabase/name").text - dbHost = doc.find("ipptopspsdatabase/host").text - dbUser = doc.find("ipptopspsdatabase/user").text - dbPass = doc.find("ipptopspsdatabase/password").text - self.url = "jdbc:mysql://"+dbHost+"/"+dbName+"?user="+dbUser+"&password="+dbPass - self.con = DriverManager.getConnection(self.url) - self.stmt = self.con.createStatement() + super(IppToPspsDb, self).__init__(logger,"ipptopspsdatabase") ''' Creates a new batch ''' - def createNewBatch(self, expID, surveyType, batchType, dvoDb, datastoreProduct): + def createNewBatch(self, batchType, survey, dvoDb, datastoreProduct): - sql = "SELECT batch_id FROM batches ORDER BY batch_id DESC LIMIT 1" + sql = "INSERT INTO batch ( \ + batch_type, \ + survey, \ + dvo_db, \ + datastore_product \ + ) VALUES ( \ + '" + batchType + "', \ + '" + survey + "', \ + '" + dvoDb + "', \ + '" + datastoreProduct + "' \ + )" + + self.stmt.execute(sql) + + sql = "SELECT MAX(batch_id) FROM batch" batchID = -1; @@ -53,27 +50,105 @@ self.logger.exception("Unable to get batch ID") - if batchID > 0: - batchID = batchID + 1 - - sql = "INSERT INTO batches \ - (batch_id, \ - exp_id, \ - survey_id, \ - batch_type, \ - dvo_db, \ - datastore_product) \ - VALUES \ - ("+str(batchID)+", \ - " + str(expID) + ", \ - '"+surveyType+"', \ - '"+batchType+"', \ - '"+dvoDb+"', \ - '"+datastoreProduct+"')" - - self.stmt.execute(sql) - - self.logger.info("Creating new batch in ippToPsps database with batchID = %d" % batchID) + self.logger.info("Created new batch in ippToPsps database with batchID = %d" % batchID) return batchID; + + ''' + Updates min/max object ID on this table and batch + ''' + def updateMinMaxObjID(self, batchID, minObjID, maxObjID): + + sql = "UPDATE batch SET \ + min_obj_id = " + str(minObjID) + ", \ + max_obj_id = " + str(maxObjID) + " \ + WHERE batch_id = " + str(batchID) + + self.stmt.execute(sql) + + ''' + Updates batch processed field + ''' + def updateProcessed(self, batchID, processed): + + sql = "UPDATE batch \ + SET processed = " + str(processed) + " \ + WHERE batch_id = " + str(batchID) + + self.stmt.execute(sql) + + ''' + Updates batch loaded_to_datastore field + ''' + def updateLoadedToDatastore(self, batchID, loadedToDatastore): + + sql = "UPDATE batch \ + SET loaded_to_datastore = " + str(loadedToDatastore) + " \ + WHERE batch_id = " + str(batchID) + + self.stmt.execute(sql) + + ''' + Have we already processed and published this batch? + ''' + def alreadyProcessed(self, table, col, value): + + sql = "SELECT COUNT(*) FROM \ + " + table + " \ + JOIN batch USING(batch_id) \ + WHERE " + col + " = " + str(value) + " \ + AND processed \ + AND loaded_to_datastore" + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.error("Batch with "+col+" = "+str(value)+" has already been processed and published to datastore") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this batch has already been processed") + + + + ''' + Inserts some detection metadata for this batch ID + ''' + def insertDetectionMeta(self, batchID, expID, filter): + + sql = "INSERT INTO detection ( \ + batch_id \ + ,exp_id \ + ,filter \ + ) VALUES ( \ + " + str(batchID) + " \ + ," + str(expID) + " \ + ,'" + filter + "' \ + )" + + self.stmt.execute(sql) + + ''' + Inserts some stack metadata for this batch ID + ''' + def insertStackMeta(self, batchID, skyID, stackID, filter, stackType): + + sql = "INSERT INTO stack ( \ + batch_id \ + ,sky_id \ + ,stack_id \ + ,filter \ + ,stack_type \ + ) VALUES ( \ + " + str(batchID) + " \ + ," + str(skyID) + " \ + ," + str(stackID) + " \ + ,'" + filter + "' \ + ,'" + stackType + "' \ + )" + + self.stmt.execute(sql) + ''' Index: /branches/czw_branch/20110406/ippToPsps/jython/mysql.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/mysql.py (revision 31434) +++ /branches/czw_branch/20110406/ippToPsps/jython/mysql.py (revision 31434) @@ -0,0 +1,200 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from java.lang import * +from java.sql import * +from xml.etree.ElementTree import ElementTree + +''' +Base class of all MySql database classes. +''' +class MySql(object): + + driverName="com.mysql.jdbc.Driver" + + ''' + Constructor + + ''' + def __init__(self, logger, dbType): + + # set up logging + self.logger = logger + self.logger.debug("MySql class constructor") + + # open config and grab database parameters + doc = ElementTree(file="config.xml") + self.dbName = doc.find(dbType +"/name").text + self.dbHost = doc.find(dbType +"/host").text + self.dbUser = doc.find(dbType +"/user").text + self.dbPass = doc.find(dbType +"/password").text + + # set up JDBC connection + self.url = "jdbc:mysql://"+self.dbHost+"/"+self.dbName+"?user="+self.dbUser+"&password="+self.dbPass + self.con = DriverManager.getConnection(self.url) + self.stmt = self.con.createStatement() + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("MySql destructor") + self.stmt.close() + self.con.close() + + ''' + Updates all rows of this column of this table with this value + ''' + def updateAllRows(self, table, column, value): + + sql = "UPDATE " + table + " SET " + column + " = " + value + self.stmt.execute(sql) + + ''' + Drops a table + ''' + def dropTable(self, table): + + sql = "DROP TABLE " + table + try: self.stmt.execute(sql) + except: return + + ''' + Alters a column to be unique + ''' + def makeColumnUnique(self, table, column): + + self.logger.debug("Making '"+column+"' unique on table '"+table+"'") + + sql = "ALTER TABLE " + table + " ADD UNIQUE (" + column + ")" + try: + self.stmt.execute(sql) + except: pass + #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") + ''' + Adds an index to the supplied table and column + ''' + def createIndex(self, table, column): + + self.logger.debug("Creating index on column '"+column+"' for table '"+table+"'") + + sql = "CREATE INDEX "+table+"_"+column+"_index ON "+table+" ("+column+")" + try: + self.stmt.execute(sql) + except: pass + #self.logger.warn("Index already in place on '" + column + "' for table '" + table + "'") + + ''' + Returns a list of column names for this table + ''' + def getColumnNames(self, tableName): + + sql = "SHOW COLUMNS FROM " + tableName + rs = self.stmt.executeQuery(sql) + columns = [] + while (rs.next()): columns.append(rs.getString(1)) + rs.close() + + return columns + + ''' + Replaces all NULL values in the provided table and column ith the provided substitute + ''' + def replaceNullsInThisColumn(self, tableName, column, sub): + + sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL" + self.stmt.execute(sql) + + ''' + Replaces all NULL values in the provided table with the provided substitute + ''' + def replaceNulls(self, tableName, sub): + + # get list of columns + columns = self.getColumnNames(tableName) + + # now loop through all columns and replace all NULLs with sub + for column in columns: + + sql = "UPDATE " + tableName + " SET " + column + " = " + sub + " WHERE " + column + " IS NULL" + self.stmt.execute(sql) + + ''' + Reports the number of rows with this columns set as this numeric value and then deletes them + ''' + def reportAndDeleteRowsWithThisValue(self, tableName, columnName, value): + + sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " = " + value + rs = self.stmt.executeQuery(sql) + rs.first() + nBadFlux = rs.getInt(1) + self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName)) + + sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value + self.stmt.execute(sql) + + ''' + Reports the number of rows with this column NULL and then deletes them + ''' + def reportAndDeleteRowsWithNULLS(self, tableName, columnName): + + sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + columnName + " IS NULL" + rs = self.stmt.executeQuery(sql) + rs.first() + nBadFlux = rs.getInt(1) + self.logger.info("%d NULL %s values in table %s. Deleting." % (nBadFlux, columnName, tableName)) + + sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL" + self.stmt.execute(sql) + + ''' + Searches a table and reports the columns that are either partially or completely populated with NULLs + ''' + def reportNulls(self, tableName, showPartials): + + # first, count rows + sql = "SELECT COUNT(*) FROM " + tableName + rs = self.stmt.executeQuery(sql) + rs.first() + numRows = rs.getInt(1) + + # get list of columns + columns = self.getColumnNames(tableName) + + print "+----------------------+---------------+" + print "| %25s |" % tableName + print "+----------------------+---------------+" + + # now see which columns are full of NULLS, with are partially NULL + for column in columns: + + sql = "SELECT COUNT(*) FROM " + tableName + " WHERE " + column + " IS NULL" + rs = self.stmt.executeQuery(sql) + rs.first() + if rs.getInt(1) == numRows: + print "| %20s | all NULL |" % column + elif showPartials and rs.getInt(1) > 0: + print "| %20s | partial NULL |" % column + rs.close() + print "+----------------------+---------------+" + + ''' + Returns a row count for this table + ''' + def getRowCount(self, table): + + sql = "SELECT COUNT(*) FROM " + table + try: + rs = self.stmt.executeQuery(sql) + rs.first() + return rs.getInt(1) + except: + self.logger.exception("Could not count rows for table: '" + table + "'") + return -1 + + Index: /branches/czw_branch/20110406/ippToPsps/jython/removeFromDatastore.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/removeFromDatastore.py (revision 31434) +++ /branches/czw_branch/20110406/ippToPsps/jython/removeFromDatastore.py (revision 31434) @@ -0,0 +1,17 @@ +#!/usr/bin/env jython + +from datastore import Datastore +import logging +import sys +import getopt + + +logging.config.fileConfig("logging.conf") +logger = logging.getLogger("datastore") + + + + +datastore = Datastore(logger) +datastore.remove(str(sys.argv[1])) + Index: /branches/czw_branch/20110406/ippToPsps/jython/run.sh =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/run.sh (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/run.sh (revision 31434) @@ -1,1 +1,1 @@ -~/jre1.6.0_24/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.2/jython.jar $1 +~/jre1.6.0_24/bin/java -Xbootclasspath/a:../jars/stilts.jar:../jars/mysql-connector-java-5.1.14-bin.jar -jar ~/jython2.5.2/jython.jar $1 $2 $3 $4 $5 $6 $7 $8 $9 Index: /branches/czw_branch/20110406/ippToPsps/jython/scratchdb.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/scratchdb.py (revision 31434) +++ /branches/czw_branch/20110406/ippToPsps/jython/scratchdb.py (revision 31434) @@ -0,0 +1,197 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from java.lang import * +from java.sql import * + +from mysql import MySql + +''' +Class for local scratch database connectivity +''' +class ScratchDb(MySql): + + ''' + Constructor + ''' + def __init__(self, logger, useFull=False): + super(ScratchDb, self).__init__(logger,"localdatabase") + + if useFull: + self.dvoMeta = "dvoMetaNew" + self.dvoDetection = "dvoDetectionFull" + else: + self.dvoMeta = "dvoMeta" + self.dvoDetection = "dvoDetection" + + self.logger.debug("ScratchDb constructor, using DVO tables: " + self.dvoMeta + " and " + self.dvoDetection) + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("ScratchDb destructor") + + ''' + Gets survey ID for this survey name + ''' + def getSurveyID(self, name): + + sql = "SELECT surveyID FROM Survey WHERE name = '" + name + "'" + try: + rs = self.stmt.executeQuery(sql) + rs.first() + return rs.getInt(1) + except: + self.logger.exception("No survey ID found for this survey: '" + self.survey + "'") + return -1 + + ''' + Gets DVO image flags + ''' + def getDvoImageFlags(self, sourceID, externID): + + flags = 0 + + sql = "SELECT flags FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + try: + rs = self.stmt.executeQuery(sql) + rs.first() + flags = rs.getInt(1) + except: + self.logger.exception("Unable to get flags from dvo table") + + return flags + + ''' + Gets imageID from extern ID + ''' + def getImageIDFromExternID(self, sourceID, externID): + + imageID = -1 + + sql = "SELECT imageID FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + try: + rs = self.stmt.executeQuery(sql) + rs.first() + imageID = rs.getInt(1) + except: + self.logger.exception("Unable to get imageID from dvo meta table") + + return imageID + + ''' + Gets photcode (aka photoCalID from dvo table) + ''' + def getPhotoCalID(self, sourceID, externID): + + photcode = -1 + + sql = "SELECT photcode FROM " + self.dvoMeta + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + try: + rs = self.stmt.executeQuery(sql) + rs.first() + photcode = rs.getInt(1) + except: + self.logger.exception("Unable to get photcode from dvo table with: " + sql) + + return photcode + + ''' + Updates a table with filterID grabbed from Filter init table + ''' + def updateFilterID(self, table, filter): + + sql = "UPDATE "+table+" AS a, Filter AS b SET a.filterID=b.filterID WHERE b.filterType = '" + filter + "'" + self.stmt.execute(sql) + + ''' + Inserts a new sourceID/imageID combo into dvoMeta + ''' + def insertNewDvoImage(self, sourceID, imageID): + + sql = "INSERT INTO dvoMeta ( \ + sourceID, \ + imageID \ + ) VALUES (\ + " + str(sourceID) + ", \ + " + str(imageID) + " \ + )" + self.stmt.execute(sql) + + ''' + Updates dvoDone table with this DVO table + ''' + def setImportedThisDvoTable(self, name): + + sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')" + self.stmt.execute(sql) + + ''' + Have we already imported this DVO table? + ''' + def alreadyImportedThisDvoTable(self, name): + + sql = "SELECT COUNT(*) FROM dvoDone WHERE name = '" + name + "'" + + try: + rs = self.stmt.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.error("DVO tables " + name + " have already been imported") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this DVO table has been imported") + + + ''' + Creates a table for for ID matching + ''' + def createDvoTables(self): + + self.logger.info("Creating DVO meta and detection tables") + + sql = "DROP TABLE dvoMeta" + try: self.stmt.execute(sql) + except: pass + + sql = "DROP TABLE dvoDetection" + try: self.stmt.execute(sql) + except: pass + + sql = "CREATE TABLE dvoMeta ( \ + sourceID INT, \ + imageID INT, \ + flags INT, \ + photcode INT, \ + PRIMARY KEY (sourceID, imageID) \ + )" + + try: self.stmt.execute(sql) + except: + self.logger.error("Unable to create DVO meta-data database table") + + sql = "CREATE TABLE dvoDetection ( \ + sourceID INT, \ + imageID INT, \ + ippDetectID BIGINT, \ + detectID BIGINT, \ + ippObjID BIGINT, \ + objID BIGINT, \ + flags INT, \ + PRIMARY KEY (sourceID, imageID, ippDetectID) \ + )" + #INDEX (sourceID), \ + #INDEX (imageID), \ + #INDEX (ippDetectID) \ + + try: self.stmt.execute(sql) + except: + self.logger.error("Unable to create DVO detection database table") Index: /branches/czw_branch/20110406/ippToPsps/jython/stackbatch.py =================================================================== --- /branches/czw_branch/20110406/ippToPsps/jython/stackbatch.py (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/jython/stackbatch.py (revision 31434) @@ -1,3 +1,6 @@ #!/usr/bin/env jython + +import os.path +import sys import stilts @@ -9,4 +12,7 @@ import logging.config +''' +StackBatch class +''' class StackBatch(Batch): @@ -14,19 +20,15 @@ Constructor ''' - def __init__(self, logger, skyID, inputFile): + def __init__(self, logger, skyID, inputFile, stackType, useFullTables=False): super(StackBatch, self).__init__( logger, "stack", inputFile, - "MD04") # TODO - - self.logger.info("StackBatch constructor. Creating batch from: ''" + inputFile + "'") - - # obs time makes no sense except for nightly stacks - if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999" - - # determine skycell from header value - self.skycell = self.header['SKYCELL'] - self.skycell = self.skycell[8:] + "MD04", + useFullTables) # TODO + + self.logger.info("StackBatch constructor. Creating batch from: '" + inputFile + "'") + + self.skyID = skyID # get filterID using init table @@ -34,21 +36,50 @@ self.filter = self.filter[0:1] + self.stackType = stackType + meta = self.gpc1Db.getStackStageMeta(self.skyID, self.header['FPA.FILTER']) + if len(meta) < 1: return + self.stackID = meta[0]; + self.skycell = meta[1]; + + # determine skycell from header value + #self.skycell = "skycell.34" #= self.header['SKYCELL'] + self.skycell = self.skycell[8:] + + self.logger.info("Processing stack with ID: %d, type: %s and skycell: %s filter: %s" % (self.stackID, self.stackType, self.skycell, self.filter)) + + + # delete PSPS tables + self.scratchDb.dropTable("StackMeta") + self.scratchDb.dropTable("StackDetection") + self.scratchDb.dropTable("StackModelFit") + self.scratchDb.dropTable("StackApFlx") + self.scratchDb.dropTable("StackToImage") + self.scratchDb.dropTable("SkinnyObject") + self.scratchDb.dropTable("ObjectCalColor") + + # delete IPP tables + #self.scratchDb.dropTable("SkyChip_psf") + #self.scratchDb.dropTable("SkyChip_xsrc") + #self.scratchDb.dropTable("SkyChip_xfit") + #self.scratchDb.dropTable("SkyChip_xrad") + + self.logger.info("Stack type: " + self.safeDictionaryAccess(self.header, self.stackType)) + # obs time makes no sense except for nightly stacks + #if self.header['STK_TYPE'] != "NIGHTLY_STACK": self.header['MJD-OBS'] = "-999" + # create an output filename, which is {filterID}{skycellID}.FITS - self.outputFitsFile = "%s%07d.FITS" % (self.filter, int(self.skycell)); - self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile); + self.outputFitsFile = "%s%07d.FITS" % (self.filter, int(self.skycell)) + self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) # set some constants self.dataRelease = "1" - self.stackVer = "1" self.historyModNum = "0" - - ''' - Updates a table with data release number - ''' - def updateDataRelease(self, table): - - sql = "UPDATE " + table + " SET dataRelease=" + self.dataRelease - self.localStmt.execute(sql) + # insert what we know about this stack batch into the stack table + self.ippToPspsDb.insertStackMeta(self.batchID, self.skyID, self.stackID, self.filter, self.stackType) + + # insert sourceID/imageID combo so DVO can look it up + if not self.useFullTables: + self.scratchDb.insertNewDvoImage(self.header['SOURCEID'], self.header['IMAGEID']) ''' @@ -57,6 +88,6 @@ def updateStackMetaID(self, table): - sql = "UPDATE " + table + " SET stackMetaID=" + self.header['STK_ID'] - self.localStmt.execute(sql) + sql = "UPDATE " + table + " SET stackMetaID=" + str(self.stackID) + self.scratchDb.stmt.execute(sql) ''' @@ -65,6 +96,6 @@ def updateStackTypeID(self, table): - sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '"+self.header['STK_TYPE']+"'" - self.localStmt.execute(sql) + sql = "UPDATE "+table+" AS a, StackType AS b SET a.stackTypeID=b.stackTypeID WHERE b.name = '" + self.stackType + "'" + self.scratchDb.stmt.execute(sql) @@ -117,5 +148,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.PSF_FWHM "+psfCondition - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) ''' @@ -165,5 +196,5 @@ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) # sersic fit has an extra parameter @@ -177,8 +208,10 @@ "+modelLong+"Covar68=b.EXT_COVAR_05_07, \ "+modelLong+"Covar78=b.EXT_COVAR_06_07, \ - "+modelLong+"Covar88=b.EXT_COVAR_07_07 \ + "+modelLong+"Covar88=b.EXT_COVAR_07_07, \ + "+"serNu=b.EXT_PAR_07, \ + "+"serNuErr=SQRT(EXT_COVAR_07_07) \ WHERE a.ippDetectID=b.IPP_IDET AND b.MODEL_TYPE = '"+ippModelType+"'" - self.localStmt.execute(sql) + self.scratchDb.stmt.execute(sql) @@ -192,6 +225,8 @@ stackMetaID \ ,skycellID \ + ,photoCalID \ ,photoZero \ - ,nP2Images \ + ,expTime \ + ,psfModelID \ ,ctype1 \ ,ctype2 \ @@ -207,8 +242,10 @@ ,pc002002 \ ) VALUES ( \ - " + self.header['STK_ID'] + " \ + " + str(self.stackID) + " \ ," + self.skycell + " \ + ," + str(self.scratchDb.getPhotoCalID(self.header['SOURCEID'], self.header['IMAGEID'])) + " \ ," + self.header['FPA.ZP'] + " \ - ," + self.header['NINPUTS'] + " \ + ," + self.header['EXPTIME'] + " \ + ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -224,10 +261,10 @@ ," + self.header['PC002002'] + " \ )" - self.localStmt.execute(sql) - - self.updateSurveyID("StackMeta") - self.updateFilterID("StackMeta") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackMeta", self.filter) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.dataRelease)) self.updateStackTypeID("StackMeta") - self.updateDataRelease("StackMeta") ''' @@ -241,5 +278,4 @@ ippDetectID \ ,skyCellID \ - ,stackVer \ ,obsTime \ ,xPos \ @@ -258,4 +294,19 @@ ,infoFlag \ ,psfCf \ + ,momentXX \ + ,momentXY \ + ,momentYY \ + ,momentM3C \ + ,momentM3S \ + ,momentM4C \ + ,momentM4S \ + ,momentR1 \ + ,momentRH \ + ,apMag \ + ,apMagErr \ + ,kronFlux \ + ,kronFluxErr \ + ,kronRad \ + ,kronRadErr \ ,nFrames \ ,assocDate \ @@ -265,5 +316,4 @@ IPP_IDET \ ," + self.skycell + " \ - ," + self.stackVer + " \ ," + self.header['MJD-OBS'] + " \ ,X_PSF \ @@ -271,6 +321,6 @@ ,X_PSF_SIG \ ,Y_PSF_SIG \ - ,PSF_INST_FLUX \ - ,PSF_INST_FLUX_SIG \ + ,POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+" \ + ,ABS((PSF_INST_MAG_SIG*(POW(10.0, (-0.4*PSF_INST_MAG)) / "+self.header['EXPTIME']+")) / 1.085736) \ ,POW(10.0, (-0.4*PEAK_FLUX_AS_MAG)) / "+self.header['EXPTIME']+" \ ,SKY \ @@ -282,16 +332,39 @@ ,FLAGS << 32 | FLAGS2 \ ,PSF_QF \ + ,MOMENTS_XX \ + ,MOMENTS_XY \ + ,MOMENTS_YY \ + ,MOMENTS_M3C \ + ,MOMENTS_M3S \ + ,MOMENTS_M4C \ + ,MOMENTS_M4S \ + ,MOMENTS_R1 \ + ,MOMENTS_RH \ + ,AP_MAG \ + , NULL \ + ,KRON_FLUX \ + ,KRON_FLUX_ERR \ + , NULL \ + , NULL \ ,N_FRAMES \ - ," + self.dateStr + " \ + , '" + self.dateStr + "' \ ," + self.historyModNum + " \ FROM SkyChip_psf" - self.localStmt.execute(sql) - - self.updateSurveyID("StackDetection") - self.updateFilterID("StackDetection") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackDetection", self.filter) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") + self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") self.updateStackMetaID("StackDetection") - self.updateDataRelease("StackDetection") self.updateStackTypeID("StackDetection") + self.updateDvoIDs("StackDetection") + + # now delete bad flux + self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") + self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "objID") + ''' @@ -306,5 +379,8 @@ DISTINCT IPP_IDET \ FROM SkyChip_xrad" - self.localStmt.execute(sql) + + try: + self.scratchDb.stmt.execute(sql) + except: return # TODO temporarily loading 1st convolved fluxes into unconvolved fields @@ -327,11 +403,14 @@ ,petR90Err=b.PETRO_RADIUS_90_ERR \ WHERE a.ippDetectID=b.IPP_IDET" - self.localStmt.execute(sql) - - self.updateSurveyID("StackApFlx") - self.updateFilterID("StackApFlx") + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackApFlx", self.filter) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") + self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") self.updateStackMetaID("StackApFlx") - self.updateDataRelease("StackApFlx") self.updateStackTypeID("StackApFlx") + self.updateDvoIDs("StackApFlx") ''' @@ -343,5 +422,9 @@ # insert all the detections sql = "INSERT INTO StackModelFit (ippDetectID) SELECT DISTINCT IPP_IDET from SkyChip_xfit" - self.localStmt.execute(sql) + try: + self.scratchDb.stmt.execute(sql) + except: + return + # populate model parameters @@ -353,9 +436,12 @@ self.updateModelFit("ser", "PS_MODEL_SERSIC") - self.updateSurveyID("StackModelFit") - self.updateFilterID("StackModelFit") + self.scratchDb.updateAllRows("StackModelFit", "surveyID", str(self.surveyID)) + self.scratchDb.updateFilterID("StackModelFit", self.filter) + self.scratchDb.updateAllRows("StackModelFit", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackModelFit", "primaryF", "0") + self.scratchDb.updateAllRows("StackModelFit", "activeFlag", "0") self.updateStackMetaID("StackModelFit") - self.updateDataRelease("StackModelFit") self.updateStackTypeID("StackModelFit") + self.updateDvoIDs("StackModelFit") ''' @@ -365,16 +451,65 @@ self.logger.info("Procesing StackToImage table") - sql = "INSERT INTO StackToImage (stackMetaID) VALUES (" + self.header['STK_ID'] + ")" - self.localStmt.execute(sql) + imageIDs = self.gpc1Db.getImageIDsForThisStackID(self.stackID) + + for imageID in imageIDs: + sql = "INSERT INTO StackToImage (stackMetaID, imageID) \ + VALUES (\ + " + str(self.stackID) + ", " + imageID + ")" + self.scratchDb.stmt.execute(sql) + + # now update StackMeta with correct number of inputs + sql = "UPDATE StackMeta SET nP2Images = (SELECT COUNT(*) FROM StackToImage)" + self.scratchDb.stmt.execute(sql) + + ''' + Populates the SkinnyObject table + ''' + def populateSkinnyObject(self): + self.logger.info("Procesing SkinnyObject table") + + sql = "INSERT INTO SkinnyObject (\ + objID \ + ,ippObjID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + FROM StackDetection" + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateAllRows("SkinnyObject", "surveyID", str(self.surveyID)) + self.scratchDb.updateAllRows("SkinnyObject", "dataRelease", str(self.dataRelease)) + + ''' + Populates the ObjectCalColor table + ''' + def populateObjectCalColor(self): + self.logger.info("Procesing ObjectCalColor table") + + sql = "INSERT INTO ObjectCalColor (\ + objID \ + ,ippObjID \ + ) \ + SELECT \ + objID \ + ,ippObjID \ + FROM StackDetection" + self.scratchDb.stmt.execute(sql) + + self.scratchDb.updateFilterID("ObjectCalColor", self.filter) + self.scratchDb.updateAllRows("ObjectCalColor", "dataRelease", str(self.dataRelease)) + ''' Applies indexes to the PSPS tables ''' - def indexPspsTables(self): - - self.logger.info("Creating indexes on PSPS tables") - self.createIndex("StackDetection", "ippDetectID") - self.createIndex("StackApFlx", "ippDetectID") - self.createIndex("StackModelFit", "ippDetectID") + def alterPspsTables(self): + + self.logger.info("Altering PSPS tables") + self.scratchDb.makeColumnUnique("StackDetection", "objID") + self.scratchDb.createIndex("StackDetection", "ippDetectID") + self.scratchDb.createIndex("StackApFlx", "ippDetectID") + self.scratchDb.createIndex("StackModelFit", "ippDetectID") ''' @@ -384,39 +519,26 @@ self.logger.info("Creating indexes on IPP tables") - self.createIndex("SkyChip_psf", "IPP_IDET") - self.createIndex("SkyChip_xfit", "IPP_IDET") - self.createIndex("SkyChip_xrad", "IPP_IDET") - self.createIndex("SkyChip_xsrc", "IPP_IDET") - - ''' - ''' - def fudgeIDs(self): - - self.logger.info("Creating bogus IDs on all detections") - - sql = "INSERT INTO dvo (ippDetectID) SELECT IPP_IDET FROM SkyChip_psf"; - self.localStmt.execute(sql) - - sql = "SELECT ippDetectID FROM dvo" - rs = self.localStmt.executeQuery(sql) - - ids = [] - while (rs.next()): - ids.append(rs.getInt(1)) - - i = 1 - o = 72010000000000001 - - for id in ids: - sql = "UPDATE dvo SET ippObjID = %d, objID = %d WHERE ippDetectID = %s" % (i, o, id) - self.localStmt.execute(sql) - i = i + 1 - o = o + 1 - - self.updateDvoIDs("StackDetection") - self.updateDvoIDs("StackApFlx") - self.updateDvoIDs("StackModelFit") - - self.setMinMaxObjID("StackDetection") + self.scratchDb.createIndex("SkyChip_psf", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xfit", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xrad", "IPP_IDET") + self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") + + ''' + Updates provided table with DVO IDs from DVO table + ''' + def updateDvoIDs(self, table): + + imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID']) + + self.logger.info("Updating table '" + table + "' with DVO IDs...") + sql = "UPDATE IGNORE " + table + " AS a, dvoDetectionFull AS b SET \ + a.ippObjID = b.ippObjID, \ + a.stackDetectID = b.detectID, \ + a.objID = b.objID \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.sourceID = " + self.header['SOURCEID'] + "\ + AND b.imageID = " + str(imageID) + self.scratchDb.stmt.execute(sql) + ''' @@ -424,4 +546,8 @@ ''' def populatePspsTables(self): + + if not self.useFullTables: + if not self.getIDsFromDVO(): + return False self.populateStackMeta() @@ -430,29 +556,59 @@ self.populateStackApFlx() self.populateStackToImage() - + self.populateSkinnyObject() + self.populateObjectCalColor() + + self.setMinMaxObjID(["StackDetection"]) + + return True + + ''' + Checks whether this batch has already been processed and published + ''' + def alreadyProcessed(self): + + return self.ippToPspsDb.alreadyProcessed("stack", "stack_id", self.stackID) logging.config.fileConfig("logging.conf") logger = logging.getLogger("stackbatch") -sky_id = 299 # TODO +logger.info("Starting") gpc1Db = Gpc1Db(logger) -cmfFiles = gpc1Db.getStackStageCmfs(sky_id) - +stackType = "NIGHTLY_STACK" +skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.Staticsky", "staticsky") +#skyIDs = gpc1Db.getIDsInThisDVODbForThisStage("MD04.GENE.PSPSDEEP", "staticsky") +#stackType = "DEEP_STACK" +#skyIDs = [689] +#skyIDs = [299] +#skyIDs = [302] +#skyIDs = [8508] i = 0 -for file in cmfFiles: - - stackBatch = StackBatch(logger, sky_id, file) - - stackBatch.createEmptyPspsTables() - stackBatch.importIppTables("") -# stackBatch.populatePspsTables() - # stackBatch.fudgeIDs() - # stackBatch.reportNullsInAllPspsTables(False) - # stackBatch.replaceAllPspsNulls("-999") - # stackBatch.exportPspsTablesToFits() - # stackBatch.writeBatchManifest() - # stackBatch.createTarball() - # stackBatch.publishToDatastore() - - i = i + 1 - if i > 0: break # TODO just doing one filter for now - +for skyID in skyIDs: + + logger.info("-------------------------------------------------- sky ID: %d" % skyID) + + cmfFiles = gpc1Db.getStackStageCmfs(skyID) + + for file in cmfFiles: + + if not os.path.isfile(file): + logger.error("Cannot read file at '" + file) + continue + + stackBatch = StackBatch(logger, skyID, file, stackType, True) + + if not stackBatch.alreadyProcessed(): + + stackBatch.createEmptyPspsTables() + stackBatch.importIppTables("") + if stackBatch.populatePspsTables(): + + #stackBatch.reportNullsInAllPspsTables(False) + stackBatch.exportPspsTablesToFits() + stackBatch.writeBatchManifest() + #stackBatch.createTarball() + #stackBatch.publishToDatastore() + + i = i + 1 + #if i > 0: sys.exit() + +logger.info("Finished") Index: /branches/czw_branch/20110406/ippToPsps/perl/pspsSchema2xml.pl =================================================================== --- /branches/czw_branch/20110406/ippToPsps/perl/pspsSchema2xml.pl (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/perl/pspsSchema2xml.pl (revision 31434) @@ -307,5 +307,5 @@ # parse line - if ($line =~ m/\s*([a-zA-Z0-9()-\[\]]+)\s+(.*)\s*--\/(.*)/) { + if ($line =~ m/\s*([a-zA-Z0-9()-_\[\]]+)\s+(.*)\s*--\/(.*)/) { $name = $1; @@ -320,5 +320,5 @@ } # no comment case - elsif ($line =~ m/\s*([a-zA-Z0-9()-\[\]]+)\s+(.*)/) { + elsif ($line =~ m/\s*([a-zA-Z0-9()-_\[\]]+)\s+(.*)/) { $name = $1; @@ -394,4 +394,5 @@ "arraysize" => $arraySize, "datatype" => $votType); + $votWriter->dataElement('DESCRIPTION', $comment); $votWriter->endTag(); Index: /branches/czw_branch/20110406/ippToPsps/src/Dvo.c =================================================================== --- /branches/czw_branch/20110406/ippToPsps/src/Dvo.c (revision 31434) +++ /branches/czw_branch/20110406/ippToPsps/src/Dvo.c (revision 31434) @@ -0,0 +1,220 @@ +/** @file Dvo.c + * + * @ingroup ippToPsps + * + * @author IfA + * Copyright 2011 Institute for Astronomy, University of Hawaii + */ + +#include "Dvo.h" + +/** + Destructor. + */ +static void destroy(Dvo* this) { + + if (this == NULL) return; + + this->logger->print(this->logger, MSG_DEBUG, "Dvo", "Destructor\n"); + + if (this->dvoConfig) dvoConfigFree(this->dvoConfig); + if (this->mysql) mysql_close(this->mysql); + + free(this); +} + +/** + Gets detections for this imageid + */ +static bool getDetections(Dvo* this, int32_t sourceId, int32_t imageId) { + + SkyList* skyList = NULL; + Image* image = NULL; + this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting skylist from DVO database...\n"); + skyList = dvoSkyListByExternID(this->dvoConfig, sourceId, imageId, &image); + if (!skyList) { + + this->logger->print(this->logger, MSG_ERROR, + "Dvo", "Could not find skylist for sourceId=%d and image ID = %d\n", + sourceId, imageId); + + return false; + } + this->logger->print(this->logger, MSG_INFO, "Dvo", "...done\n"); + + // now get detections + this->logger->print(this->logger, MSG_INFO, "Dvo", "Getting detections...\n"); + dvoDetection* dvoDetections = NULL; + int32_t maxDetectionId = -1; + int32_t numDetections = + dvoGetDetections(skyList, image->imageID, &dvoDetections, &maxDetectionId); + + this->logger->print(this->logger, MSG_INFO, "Dvo", + "Found %d detections with a max detection ID of %d\n", + numDetections, maxDetectionId); + + if (dvoDetections) dvoFree(dvoDetections); + SkyListFree(skyList); + + this->logger->print(this->logger, MSG_INFO, "Dvo", "Deleting everything from dvo table\n"); + mysql_query(this->mysql, "DELETE FROM dvo"); + + this->logger->print(this->logger, MSG_INFO, "Dvo", "Inserting IDs into dvo table...\n"); + + char sql[500]; + for (int i=0; imysql, sql); + } + this->logger->print(this->logger, MSG_INFO, "Dvo", "...done\n"); + +#if 0 + mysql_query(this->mysql, "SELECT * FROM Filter"); + MYSQL_RES* result = mysql_store_result(this->mysql); + + int num_fields = mysql_num_fields(result); + + MYSQL_ROW row; + + while ((row = mysql_fetch_row(result))) { + + for(int i = 0; i < num_fields; i++) { + + printf("%s ", row[i] ? row[i] : "NULL"); + } + printf("\n"); + } + + mysql_free_result(result); +#endif + return true; +} + +/** + Constructor. + */ +Dvo* new_Dvo(Logger* logger, const char* dvoDatabase) { + + Dvo* this = (Dvo*)calloc(1, sizeof(Dvo)); + this->logger = logger; + this->logger->print(this->logger, MSG_DEBUG, "Dvo", "Constructor\n"); + + const char *configFile = "jython/config.xml"; + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using config file here '%s'\n", configFile); + xmlDoc* doc = xmlReadFile(configFile, NULL, 0); + + // database parameters to be retrieved from config file + char host[20]; + char user[20]; + char passwd[20]; + char dbname[20]; + + + if (doc == NULL) { + + this->logger->print(this->logger, MSG_ERROR, "Dvo", "Could not load config file\n"); + } + else { + + xmlNode* root_element = xmlDocGetRootElement(doc); + + + /* We don't care what the top level element name is */ + xmlNodePtr cur = root_element->xmlChildrenNode; + char name[50]; + unsigned char* value; + while (cur != NULL) { + + // get database stuff + if (strcmp((char*)cur->name, "localdatabase") == 0) { + + xmlNodePtr dbParams = cur->xmlChildrenNode; + + while (dbParams != NULL) { + + sprintf(name, "%s", dbParams->name); + value = xmlNodeListGetString(doc, dbParams->xmlChildrenNode, 1); + if (strcmp(name, "name") == 0) sprintf(dbname, "%s", value); + else if (strcmp(name, "user") == 0) sprintf(user, "%s", value); + else if (strcmp(name, "host") == 0) sprintf(host, "%s", value); + else if (strcmp(name, "password") == 0) sprintf(passwd, "%s", value); + + dbParams = dbParams->next; + } + } + + cur = cur->next; + } + + xmlFreeDoc(doc); + } + + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using DVO database = '%s'\n", dvoDatabase); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db name = '%s'\n", dbname); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db host = '%s'\n", host); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db user = '%s'\n", user); + this->logger->print(this->logger, MSG_INFO, "Dvo", "Using db password = '%s'\n", passwd); + + int argc = 4; + char** argv; + argv = (char**)calloc(argc, sizeof(char *)); + for (int i=0; idvoConfig = dvoConfigRead(&argc, argv); + + for (int i=0; idestroy = destroy; + this->getDetections = getDetections; + + // set up MySQL connection + this->mysql = mysql_init(NULL); + + if (!this->mysql) + this->logger->print(this->logger, MSG_ERROR, "Dvo", "mysql_init(), out of memory\n"); + else { + + if (!mysql_real_connect(this->mysql, host, user, passwd, dbname, 0, NULL, 0)) + this->logger->print(this->logger, MSG_ERROR, "Dvo", "Failed to connect to database: %s\n", + mysql_error(this->mysql)); + + else + this->logger->print(this->logger, MSG_INFO, "Dvo", "Connected to %s@%s as user '%s'\n", + dbname, host, user); + + } + + return this; +} + + +/** + Main + */ +int main(int argc, char **argv) { + + Logger* logger = new_Logger(NULL, true); + + Dvo * dvo = new_Dvo(logger, "/data/ipp004.0/gpc1/catdirs/ThreePi.V1"); + dvo->getDetections(dvo, 33, 8345290); + dvo->destroy(dvo); + + logger->destroy(logger); + + return 0; +} + Index: /branches/czw_branch/20110406/ippToPsps/src/Dvo.h =================================================================== --- /branches/czw_branch/20110406/ippToPsps/src/Dvo.h (revision 31434) +++ /branches/czw_branch/20110406/ippToPsps/src/Dvo.h (revision 31434) @@ -0,0 +1,47 @@ +/** @file Dvo.h + * + * @ingroup ippToPsps + * + * @author IfA + * Copyright 2011 Institute for Astronomy, University of Hawaii + */ + +#ifndef IPPTOPSPS_DVO_H +#define IPPTOPSPS_DVO_H + +#include +#include + +#include +#include + +#include "dvo_util.h" +#include + +#include "Logger.h" + +/** + + Class encapsulating the DVO database + +*/ +typedef struct Dvo { + + // fields + Logger* logger; + dvoConfig* dvoConfig; + MYSQL* mysql; + + // methods + bool (*getDetections)(); + + // destructor + void (*destroy)(); + +} Dvo; + +// constructor +Dvo* new_Dvo(Logger* logger, const char* dvoDatabase); + +#endif // IPPTOPSPS_DVO_H + Index: /branches/czw_branch/20110406/ippToPsps/src/Makefile.am =================================================================== --- /branches/czw_branch/20110406/ippToPsps/src/Makefile.am (revision 31433) +++ /branches/czw_branch/20110406/ippToPsps/src/Makefile.am (revision 31434) @@ -21,6 +21,13 @@ # all the programs we create -bin_PROGRAMS = initbatch detectionbatch stackbatch +bin_PROGRAMS = dvo initbatch detectionbatch stackbatch + +# DVO +dvo_CFLAGS = $(IPPTOPSPS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -I/usr/include/libxml2 +dvo_LDFLAGS = $(IPPTOPSPS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) -lxml2 +dvo_SOURCES = \ + Dvo.c \ + Logger.c # initbatch Index: /branches/czw_branch/20110406/ippTools/share/Makefile.am =================================================================== --- /branches/czw_branch/20110406/ippTools/share/Makefile.am (revision 31433) +++ /branches/czw_branch/20110406/ippTools/share/Makefile.am (revision 31434) @@ -8,7 +8,16 @@ addtool_find_cam_id_dvo.sql \ addtool_find_cam_id.sql \ + addtool_find_sky_id_dvo.sql \ + addtool_find_sky_id.sql \ + addtool_find_stack_id_dvo.sql \ + addtool_find_stack_id.sql \ addtool_find_pendingexp.sql \ + addtool_find_pendingexp_cam.sql \ + addtool_find_pendingexp_stack.sql \ + addtool_find_pendingexp_staticsky.sql \ addtool_find_pendingmergeprocess.sql \ - addtool_find_processedexp.sql \ + addtool_find_processedexp_cam.sql \ + addtool_find_processedexp_stack.sql \ + addtool_find_processedexp_staticsky.sql \ addtool_find_minidvodbprocessed.sql \ addtool_find_minidvodbrun.sql \ @@ -16,7 +25,11 @@ addtool_pendingcleanuprun.sql \ addtool_queue_cam_id.sql \ + addtool_queue_stack_id.sql \ + addtool_queue_sky_id.sql \ addtool_queue_minidvodbrun.sql \ addtool_revertminidvodbprocessed.sql \ - addtool_revertprocessedexp.sql \ + addtool_revertprocessedexp_cam.sql \ + addtool_revertprocessedexp_stack.sql \ + addtool_revertprocessedexp_staticsky.sql \ bgtool_advancechip.sql \ bgtool_advancewarp.sql \ Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_cam_id.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_cam_id.sql (revision 31433) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_cam_id.sql (revision 31434) @@ -11,8 +11,9 @@ addRun.dvodb AS previous_dvodb FROM addRun - JOIN camRun USING(cam_id) + JOIN camRun on cam_id=stage_id JOIN chipRun USING(chip_id) ) as foo ON exp_id = added_exp_id + AND stage = 'cam' -- hook for qualifying the join on the previous_dvodb AND %s Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_cam_id_dvo.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_cam_id_dvo.sql (revision 31433) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_cam_id_dvo.sql (revision 31434) @@ -6,6 +6,6 @@ AND exp_id NOT IN (SELECT exp_id FROM addRun - JOIN camRun USING(cam_id) + JOIN camRun on cam_id=stage_id JOIN chipRun USING(chip_id) - WHERE %s + WHERE stage = 'cam' AND %s ) Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp.sql (revision 31433) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp.sql (revision 31434) @@ -1,20 +1,5 @@ SELECT - addRun.*, - camProcessedExp.path_base as camroot, - rawExp.exp_tag, - rawExp.exp_id, - rawExp.exp_name, - rawExp.camera, - rawExp.telescope, - rawExp.filelevel + addRun.* FROM addRun -JOIN camRun - USING(cam_id) -JOIN camProcessedExp - USING(cam_id) -JOIN chipRun - USING(chip_id) -JOIN rawExp - USING(exp_id) LEFT JOIN addProcessedExp USING(add_id) @@ -22,6 +7,5 @@ ON addRun.label = addMask.label WHERE - camRun.state = 'full' - AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') +((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') AND addRun.dvodb IS NOT NULL AND addRun.workdir IS NOT NULL Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_cam.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_cam.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_cam.sql (revision 31434) @@ -0,0 +1,30 @@ +SELECT + addRun.*, + camProcessedExp.path_base as stageroot, + rawExp.exp_tag, + rawExp.exp_id, + rawExp.exp_name, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM addRun +JOIN camRun + on cam_id = stage_id +JOIN camProcessedExp + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) +LEFT JOIN addProcessedExp + USING(add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + camRun.state = 'full' + AND addRun.stage = 'cam' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_stack.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_stack.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_stack.sql (revision 31434) @@ -0,0 +1,28 @@ +SELECT + addRun.*, + stackSumSkyfile.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN stackRun + ON stack_id = stage_id +JOIN stackSumSkyfile + USING(stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + stackRun.state = 'full' + AND stage = 'stack' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_staticsky.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_staticsky.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_pendingexp_staticsky.sql (revision 31434) @@ -0,0 +1,32 @@ +SELECT + addRun.*, + staticskyResult.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN staticskyRun + ON sky_id = stage_id +JOIN staticskyResult + USING (sky_id) +JOIN staticskyInput + USING (sky_id) +JOIN stackRun + USING (stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + staticskyRun.state = 'full' + AND stage = 'staticsky' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_cam.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_cam.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_cam.sql (revision 31434) @@ -0,0 +1,12 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) +JOIN camRun + on cam_id = stage_id +JOIN chipRun + USING(chip_id) +JOIN rawExp + ON chipRun.exp_id = rawExp.exp_id Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_stack.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_stack.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_stack.sql (revision 31434) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_staticsky.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_staticsky.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_processedexp_staticsky.sql (revision 31434) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_sky_id.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_sky_id.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_sky_id.sql (revision 31434) @@ -0,0 +1,22 @@ +SELECT + staticskyRun.* +FROM + staticskyResult +join staticSkyRun using (sky_id) +join staticSkyInput using (stack_id) +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT sky_id AS added_sky_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN staticskyRun on sky_id = stage_id + ) as foo + ON sky_id = added_sky_id + AND stage = 'sky' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + AND added_exp_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_sky_id_dvo.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_sky_id_dvo.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_sky_id_dvo.sql (revision 31434) @@ -0,0 +1,9 @@ +SELECT staticskyRun.* FROM staticskyResult +JOIN staticskyRun USING(sky_id) +WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 + AND sky_id NOT IN (SELECT sky_id + FROM addRun + JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id + JOIN staticskyRun USING(sky_id) + WHERE addRun.stage = 'staticsky' AND %s + ) Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_stack_id.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_stack_id.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_stack_id.sql (revision 31434) @@ -0,0 +1,19 @@ +SELECT + stackRun.* +FROM stackSumSkyfile +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT stack_id AS added_stack_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN stackRun on stack_id = stage_id + ) as foo + ON stack_id = added_stack_id + AND stage = 'stack' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + stackRun.state = 'full' + AND stackSumSkyfile.quality = 0 + AND added_exp_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: /branches/czw_branch/20110406/ippTools/share/addtool_find_stack_id_dvo.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_find_stack_id_dvo.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_find_stack_id_dvo.sql (revision 31434) @@ -0,0 +1,9 @@ +SELECT stackRun.* FROM stackSumSkyfile +JOIN stackRun USING(stack_id) +WHERE stackRun.state = 'full' and stackSumSkyfile.quality = 0 + AND stack_id NOT IN (SELECT stack_id + FROM addRun + JOIN stackInputSkyfile on stackInputSkyfile.stack_id = addRun.stage_id + JOIN stackRun USING(stack_id) + WHERE addRun.stage = 'stack' AND %s + ) Index: /branches/czw_branch/20110406/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_queue_cam_id.sql (revision 31433) +++ /branches/czw_branch/20110406/ippTools/share/addtool_queue_cam_id.sql (revision 31434) @@ -2,5 +2,6 @@ SELECT 0, -- add_id - cam_id, -- cam_id + 'cam', -- stage + cam_id, -- stage_id '%s', -- state '%s', -- workdir Index: /branches/czw_branch/20110406/ippTools/share/addtool_queue_sky_id.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_queue_sky_id.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_queue_sky_id.sql (revision 31434) @@ -0,0 +1,21 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'staticsky', -- stage + sky_id, -- stage_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM staticskyRun + WHERE + staticskyRun.state = 'full' + AND staticskyRun.sky_id = %lld Index: /branches/czw_branch/20110406/ippTools/share/addtool_queue_stack_id.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_queue_stack_id.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_queue_stack_id.sql (revision 31434) @@ -0,0 +1,21 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'stack', -- stage + stack_id, -- stage_id + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM stackRun + WHERE + stackRun.state = 'full' + AND stackRun.stack_id = %lld Index: /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_cam.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_cam.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_cam.sql (revision 31434) @@ -0,0 +1,10 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun, camRun, chipRun, rawExp +WHERE + addRun.add_id = addProcessedExp.add_id + AND addRun.stage_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'cam' Index: /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_stack.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_stack.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_stack.sql (revision 31434) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'stack' Index: /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_staticsky.sql =================================================================== --- /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_staticsky.sql (revision 31434) +++ /branches/czw_branch/20110406/ippTools/share/addtool_revertprocessedexp_staticsky.sql (revision 31434) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'staticsky' Index: /branches/czw_branch/20110406/ippTools/src/addtool.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/addtool.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/addtool.c (revision 31434) @@ -119,9 +119,26 @@ pxcamGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + if (strcmp(stage, "cam")== 0) { + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); // define using camRun label pxAddLabelSearchArgs (config, where, "-data_group","camRun.data_group", "=="); // define using camRun label PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - - + } + if (strcmp(stage, "stack")== 0) { + + pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","stackRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "stackRun.reduction", "=="); + } + if (strcmp(stage, "staticsky")== 0) { + + pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); + } + if (!psListLength(where->list)) { psFree(where); @@ -129,5 +146,5 @@ return false; } - + // PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); @@ -148,6 +165,8 @@ psString dvodb_string = NULL; psString bare_query = NULL; + + if (strcmp(stage,"cam") == 0 ) { if (dvodb) { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_cam_id_dvo.sql\n", dvodb); + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_cam_id_dvo.sql\n%s\n", dvodb,stage); // find the cam_id of all the exposures that we want to queue up. bare_query = pxDataGet("addtool_find_cam_id_dvo.sql"); @@ -155,5 +174,5 @@ psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); } else { - psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_cam_id.sql\n"); + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_cam_id.sql\n%s\n",stage); // find the cam_id of all the exposures that we want to queue up. bare_query = pxDataGet("addtool_find_cam_id.sql"); @@ -161,4 +180,38 @@ psStringAppend(&dvodb_string, "(camRun.dvodb IS NOT NULL AND previous_dvodb = camRun.dvodb)"); } + } + if (strcmp(stage,"stack") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_stack_id_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_stack_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_stack_id.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_stack_id.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(stackRun.dvodb IS NOT NULL AND previous_dvodb = stackRun.dvodb)"); + } + } + if (strcmp(stage,"staticsky") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); + } + } + + + if (!bare_query) { psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); @@ -184,9 +237,17 @@ if (destreaked) { + if (strcmp(stage,"cam") == 0) { psStringAppend(&query, " AND (camRun.magicked > 0)"); - } + } + if (strcmp(stage,"stack") == 0) { + psStringAppend(&query, " AND (stackRun.magicked > 0)"); + } + // staticSky has no magicked column. + } + + psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); psFree(query); return false; @@ -217,8 +278,10 @@ // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + if (strcmp(stage,"cam") == 0) { for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - camRunRow *row = camRunObjectFromMetadata(md); + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); @@ -240,4 +303,57 @@ psFree(row); } + } + if (strcmp(stage,"stack") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + stackRunRow *row = stackRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb && !row->dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, stack_id %" PRId64, row->label, row->stack_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, stack_id %" PRId64, row->label, row->stack_id); + psFree(output); + return false; + } + + psFree(row); + } + } + if (strcmp(stage,"staticsky") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb) { //there's no staticsky.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + + psFree(row); + } + } // start a transaction so we don't end up with an exp without any associted @@ -255,8 +371,12 @@ // loop over our list of camRun rows + if (strcmp(stage,"cam") == 0) { for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - - camRunRow *row = camRunObjectFromMetadata(md); + psS64 stage_id =0; + + camRunRow *row = camRunObjectFromMetadata(md); + stage_id = row->cam_id; + if (!row) { psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); @@ -267,5 +387,6 @@ // queue the exp if (!pxaddQueueByCamID(config, - row->cam_id, + stage, + stage_id, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -283,5 +404,5 @@ } psError(PS_ERR_UNKNOWN, false, - "failed to trying to queue chip_id: %" PRId64, row->cam_id); + "failed to trying to queue stage %s %" PRId64,stage, stage_id); psFree(row); psFree(output); @@ -290,4 +411,88 @@ psFree(row); } + } + if (strcmp(stage,"stack") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + stackRunRow *row = stackRunObjectFromMetadata(md); + stage_id = row->stack_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : row->dvodb, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + if (strcmp(stage,"staticsky") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + stage_id = row->sky_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + stage, + stage_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + } + psFree(output); @@ -308,4 +513,8 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); + + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); // most search arguments based on camera PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); @@ -319,6 +528,14 @@ return false; } - - psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + psString query = NULL; + if (strcmp(stage, "cam")==0) { + query = psStringCopy("UPDATE addRun JOIN camRun on cam_id = stage_id JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + } + if (strcmp(stage, "stack")==0) { + query = psStringCopy("UPDATE addRun JOIN stackRun on stack_id = stage_id"); + } + if (strcmp(stage, "staticsky")==0) { + query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); + } // pxUpdateRun gets parameters from config->args and runs the update query @@ -340,4 +557,7 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "staticskyRun.sky_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -345,6 +565,18 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("addtool_find_pendingexp.sql"); + + psString query = NULL; + + if (strcmp(stage, "cam")==0) { + query = pxDataGet("addtool_find_pendingexp_cam.sql"); + } + if (strcmp(stage, "stack")==0) { + query = pxDataGet("addtool_find_pendingexp_stack.sql"); + } + if (strcmp(stage, "staticsky")==0) { + query = pxDataGet("addtool_find_pendingexp_staticsky.sql"); + } + + if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -359,4 +591,12 @@ } psFree(where); + if (strcmp(stage, "stack") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "stack_id"); + } + if (strcmp(stage, "staticsky") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "sky_id"); + } // treat limit == 0 as "no limit" @@ -522,5 +762,6 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); - PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -537,6 +778,16 @@ return false; } - - psString query = pxDataGet("addtool_find_processedexp.sql"); + psString query = NULL; + + if (strcmp (stage,"cam") == 0) { + query = pxDataGet("addtool_find_processedexp_cam.sql"); + } + if (strcmp (stage,"stack") == 0) { + query = pxDataGet("addtool_find_processedexp_stack.sql"); + } + if (strcmp (stage,"staticsky") == 0) { + query = pxDataGet("addtool_find_processedexp_staticsky.sql"); + } + if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -567,4 +818,6 @@ psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); } + psStringAppend(&query, " AND stage = '%s'", stage); + psFree(where); @@ -616,5 +869,6 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); - PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -634,8 +888,18 @@ { - psString query = pxDataGet("addtool_revertprocessedexp.sql"); - if (!query) { + psString query = NULL; + if (strcmp(stage, "cam") == 0) { + query = pxDataGet("addtool_revertprocessedexp_cam.sql"); + } + + if (strcmp(stage, "stack") == 0) { + query = pxDataGet("addtool_revertprocessedexp_stack.sql"); + } + if (strcmp(stage, "staticsky") == 0) { + query = pxDataGet("addtool_revertprocessedexp_staticsky.sql"); + } + if (!query) { // rollback - if (!psDBRollback(config->dbh)) { + if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); } @@ -681,5 +945,6 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); - PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); Index: /branches/czw_branch/20110406/ippTools/src/addtoolConfig.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/addtoolConfig.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/addtoolConfig.c (revision 31434) @@ -49,5 +49,8 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stage", 0, "set the stage (required)", NULL); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); pxcamSetSearchArgs(definebyqueryArgs); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); @@ -72,6 +75,9 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "set the stage (required)", NULL); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); pxcamSetSearchArgs(updaterunArgs); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); @@ -89,6 +95,9 @@ // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-stage", 0, "set the stage (required)", NULL); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-sky_id", 0, "search by sky_id", 0); pxcamSetSearchArgs(pendingexpArgs); psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); @@ -109,5 +118,6 @@ psMetadata *processedexpArgs = psMetadataAlloc(); psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); - psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-stage", 0, "set the stage", NULL); pxcamSetSearchArgs(processedexpArgs); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); @@ -122,5 +132,6 @@ psMetadata *revertprocessedexpArgs = psMetadataAlloc(); psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); - psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-stage", 0, "set the stage", NULL); pxcamSetSearchArgs(revertprocessedexpArgs); psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); @@ -132,5 +143,6 @@ psMetadata *updateprocessedexpArgs = psMetadataAlloc(); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); - psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-stage", 0, "set the stage", NULL); psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); Index: /branches/czw_branch/20110406/ippTools/src/dettoolConfig.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/dettoolConfig.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/dettoolConfig.c (revision 31434) @@ -103,5 +103,6 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_begin", 0, "detrend applies to exposures taken during this period (required)", NULL); + // i'm requiring time_begin because otherwise if not specified detselect will pick the wrong detrend for old date ranges (it will pick the most recent one with NULL) psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-time_end", 0, "detrend applies to exposures taken during this period", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-use_begin", 0, "start of detrend run applicable period", NULL); Index: /branches/czw_branch/20110406/ippTools/src/flatcorr.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/flatcorr.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/flatcorr.c (revision 31434) @@ -664,4 +664,5 @@ if (!pxaddQueueByCamID( config, + "cam", row->cam_id, row->workdir, Index: /branches/czw_branch/20110406/ippTools/src/pxadd.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/pxadd.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/pxadd.c (revision 31434) @@ -37,5 +37,5 @@ if (!pxIsValidState(state)) { psError(PS_ERR_UNKNOWN, false, - "invalid camRun state: %s", state); + "invalid stageRun state: %s", state); return false; } @@ -51,5 +51,5 @@ -bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *stage, const char *state) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -59,10 +59,16 @@ if (!pxIsValidState(state)) { psError(PS_ERR_UNKNOWN, false, - "invalid chipRun state: %s", state); + "invalid stageRun state: %s", state); return false; } - - psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); - + psString query = NULL; + if (strcmp(stage, "cam") == 0) { + query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + } + if (strcmp(stage, "stack") == 0) { + ///xxx this needs to be fixed + + /// psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + } if (where) { psString whereClause = psDBGenerateWhereSQL(where, NULL); @@ -98,9 +104,9 @@ } -bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *stage, const char *label) { PS_ASSERT_PTR_NON_NULL(config, false); // note label == NULL should be explicitly allowed - + //xxx fix for stack psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); @@ -123,5 +129,6 @@ bool pxaddQueueByCamID(pxConfig *config, - psS64 cam_id, + char *stage, + psS64 stage_id, char *workdir, char *reduction, @@ -140,6 +147,16 @@ static psString query = NULL; if (!query) { + if (strcmp( stage , "cam") == 0) { query = pxDataGet("addtool_queue_cam_id.sql"); psMemSetPersistent(query, true); + } + if (strcmp(stage,"stack") == 0) { + query = pxDataGet("addtool_queue_stack_id.sql"); + psMemSetPersistent(query, true); + } + if (strcmp(stage,"staticsky") == 0) { + query = pxDataGet("addtool_queue_sky_id.sql"); + psMemSetPersistent(query, true); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -148,11 +165,12 @@ } + psTrace("addtool.c", PS_LOG_INFO, "pxadd query \n%s\n",query); // queue the exp // Note: cam_id is being cast here work around psS64 have a different type different // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, - "new", // state + "new", // state workdir ? workdir : "NULL", - "dirty", //workdir_state + "dirty", //workdir_state reduction? reduction : "NULL", label ? label : "NULL", @@ -164,5 +182,5 @@ minidvodb_group, minidvodb_name, - (long long) cam_id + (long long) stage_id )) { psError(PS_ERR_UNKNOWN, false, "database error %s", query); Index: /branches/czw_branch/20110406/ippTools/src/pxadd.h =================================================================== --- /branches/czw_branch/20110406/ippTools/src/pxadd.h (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/pxadd.h (revision 31434) @@ -26,10 +26,11 @@ bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state); -bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *stage, const char *state); bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); -bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *stage, const char *label); bool pxaddQueueByCamID(pxConfig *config, - psS64 cam_id, + char *stage, + psS64 stage_id, char *workdir, char *reduction, Index: /branches/czw_branch/20110406/ippTools/src/regtool.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/regtool.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/regtool.c (revision 31434) @@ -326,4 +326,5 @@ query = rep; + // This matches against summitExp.dateobs psStringSubstitute(&query,dateobs_begin,"@DATEOBS_BEGIN@"); psStringSubstitute(&query,dateobs_end,"@DATEOBS_END@"); Index: /branches/czw_branch/20110406/ippTools/src/staticskytool.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/staticskytool.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/staticskytool.c (revision 31434) @@ -115,4 +115,5 @@ PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); pxAddLabelSearchArgs(config, whereMD, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); @@ -279,5 +280,5 @@ } - // create a chipRun + // create a staticskyRun if (!staticskyRunInsert(config->dbh, 0x0, // sky_id Index: /branches/czw_branch/20110406/ippTools/src/staticskytoolConfig.c =================================================================== --- /branches/czw_branch/20110406/ippTools/src/staticskytoolConfig.c (revision 31433) +++ /branches/czw_branch/20110406/ippTools/src/staticskytoolConfig.c (revision 31434) @@ -52,4 +52,5 @@ psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiplei OK)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); Index: /branches/czw_branch/20110406/ippconfig/gpc1/format_20080925.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/gpc1/format_20080925.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/gpc1/format_20080925.config (revision 31434) @@ -670,3 +670,65 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END - +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 BOOL FALSE + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END + Index: /branches/czw_branch/20110406/ippconfig/gpc1/format_20080929.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/gpc1/format_20080929.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/gpc1/format_20080929.config (revision 31434) @@ -666,3 +666,65 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END - +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 BOOL FALSE + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END + Index: /branches/czw_branch/20110406/ippconfig/gpc1/format_20081011.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/gpc1/format_20081011.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/gpc1/format_20081011.config (revision 31434) @@ -656,2 +656,64 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 BOOL FALSE + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END Index: /branches/czw_branch/20110406/ippconfig/gpc1/format_20090120.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/gpc1/format_20090120.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/gpc1/format_20090120.config (revision 31434) @@ -588,2 +588,64 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 BOOL FALSE + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END Index: /branches/czw_branch/20110406/ippconfig/gpc1/format_20090220.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/gpc1/format_20090220.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/gpc1/format_20090220.config (revision 31434) @@ -590,3 +590,65 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END - +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 BOOL FALSE + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END + Index: /branches/czw_branch/20110406/ippconfig/gpc1/psastro.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/gpc1/psastro.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/gpc1/psastro.config (revision 31434) @@ -16,6 +16,6 @@ PSASTRO.GRID.SCALE F32 50 PSASTRO.GRID.NSTAR.MAX S32 800 # deprecated -PSASTRO.GRID.NREF.MAX S32 800 # max stars accepted for fitting from ref catalog -PSASTRO.GRID.NRAW.MAX S32 1200 # max stars accepted for fitting from raw image +PSASTRO.GRID.NREF.MAX S32 800 # max stars accepted for grid search from ref catalog +PSASTRO.GRID.NRAW.MAX S32 1200 # max stars accepted for grid search from raw image PSASTRO.MAX.NRAW S32 1000 # max stars accepted for fitting (0 for all) @@ -28,4 +28,6 @@ PSASTRO.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for fitting PSASTRO.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for fitting +PSASTRO.GRID.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for grid search +PSASTRO.GRID.MAX.INST.MAG.RAW F32 -8.0 # max instrumental magnitude for stars accepted for grid search PSASTRO.GRID.MIN.ANGLE F32 -1.0 # start angle (degrees) @@ -314,7 +316,16 @@ # allow more stars per chip and boost the density to which we query # from the reference database. - PSASTRO.MAX.NRAW S32 2000 # - PSASTRO.MAX.NREF S32 8000 # max stars accepted for fitting (0 for all) - DVO.GETSTAR.MAX.RHO F32 60000.0 + PSASTRO.MAX.NRAW S32 2000 # + PSASTRO.MAX.NREF S32 8000 # max stars accepted for fitting (0 for all) + # use wider range of magnitudes for grid search + PSASTRO.GRID.MIN.INST.MAG.RAW F32 -15.0 # min instrumental magnitude for stars accepted for fitting + PSASTRO.GRID.MAX.INST.MAG.RAW F32 -11.0 # max instrumental magnitude for stars accepted for fitting + + # reduce the magnitude range to avoid the bright stars have bad astrometry problem + PSASTRO.MIN.INST.MAG.RAW F32 -12.0 # min instrumental magnitude for stars accepted for fitting + PSASTRO.MAX.INST.MAG.RAW F32 -11.0 # min instrumental magnitude for stars accepted for fitting + + DVO.GETSTAR.MIN.MAG.INST F32 -15.0 + DVO.GETSTAR.MAX.RHO F32 60000.0 # the crowding is high, so unless we restrict somewhat tightly early @@ -324,5 +335,5 @@ # single-chip radius match in pixels - PSASTRO.MATCH.RADIUS.N0 F32 10 + PSASTRO.MATCH.RADIUS.N0 F32 10 PSASTRO.MATCH.RADIUS.N1 F32 5 PSASTRO.MATCH.RADIUS.N2 F32 5 Index: /branches/czw_branch/20110406/ippconfig/recipes/dqStatsTool.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/dqStatsTool.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/dqStatsTool.config (revision 31434) @@ -15,167 +15,166 @@ # echo "select zpt_obs,fwhm_major,deteff from camProcessedExp JOIN camRun USING (cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE camRun.label LIKE '%.nightlyscience' AND quality = 0 AND filter = 'FILTER';" | sed 's/FILTER/g.00000/' | mysql -h ippdb01 -u ipp -pipp gpc1 | awk '{print($3)}' | grep -v deteff | grep -v NULL | mk_cdf.pl | awk '{print($2,$1)}' | spline - `echo 0.0 0.05 0.1 0.15 0.2 0.25 0.3 0.35 0.4 0.45 0.5 0.55 0.6 0.65 0.7 0.75 0.8 0.85 0.9 0.95 0.99 1.0` # CZW: 2011-03-10 We've been off the sky too much to get much new science data. I've added the ThreePi.136 reprocessing to this to help gri - RULE METADATA - COLNAME STR DETEFF - RULETYPE STR CDF - FILTER STR g.00000 - CDF00 F32 20.199000 - CDF05 F32 21.471400 - CDF10 F32 21.725000 - CDF15 F32 21.857800 - CDF20 F32 21.918700 - CDF25 F32 21.976500 - CDF30 F32 22.020800 - CDF35 F32 22.062900 - CDF40 F32 22.104200 - CDF45 F32 22.136400 - CDF50 F32 22.173500 - CDF55 F32 22.210900 - CDF60 F32 22.243500 - CDF65 F32 22.280600 - CDF70 F32 22.317500 - CDF75 F32 22.360300 - CDF80 F32 22.405100 - CDF85 F32 22.453400 - CDF90 F32 22.508500 - CDF95 F32 22.575100 - CDF99 F32 22.715100 +# CZW: 2011-05-04 Updated values. + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR g.00000 + CDF00 F32 19.245000 + CDF05 F32 21.519900 + CDF10 F32 21.705100 + CDF15 F32 21.831400 + CDF20 F32 21.901700 + CDF25 F32 21.961500 + CDF30 F32 22.009200 + CDF35 F32 22.049700 + CDF40 F32 22.085100 + CDF45 F32 22.126400 + CDF50 F32 22.161100 + CDF55 F32 22.196100 + CDF60 F32 22.230600 + CDF65 F32 22.261400 + CDF70 F32 22.294200 + CDF75 F32 22.332200 + CDF80 F32 22.372500 + CDF85 F32 22.421000 + CDF90 F32 22.471900 + CDF95 F32 22.540200 + CDF99 F32 22.657600 CDF100 F32 22.992000 END RULE METADATA - COLNAME STR DETEFF - RULETYPE STR CDF - FILTER STR r.00000 + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR r.00000 CDF00 F32 16.424200 - CDF05 F32 20.942300 - CDF10 F32 21.219600 - CDF15 F32 21.432500 - CDF20 F32 21.605300 - CDF25 F32 21.731400 - CDF30 F32 21.803200 - CDF35 F32 21.890000 - CDF40 F32 21.954700 - CDF45 F32 22.006200 - CDF50 F32 22.052200 - CDF55 F32 22.098400 - CDF60 F32 22.132500 - CDF65 F32 22.163900 - CDF70 F32 22.198400 - CDF75 F32 22.229200 - CDF80 F32 22.267200 - CDF85 F32 22.307500 - CDF90 F32 22.356100 - CDF95 F32 22.419900 - CDF99 F32 22.555900 + CDF05 F32 21.111400 + CDF10 F32 21.317500 + CDF15 F32 21.472400 + CDF20 F32 21.615500 + CDF25 F32 21.711900 + CDF30 F32 21.789700 + CDF35 F32 21.846100 + CDF40 F32 21.898400 + CDF45 F32 21.945800 + CDF50 F32 21.989500 + CDF55 F32 22.034000 + CDF60 F32 22.075800 + CDF65 F32 22.116800 + CDF70 F32 22.152700 + CDF75 F32 22.189200 + CDF80 F32 22.230300 + CDF85 F32 22.271900 + CDF90 F32 22.329200 + CDF95 F32 22.394500 + CDF99 F32 22.532000 CDF100 F32 22.870400 END RULE METADATA - COLNAME STR DETEFF - RULETYPE STR CDF - FILTER STR i.00000 + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR i.00000 CDF00 F32 15.811400 - CDF05 F32 20.082600 - CDF10 F32 20.829900 - CDF15 F32 21.044500 - CDF20 F32 21.163700 - CDF25 F32 21.256500 - CDF30 F32 21.343700 - CDF35 F32 21.412300 - CDF40 F32 21.474900 - CDF45 F32 21.533000 - CDF50 F32 21.593000 - CDF55 F32 21.658700 - CDF60 F32 21.734000 - CDF65 F32 21.790500 - CDF70 F32 21.861200 - CDF75 F32 21.927400 - CDF80 F32 22.004800 - CDF85 F32 22.103500 - CDF90 F32 22.238100 - CDF95 F32 22.378100 - CDF99 F32 22.489700 + CDF05 F32 20.490100 + CDF10 F32 21.001400 + CDF15 F32 21.176500 + CDF20 F32 21.286300 + CDF25 F32 21.368800 + CDF30 F32 21.437400 + CDF35 F32 21.509900 + CDF40 F32 21.572000 + CDF45 F32 21.629900 + CDF50 F32 21.697600 + CDF55 F32 21.756500 + CDF60 F32 21.809000 + CDF65 F32 21.857300 + CDF70 F32 21.907200 + CDF75 F32 21.951700 + CDF80 F32 22.006600 + CDF85 F32 22.070800 + CDF90 F32 22.162300 + CDF95 F32 22.323700 + CDF99 F32 22.476700 CDF100 F32 22.892300 END -# Updated with somewhat questionable values due to low numbers of zy data. - RULE METADATA - COLNAME STR DETEFF - RULETYPE STR CDF - FILTER STR z.00000 + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR z.00000 CDF00 F32 14.759100 - CDF05 F32 17.486900 - CDF10 F32 18.923700 - CDF15 F32 19.383900 - CDF20 F32 19.760100 - CDF25 F32 19.950600 - CDF30 F32 20.097300 - CDF35 F32 20.203800 - CDF40 F32 20.286700 - CDF45 F32 20.377800 - CDF50 F32 20.455000 - CDF55 F32 20.527300 - CDF60 F32 20.621800 - CDF65 F32 20.713000 - CDF70 F32 20.787700 - CDF75 F32 20.884300 - CDF80 F32 21.027100 - CDF85 F32 21.183500 - CDF90 F32 21.275500 - CDF95 F32 21.383300 - CDF99 F32 21.577900 - CDF100 F32 21.670700 - END - RULE METADATA - COLNAME STR DETEFF - RULETYPE STR CDF - FILTER STR y.00000 + CDF05 F32 18.965400 + CDF10 F32 19.733200 + CDF15 F32 20.050200 + CDF20 F32 20.224400 + CDF25 F32 20.317200 + CDF30 F32 20.403300 + CDF35 F32 20.491200 + CDF40 F32 20.581900 + CDF45 F32 20.656000 + CDF50 F32 20.720000 + CDF55 F32 20.778800 + CDF60 F32 20.843400 + CDF65 F32 20.916000 + CDF70 F32 20.998600 + CDF75 F32 21.093300 + CDF80 F32 21.182900 + CDF85 F32 21.242400 + CDF90 F32 21.297500 + CDF95 F32 21.383500 + CDF99 F32 21.553000 + CDF100 F32 21.701800 + END + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR y.00000 CDF00 F32 14.606500 - CDF05 F32 17.436900 - CDF10 F32 18.090700 - CDF15 F32 18.350700 - CDF20 F32 18.549300 - CDF25 F32 18.727300 - CDF30 F32 18.934300 - CDF35 F32 19.140200 - CDF40 F32 19.294800 - CDF45 F32 19.400400 - CDF50 F32 19.493000 - CDF55 F32 19.564800 - CDF60 F32 19.665600 - CDF65 F32 19.744900 - CDF70 F32 19.821300 - CDF75 F32 19.900400 - CDF80 F32 20.005400 - CDF85 F32 20.087600 - CDF90 F32 20.148200 - CDF95 F32 20.220300 - CDF99 F32 20.365800 + CDF05 F32 17.858300 + CDF10 F32 18.328600 + CDF15 F32 18.584100 + CDF20 F32 18.870500 + CDF25 F32 19.100900 + CDF30 F32 19.237800 + CDF35 F32 19.339200 + CDF40 F32 19.407300 + CDF45 F32 19.473800 + CDF50 F32 19.541000 + CDF55 F32 19.608900 + CDF60 F32 19.682500 + CDF65 F32 19.743900 + CDF70 F32 19.811100 + CDF75 F32 19.899000 + CDF80 F32 20.011200 + CDF85 F32 20.089600 + CDF90 F32 20.148100 + CDF95 F32 20.221100 + CDF99 F32 20.340000 CDF100 F32 20.594900 END -# Not updated on 2011-03-28 due to insufficient w data. - RULE METADATA - COLNAME STR DETEFF - RULETYPE STR CDF - FILTER STR w.00000 - CDF00 F32 17.4082 - CDF05 F32 19.4333 - CDF10 F32 19.9698 - CDF15 F32 20.2711 - CDF20 F32 20.3919 - CDF25 F32 20.5038 - CDF30 F32 20.6192 - CDF35 F32 20.7273 - CDF40 F32 20.8184 - CDF45 F32 20.9186 - CDF50 F32 21.0151 - CDF55 F32 21.1326 - CDF60 F32 21.2442 - CDF65 F32 21.3412 - CDF70 F32 21.4394 - CDF75 F32 21.5105 - CDF80 F32 21.5567 - CDF85 F32 21.6281 - CDF90 F32 21.6908 - CDF95 F32 21.7566 - CDF99 F32 21.8907 - CDF100 F32 23.6312 + RULE METADATA + COLNAME STR DETEFF + RULETYPE STR CDF + FILTER STR w.00000 + CDF00 F32 21.986000 + CDF05 F32 21.990000 + CDF10 F32 22.056500 + CDF15 F32 22.097000 + CDF20 F32 22.145000 + CDF25 F32 22.186500 + CDF30 F32 22.209500 + CDF35 F32 22.286500 + CDF40 F32 22.339800 + CDF45 F32 22.385200 + CDF50 F32 22.413200 + CDF55 F32 22.506700 + CDF60 F32 22.554200 + CDF65 F32 22.587100 + CDF70 F32 22.594200 + CDF75 F32 22.616800 + CDF80 F32 22.688200 + CDF85 F32 22.723000 + CDF90 F32 22.765800 + CDF95 F32 22.891600 + CDF99 F32 22.964500 + CDF100 F32 22.977000 END RULE METADATA Index: /branches/czw_branch/20110406/ippconfig/recipes/filerules-mef.mdc =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/filerules-mef.mdc (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/filerules-mef.mdc (revision 31434) @@ -225,4 +225,8 @@ PSPHOT.PSF.RAW.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF PSPHOT.PSF.SKY.SAVE OUTPUT {OUTPUT}.psf PSF NONE CHIP TRUE MEF + +# used by staticsky with single input +PSPHOT.SKY.CONFIG OUTPUT {OUTPUT}.psphot.mdc TEXT NONE FPA TRUE NONE + # outputs for psphotStack: Index: /branches/czw_branch/20110406/ippconfig/recipes/filerules-simple.mdc =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/filerules-simple.mdc (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/filerules-simple.mdc (revision 31434) @@ -193,4 +193,7 @@ PSPHOT.PSF.RAW.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE PSPHOT.PSF.SKY.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE + +# used by staticsky with single input +PSPHOT.SKY.CONFIG OUTPUT {OUTPUT}.psphot.mdc TEXT NONE FPA TRUE NONE # outputs for psphotStack: Index: /branches/czw_branch/20110406/ippconfig/recipes/filerules-split.mdc =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/filerules-split.mdc (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/filerules-split.mdc (revision 31434) @@ -211,4 +211,7 @@ PSPHOT.PSF.RAW.SAVE OUTPUT {OUTPUT}.{CHIP.NAME}.psf PSF NONE CHIP TRUE MEF PSPHOT.PSF.SKY.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE MEF + +# used by staticsky with single input +PSPHOT.SKY.CONFIG OUTPUT {OUTPUT}.psphot.mdc TEXT NONE FPA TRUE NONE # outputs for psphotStack: Index: /branches/czw_branch/20110406/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/nightly_science.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/nightly_science.config (revision 31434) @@ -22,5 +22,5 @@ CLEAN_MODES METADATA MODE STR MAGICDS - COMMAND STR magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ + COMMAND STR magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME S16 2 END @@ -203,9 +203,13 @@ NAME STR STS DISTRIBUTION STR STS - TESS STR STS.V3 + # XXX: STS observations are temporarily being taken outside the STS tessellation + # switch tess_id from STS.V3 to RINGS + TESS STR RINGS.V3 + # TESS STR STS.V3 OBSMODE STR STS% OBJECT STR STS% STACKABLE BOOL FALSE DIFFABLE BOOL FALSE + REDUCTION STR STS_DATASET END TARGETS METADATA Index: /branches/czw_branch/20110406/ippconfig/recipes/ppStack.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/ppStack.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/ppStack.config (revision 31434) @@ -97,4 +97,6 @@ OUTPUT.REPLICATE BOOL TRUE +STACK.TYPE STR DEEP_STACK + # Recipe overrides for STACK STACK METADATA @@ -136,4 +138,5 @@ STACK_NIGHTLY METADATA + STACK.TYPE STR NIGHTLY_STACK END Index: /branches/czw_branch/20110406/ippconfig/recipes/psastro.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/psastro.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/psastro.config (revision 31434) @@ -83,4 +83,6 @@ PSASTRO.MIN.INST.MAG.RAW F32 0.0 # min instrumental magnitude for stars accepted for fitting PSASTRO.MAX.INST.MAG.RAW F32 0.0 # max instrumental magnitude for stars accepted for fitting +PSASTRO.GRID.MIN.INST.MAG.RAW F32 0.0 # min instrumental magnitude for stars accepted for grid search +PSASTRO.GRID.MAX.INST.MAG.RAW F32 0.0 # max instrumental magnitude for stars accepted for grid search PSASTRO.MATCH.LUMFUNC BOOL FALSE Index: /branches/czw_branch/20110406/ippconfig/recipes/psphot.config =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/psphot.config (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/psphot.config (revision 31434) @@ -377,4 +377,9 @@ END +STACKPHOT_SINGLE METADATA + PSPHOT.STACK.USE.RAW BOOL F # since false perform photometry & morphology analysis on the convolved images + SAVE.BACKMDL BOOL F +END + # Recipe overrides for CHIP CHIP METADATA Index: /branches/czw_branch/20110406/ippconfig/recipes/reductionClasses.mdc =================================================================== --- /branches/czw_branch/20110406/ippconfig/recipes/reductionClasses.mdc (revision 31433) +++ /branches/czw_branch/20110406/ippconfig/recipes/reductionClasses.mdc (revision 31434) @@ -160,4 +160,5 @@ STACKPHOT_PPSUB STR STACKPHOT STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE BACKGROUND_PPBACKGROUND STR BACKGROUND BACKGROUND_PSWARP STR BACKGROUND @@ -362,4 +363,5 @@ STACKPHOT_PPSUB STR STACKPHOT STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE END Index: /branches/czw_branch/20110406/ppStack/src/ppStackArguments.c =================================================================== --- /branches/czw_branch/20110406/ppStack/src/ppStackArguments.c (revision 31433) +++ /branches/czw_branch/20110406/ppStack/src/ppStackArguments.c (revision 31434) @@ -204,5 +204,5 @@ } psArgumentRemove (argNum, &argc, argv); - if (strcasecmp(argv[argNum], "NIGHTLY") && strcasecmp(argv[argNum], "DEEP") && strcasecmp(argv[argNum], "BEST_IQ")) { + if (strcasecmp(argv[argNum], "NIGHTLY_STACK") && strcasecmp(argv[argNum], "DEEP_STACK") && strcasecmp(argv[argNum], "IQ_STACK")) { psErrorStackPrint(stderr, "Invalid option for -stack-type %s (must be one of NIGHTLY_STACK, DEEP_STACK, IQ_STACK)", argv[argNum]); exit(PS_EXIT_CONFIG_ERROR); Index: /branches/czw_branch/20110406/ppStack/src/ppStackLoop.c =================================================================== --- /branches/czw_branch/20110406/ppStack/src/ppStackLoop.c (revision 31433) +++ /branches/czw_branch/20110406/ppStack/src/ppStackLoop.c (revision 31434) @@ -134,4 +134,12 @@ ppStackMemDump("final"); + // Update Header + // Do this before performing photometry so that the cmf header gets all of the information. + if (!ppStackUpdateHeader(stack, options, config)) { + psError(psErrorCodeLast(), false, "Unable to update header."); + psFree(stack); + return false; + } + // Photometry psTrace("ppStack", 1, "Photometering stacked image....\n"); @@ -143,10 +151,4 @@ ppStackMemDump("photometry"); - // Update Header - if (!ppStackUpdateHeader(stack, options, config)) { - psError(psErrorCodeLast(), false, "Unable to update header."); - psFree(stack); - return false; - } // Create JPEGS if (!ppStackJPEGs(stack, options, config)) { Index: /branches/czw_branch/20110406/psModules/src/extras/pmVisualUtils.c =================================================================== --- /branches/czw_branch/20110406/psModules/src/extras/pmVisualUtils.c (revision 31433) +++ /branches/czw_branch/20110406/psModules/src/extras/pmVisualUtils.c (revision 31434) @@ -416,10 +416,11 @@ return; } else { + int nwritten = 0; if (comp->level == PM_DEFAULT_VISUAL_LEVEL) { sprintf(line,"%*s%-*s %d\n", depth, "", 20 - depth, comp->name, defLevel); - fwrite (line, 1, strlen(line), output); + nwritten = fwrite (line, 1, strlen(line), output); } else { sprintf(line, "%*s%-*s %d\n", depth, "", 20 - depth, comp->name, comp->level); - fwrite (line, 1, strlen(line), output); + nwritten = fwrite (line, 1, strlen(line), output); } } Index: /branches/czw_branch/20110406/psModules/src/objects/pmFootprintAssignPeaks.c =================================================================== --- /branches/czw_branch/20110406/psModules/src/objects/pmFootprintAssignPeaks.c (revision 31433) +++ /branches/czw_branch/20110406/psModules/src/objects/pmFootprintAssignPeaks.c (revision 31434) @@ -35,10 +35,8 @@ assert (peaks->n == 0 || psMemCheckPeak(peaks->data[0])); - if (footprints->n == 0) { - if (peaks->n > 0) { - return psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Your list of footprints is empty"); - } + if ((footprints->n == 0) && (peaks->n == 0)) { return PS_ERR_NONE; } + /* * Create an image filled with the object IDs, and use it to assign pmPeaks to the @@ -59,6 +57,5 @@ if (ids) { assert (x >= 0 && x < numCols && y >= 0 && y < numRows);} - int id = ids ? ids->data.S32[y][x - col0] : 0; - // XXX I think the '[x - col0]' above is just wrong (should be [x], but never gets triggerd. + int id = ids ? ids->data.S32[y][x] : 0; if (id == 0) { // peak isn't in a footprint, so make one for it Index: /branches/czw_branch/20110406/psModules/src/objects/pmFootprintCullPeaks.c =================================================================== --- /branches/czw_branch/20110406/psModules/src/objects/pmFootprintCullPeaks.c (revision 31433) +++ /branches/czw_branch/20110406/psModules/src/objects/pmFootprintCullPeaks.c (revision 31434) @@ -102,5 +102,6 @@ float beta = nsigma_delta * ALPHA; float beta2 = PS_SQR(beta); - int nBins = sqrt(4.0*(maxFlux - min_threshold)/beta2) + 10; // let's be extra generous + float value = 4.0*(maxFlux - min_threshold)/beta2; + int nBins = (value > 1.0) ? sqrt(value) + 10 : 10; // let's be extra generous // create a vector to store the threshold bins used for each peak Index: /branches/czw_branch/20110406/psModules/src/objects/pmFootprintIDs.c =================================================================== --- /branches/czw_branch/20110406/psModules/src/objects/pmFootprintIDs.c (revision 31433) +++ /branches/czw_branch/20110406/psModules/src/objects/pmFootprintIDs.c (revision 31434) @@ -66,6 +66,4 @@ if (footprints->n == 0) { - // XXX this was an error, but is an allowed condition -- NULL returned image means no footprints for any peaks - // psError(PS_ERR_BAD_PARAMETER_SIZE, true, "You didn't provide any footprints"); return NULL; } Index: /branches/czw_branch/20110406/psastro/src/psastroAstromGuess.c =================================================================== --- /branches/czw_branch/20110406/psastro/src/psastroAstromGuess.c (revision 31433) +++ /branches/czw_branch/20110406/psastro/src/psastroAstromGuess.c (revision 31434) @@ -161,4 +161,27 @@ psastroPlotRawstars (rawstars, fpa, chip, recipe); } + + // Next if we are using a different set of stars for grid search fill out their pmAstromObjs + psArray *grid_rawstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.GRID.RAWSTARS"); + if (grid_rawstars == NULL || grid_rawstars == rawstars) { continue; } + + for (int i = 0; i < grid_rawstars->n; i++) { + pmAstromObj *raw = grid_rawstars->data[i]; + + psPlaneTransformApply (raw->FP, chip->toFPA, raw->chip); + psPlaneTransformApply (raw->TP, fpa->toTPA, raw->FP); + psDeproject (raw->sky, raw->TP, fpa->toSky); + + // rationalize ra to sky range centered on boresite + while (raw->sky->r < RAminSky) raw->sky->r += 2.0*M_PI; + while (raw->sky->r > RAmaxSky) raw->sky->r -= 2.0*M_PI; + + RAmin = PS_MIN (raw->sky->r, RAmin); + RAmax = PS_MAX (raw->sky->r, RAmax); + + DECmin = PS_MIN (raw->sky->d, DECmin); + DECmax = PS_MAX (raw->sky->d, DECmax); + } + // XXX: should we plot grid_rawstars? } } Index: /branches/czw_branch/20110406/psastro/src/psastroChipAstrom.c =================================================================== --- /branches/czw_branch/20110406/psastro/src/psastroChipAstrom.c (revision 31433) +++ /branches/czw_branch/20110406/psastro/src/psastroChipAstrom.c (revision 31434) @@ -60,4 +60,16 @@ // select the raw objects for this readout + psArray *gridrawstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.GRID.RAWSTARS.SUBSET"); + if (gridrawstars == NULL) { + gridrawstars = rawstars; + } else { + // the absolute minimum number of stars is 4 (for order = 1) + if (gridrawstars->n < 4) { + readout->data_exists = false; + psLogMsg ("psastro", 3, "insufficient gird rawstars (%ld)", gridrawstars->n); + continue; + } + } + psArray *refstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.REFSTARS.SUBSET"); if (refstars == NULL) { continue; } @@ -80,5 +92,5 @@ // XXX update the header with info to reflect the failure - if (!psastroOneChipGrid (fpa, chip, refstars, rawstars, recipe, updates)) { + if (!psastroOneChipGrid (fpa, chip, refstars, gridrawstars, recipe, updates)) { readout->data_exists = false; psLogMsg ("psastro", 3, "failed to find a solution\n"); Index: /branches/czw_branch/20110406/psastro/src/psastroConvert.c =================================================================== --- /branches/czw_branch/20110406/psastro/src/psastroConvert.c (revision 31433) +++ /branches/czw_branch/20110406/psastro/src/psastroConvert.c (revision 31434) @@ -14,4 +14,6 @@ // leak free 2006.04.27 +static psArray * chooseStars(psArray *inStars, char *listName, psArray *sources, psVector *index, int nMax, float iMagMin, float iMagMax, pmSourceMode skip); + bool psastroConvertFPA (pmFPA *fpa, psMetadata *recipe) { @@ -87,7 +89,36 @@ } + psArray *rawStars = chooseStars(inStars, "", sources, index, PS_MIN(nMax, inStars->n), iMagMin, iMagMax, skip); + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.RAWSTARS", PS_DATA_ARRAY, "astrometry objects", rawStars); + + bool gridSearch = psMetadataLookupBool (&status, recipe, "PSASTRO.GRID.SEARCH"); + if (gridSearch) { + // See if different magnitude limits have been specified for grid search. If so, create a separate list of stars to use. + float iGridMagMax = psMetadataLookupF32 (&status, recipe, "PSASTRO.GRID.MAX.INST.MAG.RAW"); + float iGridMagMin = psMetadataLookupF32 (&status, recipe, "PSASTRO.GRID.MIN.INST.MAG.RAW"); + int nMaxGrid = psMetadataLookupS32 (&status, recipe, "PSASTRO.GRID.NRAW.MAX"); + + // XXX Should we check PSASTRO.GRID.NRAW.MAX != PSASTRO.MAX.NRAW as well? It usually is smaller so that would cause + // us to always create a separate list. So I won't check. + + if ((iGridMagMax != iMagMax) || (iGridMagMin != iMagMin)) { + psArray *gridStars = chooseStars(inStars, "grid search ", sources, index, PS_MIN(nMaxGrid, inStars->n), iGridMagMin, iGridMagMax, skip); + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.GRID.RAWSTARS", PS_DATA_ARRAY, "astrometry objects for grid search", gridStars); + psFree(gridStars); + } + } + + psFree (index); + psFree (inStars); + psFree (rawStars); + + return true; +} + + +psArray * chooseStars(psArray *inStars, char *listName, psArray *sources, psVector *index, int nMax, float iMagMin, float iMagMax, pmSourceMode skip) { // choose the first nMax sources int j = 0; - psArray *rawStars = psArrayAlloc (PS_MIN (nMax, inStars->n)); + psArray *rawStars = psArrayAlloc(nMax); float mMin = +100.0; @@ -128,14 +159,8 @@ rawStars->n = j; - psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.RAWSTARS", PS_DATA_ARRAY, "astrometry objects", rawStars); - - psLogMsg ("psastro", 4, "loaded %ld sources, using %ld of %ld good sources (inst mag: %f to %f)\n", sources->n, rawStars->n, inStars->n, mMin, mMax); + psLogMsg ("psastro", 4, "loaded %ld %ssources, using %ld of %ld good sources (inst mag: %f to %f)\n", sources->n, listName, rawStars->n, inStars->n, mMin, mMax); psLogMsg ("psastro", 4, "skip reasons: mode: %d, faint: %d, bright: %d, inf: %d\n", nModeSkip, nFaintSkip, nBrightSkip, nInfSkip); - psFree (index); - psFree (inStars); - psFree (rawStars); - - return true; + return rawStars; } Index: /branches/czw_branch/20110406/psastro/src/psastroRemoveClumps.c =================================================================== --- /branches/czw_branch/20110406/psastro/src/psastroRemoveClumps.c (revision 31433) +++ /branches/czw_branch/20110406/psastro/src/psastroRemoveClumps.c (revision 31434) @@ -61,4 +61,11 @@ psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.RAWSTARS.SUBSET", PS_DATA_ARRAY, "astrometry objects", subset); psFree (subset); + + psArray *gridstars = psMetadataLookupPtr(&status, readout->analysis, "PSASTRO.GRID.RAWSTARS"); + if ((gridstars == rawstars) || (gridstars == NULL)) { continue; } + + psArray *gridstars_subset = psastroRemoveClumpsIterate(gridstars, 150, 3); + psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.GRID.RAWSTARS.SUBSET", PS_DATA_ARRAY, "astrometry objects", gridstars_subset); + psFree (gridstars_subset); } } Index: /branches/czw_branch/20110406/tools/czartool/MetricsIndex.pm =================================================================== --- /branches/czw_branch/20110406/tools/czartool/MetricsIndex.pm (revision 31433) +++ /branches/czw_branch/20110406/tools/czartool/MetricsIndex.pm (revision 31434) @@ -53,4 +53,6 @@ $self->createHtml("IPP Metrics"); my $htmlFile = $self->{htmlFile}; + + print $htmlFile "\"master\n"; opendir(DIR, $self->{path}) or die $!; Index: /branches/czw_branch/20110406/tools/czartool/Nebulous.pm =================================================================== --- /branches/czw_branch/20110406/tools/czartool/Nebulous.pm (revision 31433) +++ /branches/czw_branch/20110406/tools/czartool/Nebulous.pm (revision 31434) @@ -110,5 +110,6 @@ next; } - elsif ($line =~ m/(ipp[0-9]+)\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i) { + elsif (($line =~ m/(ipp[0-9]+)\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)|| + ($line =~ m/(ippb[0-9]+)\s.+[0-9]+\s+([0-9]+)\s+([0-9]+)\s+([0-9]+)%.*/i)) { $self->{_totalHosts}++; Index: /branches/czw_branch/20110406/tools/mysql-dump/gpc1Import.py =================================================================== --- /branches/czw_branch/20110406/tools/mysql-dump/gpc1Import.py (revision 31433) +++ /branches/czw_branch/20110406/tools/mysql-dump/gpc1Import.py (revision 31434) @@ -15,5 +15,5 @@ gpc1dumps=commands.getoutput('ls -t -1 /export/ipp001.0/ipp/mysql-dumps/mysql-gpc1*') dumparray=gpc1dumps.split()[1] -cmd='gunzip < '+dumparray+' | mysql -hipp001 -u ipp -pipp -f ' + newname +cmd='bunzip2 < '+dumparray+' | mysql -hipp001 -u ipp -pipp -f ' + newname results = commands.getstatusoutput(cmd) Index: /branches/czw_branch/20110406/tools/mysql-dump/gpc1_dump.sh =================================================================== --- /branches/czw_branch/20110406/tools/mysql-dump/gpc1_dump.sh (revision 31433) +++ /branches/czw_branch/20110406/tools/mysql-dump/gpc1_dump.sh (revision 31434) @@ -8,5 +8,5 @@ # - it deletes the old MySQL dumps # - it deletes the old MD5 checksum file -# 2) It calls mysqldump to dump gpc1 to some gzipped file +# 2) It calls mysqldump to dump gpc1 to some bzipped file # 3) It computes the checksum # 4) It copies the dump and the checksum files onto the TARGET_HOST @@ -16,5 +16,5 @@ . /home/panstarrs/ipp/mysql-dump/password.sh -EMAILTO=schastel@ifa.hawaii.edu +EMAILTO=ps-ipp-ops@ifa.hawaii.edu HOST=ippc02 @@ -28,5 +28,5 @@ #1) Cleaning log INFO "Deleting old dumps" -for file in `find $DUMP_PATH -name mysql-gpc1-$HOST-\*.dump.gz`; do +for file in `find $DUMP_PATH -name mysql-gpc1-$HOST-\*.dump.bz`; do log INFO "Deleting $file" /bin/rm -f $file @@ -37,9 +37,9 @@ #2) Dumping DATEVAR=`date +%FT%T` #date -FILENAME=$DUMP_PATH/mysql-gpc1-$HOST-$DATEVAR.dump.gz #dump file name +FILENAME=$DUMP_PATH/mysql-gpc1-$HOST-$DATEVAR.dump.bz #dump file name DB_USERNAME=gpc1_dump MD5FILENAME=$DUMP_PATH/gpc1_checksum.md5 log INFO "Dumping to $FILENAME" -/usr/bin/mysqldump -h localhost -u $DB_USERNAME -p$DB_PASSWORD --flush-logs --single-transaction gpc1 | gzip -9 > $FILENAME +/usr/bin/mysqldump -h localhost -u $DB_USERNAME -p$DB_PASSWORD --flush-logs --single-transaction gpc1 | ~ipp/local/bin/pbzip2 > $FILENAME dump_status=${PIPESTATUS[0]} if [ "$dump_status" -ne "0" ]; then Index: /branches/czw_branch/20110406/tools/mysql-dump/gpc1_install.sh =================================================================== --- /branches/czw_branch/20110406/tools/mysql-dump/gpc1_install.sh (revision 31433) +++ /branches/czw_branch/20110406/tools/mysql-dump/gpc1_install.sh (revision 31434) @@ -10,10 +10,10 @@ # 2) Check the gpc1 dump file against its checksum. # 3) Distribute the validated files, that is: -# - Hard-links the gzipped MySQL dump in the distribution directory +# - Hard-links the bzipped MySQL dump in the distribution directory # - Moves the MD5 checksum to the distribution directory # 4) Import gpc1 into gpc1_0 or gpc1_1 (ensured by gpc1Import.py utility) . /home/panstarrs/ipp/mysql-dump/functions.sh -EMAILTO=schastel@ifa.hawaii.edu +EMAILTO=ps-ipp-ops@ifa.hawaii.edu HOST=ipp001 @@ -21,5 +21,5 @@ MD5FILE=gpc1_checksum.md5 -DISTRIBUTION_TARGET=$TARGET/distribution/ippdb01-gpc1.dump.gz +DISTRIBUTION_TARGET=$TARGET/distribution/ippdb01-gpc1.dump.bz DISTRIBUTION_MD5=$TARGET/distribution/ippdb01-gpc1.md5 Index: /branches/czw_branch/20110406/tools/mysql-dump/neb_copy.sh =================================================================== --- /branches/czw_branch/20110406/tools/mysql-dump/neb_copy.sh (revision 31433) +++ /branches/czw_branch/20110406/tools/mysql-dump/neb_copy.sh (revision 31434) @@ -11,5 +11,6 @@ . /home/panstarrs/ipp/mysql-dump/functions.sh -EMAILTO=schastel@ifa.hawaii.edu + +EMAILTO=ps-ipp-ops@ifa.hawaii.edu SOURCE_HOST=ippdb02 Index: /branches/czw_branch/20110406/tools/mysql-dump/neb_dump.sh =================================================================== --- /branches/czw_branch/20110406/tools/mysql-dump/neb_dump.sh (revision 31433) +++ /branches/czw_branch/20110406/tools/mysql-dump/neb_dump.sh (revision 31434) @@ -12,5 +12,6 @@ . /home/panstarrs/ipp/mysql-dump/password.sh -EMAILTO=schastel@ifa.hawaii.edu +EMAILTO=ps-ipp-ops@ifa.hawaii.edu + HOST=ippdb02 @@ -20,10 +21,10 @@ log INFO "Deleting old dumps" -for file in `find $DUMP_PATH -name mysql-neb-$HOST-\*.dump.gz`; do +for file in `find $DUMP_PATH -name mysql-neb-$HOST-\*.dump.bz`; do log INFO "Deleting $file" /bin/rm -f $file done DATEVAR=`date +%FT%T` #date -FILENAME=$DUMP_PATH/mysql-neb-$HOST-$DATEVAR.dump.gz #dump file name +FILENAME=$DUMP_PATH/mysql-neb-$HOST-$DATEVAR.dump.bz #dump file name DB_USERNAME=neb_dump MD5FILENAME=$DUMP_PATH/neb_checksum.md5 @@ -33,5 +34,5 @@ log INFO "Dumping to [$FILENAME]" -/usr/bin/mysqldump -h localhost -u $DB_USERNAME -p$DB_PASSWORD --flush-logs --single-transaction nebulous | gzip -9 > $FILENAME +/usr/bin/mysqldump -h localhost -u $DB_USERNAME -p$DB_PASSWORD --flush-logs --single-transaction nebulous | ~ipp/local/bin/pbzip2 > $FILENAME dump_status=${PIPESTATUS[0]} if [ "$dump_status" -ne "0" ]; then Index: /branches/czw_branch/20110406/tools/mysql-dump/ops_dump.csh =================================================================== --- /branches/czw_branch/20110406/tools/mysql-dump/ops_dump.csh (revision 31433) +++ /branches/czw_branch/20110406/tools/mysql-dump/ops_dump.csh (revision 31434) @@ -3,9 +3,9 @@ echo "## `/bin/date +%FT%T`" -set EMAILTO = schastel@ifa.hawaii.edu +set EMAILTO = ps-ipp-ops@ifa.hawaii.edu # dump ippRequestServer from ippc19 set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps #path(SAB) -mysqldump --compress -hippc19 -uipp -pipp --flush-logs --single-transaction --log-error=RS-err ippRequestServer | gzip -9 > $DUMP_PATH/mysql-RS-ippc19-`date +%FT%T`.dump.gz +mysqldump --compress -hippc19 -uipp -pipp --flush-logs --single-transaction --log-error=RS-err ippRequestServer | bzip2 > $DUMP_PATH/mysql-RS-ippc19-`date +%FT%T`.dump.bz if ($status != 0) then /bin/mail -s "Warning ippRequestServer/RS dump failed" $EMAILTO < msg @@ -15,6 +15,6 @@ set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps #path(SAB) set DATEVAR=`date +%FT%T` #date -set FILENAME = $DUMP_PATH/mysql-ippadmin-ippdb01-$DATEVAR.dump.gz -mysqldump --compress -hippdb01 -uipp -pipp --flush-logs --single-transaction ippadmin | gzip -9 > $FILENAME +set FILENAME = $DUMP_PATH/mysql-ippadmin-ippdb01-$DATEVAR.dump.bz +mysqldump --compress -hippdb01 -uipp -pipp --flush-logs --single-transaction ippadmin | bzip2 > $FILENAME if ($status != 0) then /bin/mail -s "Warning ippadmin dump failed" $EMAILTO < msg @@ -24,5 +24,5 @@ set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps #path(SAB) cd $DUMP_PATH/distribution/ #link dir -ln -f ../mysql-ippadmin-ippdb01-$DATEVAR.dump.gz ippdb01-ippadmin.dump.gz +ln -f ../mysql-ippadmin-ippdb01-$DATEVAR.dump.bz ippdb01-ippadmin.dump.bz @@ -30,6 +30,6 @@ set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps #path(SAB) set DATEVAR=`date +%FT%T` #date -set FILENAME = $DUMP_PATH/mysql-isp-ippdb01-$DATEVAR.dump.gz -mysqldump --compress -hippdb01 -uipp -pipp --flush-logs --single-transaction isp | gzip -9 > $FILENAME +set FILENAME = $DUMP_PATH/mysql-isp-ippdb01-$DATEVAR.dump.bz +mysqldump --compress -hippdb01 -uipp -pipp --flush-logs --single-transaction isp | bzip2 > $FILENAME if ($status != 0) then /bin/mail -s "Warning isp dump failed" $EMAILTO < msg @@ -39,4 +39,4 @@ set DUMP_PATH = /data/ipp001.0/ipp/mysql-dumps #path(SAB) cd $DUMP_PATH/distribution/ #link dir -ln -f ../mysql-isp-ippdb01-$DATEVAR.dump.gz ippdb01-isp.dump.gz +ln -f ../mysql-isp-ippdb01-$DATEVAR.dump.bz ippdb01-isp.dump.bz Index: /branches/czw_branch/20110406/tools/regpeek.pl =================================================================== --- /branches/czw_branch/20110406/tools/regpeek.pl (revision 31433) +++ /branches/czw_branch/20110406/tools/regpeek.pl (revision 31434) @@ -17,4 +17,5 @@ } print "Date: $date\n"; +print "CMD: regtool -checkstatus -date $date -class_id ota33 -dbname gpc1 -simple\n"; chomp(my $bad_exp = `regtool -checkstatus -date $date -class_id ota33 -dbname gpc1 -simple | grep 'stop run' | head -1`); my @row = split /\s+/, $bad_exp;