Index: /branches/eam_branches/ipp-20110213/Ohana/src/addstar/src/ReadStarsFITS.c =================================================================== --- /branches/eam_branches/ipp-20110213/Ohana/src/addstar/src/ReadStarsFITS.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/Ohana/src/addstar/src/ReadStarsFITS.c (revision 30812) @@ -45,4 +45,8 @@ } if (!strcmp (type, "PS1_V2")) { + if (table.header[0].Naxis[0] == 20) { + // skip the invalid DETEFF tables which were mistakenly labeled as PS1_V2 + return (NULL); + } stars = Convert_PS1_V2 (&table, &Nstars); } @@ -51,5 +55,5 @@ } if (stars == NULL) { - fprintf (stderr, "ERROR: invalid table type %s\n", type); + fprintf (stderr, "invalid table type %s\n", type); return (NULL); } @@ -212,4 +216,85 @@ Stars *stars; CMF_PS1_V1 *ps1data; + + // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec. this was a + // mistake in two ways: a new format should have been defined (eg, CMF_PS1_V2), and the + // layout used did not have clean byte-boundaries for the corresponding structure. The + // former means we have two varieties of CMF_PS1_V1 out there; the latter means that the + // autocode tools do not work to read in the new version, even if we recognize it. Here we + // test for the existence of the broken version (table[0].headers[0].Naxis[0] == 136), and + // call a special conversion function if it is found. + + if (table[0].header[0].Naxis[0] == 136) { + stars = Convert_PS1_V1_Alt (table, nstars); + return (stars); + } + + ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL); + if (!ps1data) { + fprintf (stderr, "skipping inconsistent entry\n"); + return (NULL); + } + ZeroPt = GetZeroPoint(); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQual = ps1data[i].psfQual; + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.crNsigma = ps1data[i].crNsigma; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); + stars[i].measure.theta = ToShortDegrees(ps1data[i].df); + + stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID, + // averef is set in find_matches, dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} + +Stars *Convert_PS1_V1_Alt (FTable *table, unsigned int *nstars) { + + off_t Nstars; + unsigned int i; + double ZeroPt; + Stars *stars; + CMF_PS1_V2 *ps1data; // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec. this was a @@ -221,10 +306,5 @@ // call a special conversion function if it is found. - if (table[0].header[0].Naxis[0] == 136) { - stars = Convert_PS1_V1_Alt (table, nstars); - return (stars); - } - - ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL); + ps1data = gfits_table_get_CMF_PS1_V1_Alt (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -286,5 +366,5 @@ } -Stars *Convert_PS1_V1_Alt (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_V2 (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -294,13 +374,5 @@ CMF_PS1_V2 *ps1data; - // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec. this was a - // mistake in two ways: a few format should have been defined (eg, CMF_PS1_V2), and the - // layout used did not have clean byte-boundaries for the corresponding structure. The - // former means we have two varieties of CMF_PS1_V1 out there; the latter means that the - // autocode tools do not work to read in the new version, even if we recognize it. Here we - // test for the existence of the broken version (table[0].headers[0].Naxis[0] == 136), and - // call a special conversion function if it is found. - - ps1data = gfits_table_get_CMF_PS1_V1_Alt (table, &Nstars, NULL); + ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -362,5 +434,5 @@ } -Stars *Convert_PS1_V2 (FTable *table, unsigned int *nstars) { +Stars *Convert_PS1_V3 (FTable *table, unsigned int *nstars) { off_t Nstars; @@ -368,7 +440,7 @@ double ZeroPt; Stars *stars; - CMF_PS1_V2 *ps1data; - - ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL); + CMF_PS1_V3 *ps1data; + + ps1data = gfits_table_get_CMF_PS1_V3 (table, &Nstars, NULL); if (!ps1data) { fprintf (stderr, "skipping inconsistent entry\n"); @@ -429,70 +501,2 @@ return (stars); } - -Stars *Convert_PS1_V3 (FTable *table, unsigned int *nstars) { - - off_t Nstars; - unsigned int i; - double ZeroPt; - Stars *stars; - CMF_PS1_V3 *ps1data; - - ps1data = gfits_table_get_CMF_PS1_V3 (table, &Nstars, NULL); - if (!ps1data) { - fprintf (stderr, "skipping inconsistent entry\n"); - return (NULL); - } - ZeroPt = GetZeroPoint(); - - ALLOCATE (stars, Stars, Nstars); - for (i = 0; i < Nstars; i++) { - InitStar (&stars[i]); - stars[i].measure.Xccd = ps1data[i].X; - stars[i].measure.Yccd = ps1data[i].Y; - stars[i].measure.dXccd = ToShortPixels(ps1data[i].dX); - stars[i].measure.dYccd = ToShortPixels(ps1data[i].dY); - - stars[i].measure.posangle = ToShortDegrees(ps1data[i].posangle); - stars[i].measure.pltscale = ps1data[i].pltscale; - - if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { - stars[i].measure.M = NAN; - } else { - stars[i].measure.M = ps1data[i].M + ZeroPt; - } - stars[i].measure.dM = ps1data[i].dM; - stars[i].measure.dMcal = ps1data[i].dMcal; - stars[i].measure.Map = ps1data[i].Map + ZeroPt; - - stars[i].measure.Sky = ps1data[i].sky; - stars[i].measure.dSky = ps1data[i].dSky; - - stars[i].measure.psfChisq = ps1data[i].psfChisq; - stars[i].measure.psfQual = ps1data[i].psfQual; - stars[i].measure.psfNdof = ps1data[i].psfNdof; - stars[i].measure.psfNpix = ps1data[i].psfNpix; - stars[i].measure.crNsigma = ps1data[i].crNsigma; - stars[i].measure.extNsigma = ps1data[i].extNsigma; - - stars[i].measure.FWx = ToShortPixels(ps1data[i].fx); - stars[i].measure.FWy = ToShortPixels(ps1data[i].fy); - stars[i].measure.theta = ToShortDegrees(ps1data[i].df); - - stars[i].measure.Mxx = ToShortPixels(ps1data[i].Mxx); - stars[i].measure.Mxy = ToShortPixels(ps1data[i].Mxy); - stars[i].measure.Myy = ToShortPixels(ps1data[i].Myy); - - stars[i].measure.photFlags = ps1data[i].flags; - - // this is may optionally be replaced by the internal sequence (see FilterStars.c) - stars[i].measure.detID = ps1data[i].detID; - - // the Average fields and the following Measure fields are set in FilterStars after - // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID, - // averef is set in find_matches, dbFlags is zero on ingest. - - // the following fields are currently not being set anywhere: t_msec - } - *nstars = Nstars; - return (stars); -} Index: /branches/eam_branches/ipp-20110213/Ohana/src/dvomerge/src/merge_catalogs_old.c =================================================================== --- /branches/eam_branches/ipp-20110213/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/Ohana/src/dvomerge/src/merge_catalogs_old.c (revision 30812) @@ -219,8 +219,14 @@ } if (fabs(output[0].measure[Nmeas].dR) > 10*RADIUS) { - fprintf (stderr, "error: %10.6f,%10.6f vs %10.6f,%10.6f (%f,%f vs %f,%f)\n", - output[0].average[n].R, output[0].average[n].D, Rin, Din, - X1[i], X2[Jmin], Y1[i], Y2[Jmin]); - // XXX abort on this? -- this is a bad failure... + // ok take declination into account and check again. + double cosD = cos(RAD_DEG*Din); + if (fabs(output[0].measure[Nmeas].dR*cosD) > 10*RADIUS) { + + fprintf (stderr, "error: %10.6f,%10.6f vs %10.6f,%10.6f (%f,%f vs %f,%f)\n", + output[0].average[n].R, output[0].average[n].D, Rin, Din, + X1[i], X2[Jmin], Y1[i], Y2[Jmin]); + + // XXX abort on this? -- this is a bad failure... + } } input[0].found[N] = Nmeas; Index: /branches/eam_branches/ipp-20110213/Ohana/src/getstar/src/MatchCoords.c =================================================================== --- /branches/eam_branches/ipp-20110213/Ohana/src/getstar/src/MatchCoords.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/Ohana/src/getstar/src/MatchCoords.c (revision 30812) @@ -3,4 +3,6 @@ # define FLT_MAX 1e32 # endif +int ComputeLMLimits(double xmin, double xmax, double ymin, double ymax, + double *Lmin, double *Lmax, double *Mmin, double *Mmax, Coords *coords); /* given coordinate, find images in list that contain the point */ @@ -12,4 +14,5 @@ double Xi[4], Yi[4]; /* image and original corners */ double xmin, xmax, ymin, ymax; + double Lmin, Lmax, Mmin, Mmax; @@ -20,5 +23,7 @@ also define the vtable entries for the images we keep */ + int haveLimits = 0; for (i = 0; i < NdbImages; i++) { + haveLimits = 0; if (!WITH_PHU && !strcmp (&dbImages[i].coords.ctype[4], "-DIS")) continue; @@ -30,5 +35,5 @@ /* define image corners */ SetImageCorners (Xi, Yi, &dbImages[i]); - // Xi[4] = Xi[0]; Yi[4] = Yi[0]; + // Xi[3] = Xi[0]; Yi[3] = Yi[0]; ymin = xmin = +FLT_MAX; @@ -52,9 +57,38 @@ continue; } - - if (VERBOSE) fprintf(stderr, "%ld %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); + if (VERBOSE) fprintf(stderr, OFF_T_FMT" %8.1f %8.1f %s\n", i, x, y, dbImages[i].name); if ((x >= xmin && x <= xmax) && (y >= ymin && y <= ymax)) { + if (VERBOSE) fprintf(stderr, "\tpotential overlap for point %d image %i\n", j, (int) i); + + // We may be using the astrometry far outside it's range of validity. + // As a sanity check make sure that the focal plane coords are in bounds. + // See ticket 1381 + if (!haveLimits) { + // We defer computing the focal plane limits until we need them + status = ComputeLMLimits(xmin, xmax, ymin, ymax, + &Lmin, &Lmax, &Mmin, &Mmax, &dbImages[i].coords); + if (!status) { + continue; + } + haveLimits = 1; + } + + double L, M; + status = RD_to_LM(&L, &M, pt->ra, pt->dec, &dbImages[i].coords); + if (!status) { + // This won't happen because RD_to_XY succeeded above + continue; + } + + if (VERBOSE) fprintf(stderr, "L: %f M: %f\n", L, M); + + if (L < Lmin || L > Lmax || M < Mmin || M > Mmax) { + if (VERBOSE) fprintf(stderr, "\trejecting match for because focal plane coordinates are out of range\n"); + continue; + } + if (VERBOSE) fprintf(stderr, "\tmatch confirmed\n"); + totalMatches++; @@ -88,2 +122,36 @@ } } + +// compute bounding box of this image in focal plane coordinates +int ComputeLMLimits(double xmin, double xmax, double ymin, double ymax, + double *Lmin, double *Lmax, double *Mmin, double *Mmax, Coords *coords) +{ + double L0, L1, M0, M1; + int status = XY_to_LM(&L0, &M0, xmin, ymin, coords); + if (!status) { + fprintf(stderr, "failed to transform image corner to LM\n"); + return 0; + } + status = XY_to_LM(&L1, &M1, xmax, ymax, coords); + if (!status) { + fprintf(stderr, "failed to transform image corner to LM\n"); + return 0; + } + + if (L1 > L0) { + *Lmin = L0; + *Lmax = L1; + } else { + *Lmin = L1; + *Lmax = L0; + } + if (M1 > M0) { + *Mmin = M0; + *Mmax = M1; + } else { + *Mmin = M1; + *Mmax = M0; + } + if (VERBOSE) fprintf(stderr, "Lmin: %f Lmax: %f Mmin: %f Mmax: %f\n", *Lmin, *Lmax, *Mmin, *Mmax); + return 1; +} Index: /branches/eam_branches/ipp-20110213/Ohana/src/libohana/include/ohana_allocate.h =================================================================== --- /branches/eam_branches/ipp-20110213/Ohana/src/libohana/include/ohana_allocate.h (revision 30811) +++ /branches/eam_branches/ipp-20110213/Ohana/src/libohana/include/ohana_allocate.h (revision 30812) @@ -49,5 +49,5 @@ # define REALLOCATE(PTR,TYPE,SIZE) { \ - PTR = (TYPE *) realloc(PTR,(unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ + PTR = (TYPE *) realloc(PTR,(size_t)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ if (PTR == NULL) { \ fprintf(stderr,"failed realloc at %d in %s\n", __LINE__, __FILE__); \ @@ -57,5 +57,5 @@ if ((NCURR) >= (SIZE)) { \ SIZE += DELTA; \ - PTR = (TYPE *) realloc(PTR,(unsigned)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ + PTR = (TYPE *) realloc(PTR,(size_t)(MAX(((SIZE)*((int)sizeof(TYPE))),1))); \ if (PTR == NULL) { \ fprintf(stderr,"failed realloc increment at %d in %s\n", __LINE__, __FILE__); \ Index: /branches/eam_branches/ipp-20110213/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- /branches/eam_branches/ipp-20110213/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 30811) +++ /branches/eam_branches/ipp-20110213/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 30812) @@ -1209,5 +1209,4 @@ if ($@) { carp "nebulous delete for $todelete failed. Ignoring.\n"; - return $PS_EXIT_SYS_ERROR; } } Index: /branches/eam_branches/ipp-20110213/ippMonitor/raw/czartool_labels.php =================================================================== --- /branches/eam_branches/ipp-20110213/ippMonitor/raw/czartool_labels.php (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippMonitor/raw/czartool_labels.php (revision 30812) @@ -220,5 +220,5 @@ showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); - + showReplicationsStatus($REPL_HOST_PSTAMP, $REPL_USER_PSTAMP, $REPL_PASSWORD_PSTAMP, $REPL_DBNAME_PSTAMP); echo ""; echo "\n"; Index: /branches/eam_branches/ipp-20110213/ippMonitor/raw/site.php.in =================================================================== --- /branches/eam_branches/ipp-20110213/ippMonitor/raw/site.php.in (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippMonitor/raw/site.php.in (revision 30812) @@ -32,3 +32,8 @@ $REPL_DBNAME_NEBULOUS = "Nebulous"; +$REPL_HOST_PSTAMP = "ippc19.IfA.Hawaii.Edu"; +$REPL_USER_PSTAMP = "ippMonitor"; +$REPL_PASSWORD_PSTAMP = "ippMonitor"; +$REPL_DBNAME_PSTAMP = "ippRequestServer"; + ?> Index: /branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- /branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_cleanup.pl (revision 30812) @@ -102,4 +102,5 @@ my $cam_path_base = $run->{cam_path_base}; my $cam_reduction = $run->{cam_reduction}; +my $replace = $run->{re_place}; $cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL'); @@ -111,5 +112,5 @@ &my_die("unexpected run state found: $state", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $state ne "goto_cleaned"; -&my_die("clean not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw"; +&my_die("cleanup not allowed for raw stage, use goto_restore", $magic_ds_id, $PS_EXIT_CONFIG_ERROR) if $stage eq "raw" and $replace; my $recipe_psastro; @@ -217,4 +218,12 @@ # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base); } + } elsif ($stage eq "raw") { + if ($backup_path_base) { + if ($backup_path_base =~ /fits/) { + $bimage = $backup_path_base; + } else { + $bimage = $backup_path_base . ".fits"; + } + } } Index: /branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- /branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_revert.pl (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_revert.pl (revision 30812) @@ -78,5 +78,15 @@ defined $outroot; -&my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'raw' and (!$bytes or !$md5sum)); +if (defined($replace) and ($replace eq "T")) { + $replace = 1; +} else { + $replace = 0; +} + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; + +&my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($replace and $stage eq 'raw' and (!$bytes or !$md5sum)); &my_die("cam_path_base is required for chip stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($stage eq 'chip' and !$cam_path_base); @@ -84,7 +94,4 @@ -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration - -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; $cam_reduction = 'DEFAULT' if !defined $cam_reduction or ($cam_reduction eq "NULL"); @@ -118,10 +125,4 @@ my $basename = basename($path_base); my $nebulousInput = inNebulous($dirname); - -if (defined($replace) and ($replace eq "T")) { - $replace = 1; -} else { - $replace = 0; -} &my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) @@ -162,5 +163,9 @@ $image = $path_base . ".fits"; } - $bimage = $backup_path_base . ".fits"; + if ($backup_path_base =~ /.*\.fits$/) { + $bimage = $backup_path_base; + } else { + $bimage = $backup_path_base . ".fits"; + } } elsif ($stage eq "chip") { # Check to see if we're using dynamic masks @@ -302,5 +307,5 @@ sub revert_files { my $replace = shift; - return if !$replace; +# return if !$replace; my $image = shift; @@ -316,10 +321,10 @@ if ($image) { - revert_file($image, $bimage) or + revert_file($replace, $image, $bimage) or &my_die("failed to restore image file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } if ($mask) { - if (!revert_file($mask, $bmask)) { + if (!revert_file($replace, $mask, $bmask)) { &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } @@ -327,5 +332,5 @@ if ($ch_mask) { - if (!revert_file($ch_mask, $bch_mask)) { + if (!revert_file($replace, $ch_mask, $bch_mask)) { &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } @@ -334,15 +339,15 @@ if ($weight) { - revert_file($weight, $bweight) or + revert_file($replace, $weight, $bweight) or &my_die("failed to restore variance image", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } if ($sources) { - revert_file($sources, $bsources) or + revert_file($replace, $sources, $bsources) or &my_die("failed to restore sources file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } if ($astrom) { - revert_file($astrom, $bastrom) or + revert_file($replace, $astrom, $bastrom) or &my_die("failed to restore astrometry file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); } @@ -366,7 +371,20 @@ sub revert_file { + my $replace = shift; my $original = shift; my $backup = shift; my $force = 1; # force deletion of backup files in nebulous + + if (!$replace) { + # we don't need to do all of this checking unless the destreak run is + # replace mode just delete the backup + my $error_code = $ipprc->kill_file($backup); + if ($error_code) { + print STDERR "failed to kill $backup: error_code $error_code\n"; + return 0; + } else { + return 1; + } + } my $o_path = $ipprc->file_resolve($original); Index: /branches/eam_branches/ipp-20110213/ippScripts/scripts/register_imfile.pl =================================================================== --- /branches/eam_branches/ipp-20110213/ippScripts/scripts/register_imfile.pl (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippScripts/scripts/register_imfile.pl (revision 30812) @@ -181,4 +181,5 @@ if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type")) eq "NULL") { &my_die_for_add ("exp_type not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); } my $dateobs = &value_for_flag($cmdflags, 0.0, "-dateobs"); +my $exp_type = &value_for_flag($cmdflags, "NULL", "-exp_type"); my $command = "$regtool -addprocessedimfile"; Index: /branches/eam_branches/ipp-20110213/ippTasks/destreak.pro =================================================================== --- /branches/eam_branches/ipp-20110213/ippTasks/destreak.pro (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTasks/destreak.pro (revision 30812) @@ -204,17 +204,20 @@ book getword magicToDS $pageName dbname -var DBNAME - sprintf logfile "%s/%s.mds.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT substr $COMPONENT 0 3 COMP_HEAD if ("$COMP_HEAD" == "sky") set.host.for.skycell $COMPONENT + set.workdir.by.skycell $COMPONENT $OUTROOT $default_host WORKDIR else # assume component is a class_id, if not we will default to anyhost set.host.for.camera $CAMERA $COMPONENT - end + set.workdir.by.camera $CAMERA $COMPONENT $OUTROOT $default_host WORKDIR + end + + sprintf logfile "%s/%s.mds.%s.%s.%s.log" $WORKDIR $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT # TODO: do not add recoveryroot or replace if they are null or zero - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE add_standard_args run @@ -407,15 +410,16 @@ book getword magicDSToRevert $pageName dbname -var DBNAME - sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $OUTROOT $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT - substr $COMPONENT 0 3 COMP_HEAD if ("$COMP_HEAD" == "sky") set.host.for.skycell $COMPONENT + set.workdir.by.skycell $COMPONENT $OUTROOT $default_host WORKDIR else # assume component is a class_id, if not we will default to anyhost set.host.for.camera $CAMERA $COMPONENT - end - - $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED --run-state $RUN_STATE --recovery_path_base $RECOVERY_PATH_BASE + set.workdir.by.camera $CAMERA $COMPONENT $OUTROOT $default_host WORKDIR + end + sprintf logfile "%s/%s.mds.revert.%s.%s.%s.log" $WORKDIR $EXP_ID $MAGIC_DS_ID $STAGE_ID $COMPONENT + + $run = magic_destreak_revert.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --replace $REPLACE --bothways $BOTHWAYS --magicked $MAGICKED --run-state $RUN_STATE --recovery_path_base $RECOVERY_PATH_BASE add_standard_args run Index: /branches/eam_branches/ipp-20110213/ippTasks/pstamp.pro =================================================================== --- /branches/eam_branches/ipp-20110213/ippTasks/pstamp.pro (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTasks/pstamp.pro (revision 30812) @@ -19,5 +19,5 @@ # give up on dependents with fault_count >= $PSTAMP_MAX_FAULT_COUNT -$PSTAMP_MAX_FAULT_COUNT = 5 +$PSTAMP_MAX_FAULT_COUNT = 3 macro set.max.fault.count $PSTAMP_MAX_FAULT_COUNT = $1 @@ -717,8 +717,8 @@ $MYLOGFILE = $OUTDIR/checkdep.$DEP_ID.log - stdout $MYLOGFILE + stdout NULL stderr $MYLOGFILE - $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT + $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC --fault_count $FAULT_COUNT --max_fault_count $PSTAMP_MAX_FAULT_COUNT --logfile $MYLOGFILE add_standard_args run @@ -928,7 +928,7 @@ task.exec + stdout $LOGSUBDIR/pstamp.stopfaulted.log stderr $LOGSUBDIR/pstamp.stopfaulted.log - stderr $LOGSUBDIR/pstamp.stopfaulted.log - $run = pstampstopfaulted + $run = pstampstopfaulted -fault_count $PSTAMP_MAX_FAULT_COUNT if ($DB:n == 0) option DEFAULT Index: /branches/eam_branches/ipp-20110213/ippTasks/register.pro =================================================================== --- /branches/eam_branches/ipp-20110213/ippTasks/register.pro (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTasks/register.pro (revision 30812) @@ -203,5 +203,5 @@ $regRevertImfile_DB = 0 $regPendingExp_DB = 0 - +$reg_datePAGE = 0 # select images ready for register analysis # new entries are added to regPendingImfile Index: /branches/eam_branches/ipp-20110213/ippToPsps/src/Batch.h =================================================================== --- /branches/eam_branches/ipp-20110213/ippToPsps/src/Batch.h (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippToPsps/src/Batch.h (revision 30812) @@ -14,5 +14,5 @@ #include #include -#include "ippToPspsConfig.h" +#include "Config.h" #include #include Index: /branches/eam_branches/ipp-20110213/ippToPsps/src/DifferenceBatch.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippToPsps/src/DifferenceBatch.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippToPsps/src/DifferenceBatch.c (revision 30812) @@ -8,5 +8,5 @@ #include "ippToPsps.h" -#include "ippToPspsConfig.h" +#include "Config.h" #include "ippToPspsDiffEnums.h" Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_todestreak_raw.sql (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_todestreak_raw.sql (revision 30812) @@ -24,5 +24,5 @@ re_place, remove, - 10000 AS priority + IFNULL(Label.priority, 10000) AS priority FROM magicDSRun JOIN magicMask USING (magic_id) @@ -35,6 +35,9 @@ ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id AND magicDSFile.component = rawImfile.class_id +LEFT JOIN Label ON magicDSRun.label = Label.label WHERE magicDSRun.state = 'new' AND magicDSRun.stage = 'raw' AND magicDSFile.component IS NULL + AND (Label.active OR Label.active IS NULL) + Index: /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/share/magicdstool_torevert_diff.sql (revision 30812) @@ -13,5 +13,4 @@ magicDSFile.recovery_path_base, "NULL" AS cam_path_base, - recovery_path_base, CAST(diffRun.bothways AS SIGNED) AS bothways, 0 AS bytes, Index: /branches/eam_branches/ipp-20110213/ippTools/src/addtool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/addtool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/addtool.c (revision 30812) @@ -324,5 +324,5 @@ // pxUpdateRun gets parameters from config->args and runs the update query bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", - "addProcessedExp", false); + "addProcessedExp", false, false); psFree(query); Index: /branches/eam_branches/ipp-20110213/ippTools/src/bgtool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/bgtool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/bgtool.c (revision 30812) @@ -501,5 +501,5 @@ psString query = psStringCopy("UPDATE chipBackgroundRun"); bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id", - "chipBackgroundImfile", true); + "chipBackgroundImfile", true, true); psFree(query); @@ -1471,5 +1471,5 @@ psString query = psStringCopy("UPDATE warpBackgroundRun"); bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id", - "warpBackgroundSkyfile", true); + "warpBackgroundSkyfile", true, true); psFree(query); Index: /branches/eam_branches/ipp-20110213/ippTools/src/camtool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/camtool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/camtool.c (revision 30812) @@ -283,5 +283,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true); + bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true, true); if (!result) { psError(psErrorCodeLast(), false, "pxUpdateRun failed"); Index: /branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/camtoolConfig.c (revision 30812) @@ -86,4 +86,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + pxmagicAddArguments(updaterunArgs); // -pendingexp @@ -299,6 +300,6 @@ PXOPT_ADD_MODE("-addprocessedexp", "add a processed exposure", CAMTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); PXOPT_ADD_MODE("-processedexp", "show processed exposures", CAMTOOL_MODE_PROCESSEDEXP, processedexpArgs); - PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); - PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exposure", CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "undo a processed exposure", CAMTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "changed processed exp properties", CAMTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CAMTOOL_MODE_BLOCK, blockArgs); PXOPT_ADD_MODE("-masked", "show blocked labels", CAMTOOL_MODE_MASKED, maskedArgs); Index: /branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/chiptool.c (revision 30812) @@ -443,5 +443,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true); + bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true, true); if (!result) { psError(psErrorCodeLast(), false, "pxUpdateRun failed"); Index: /branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/chiptoolConfig.c (revision 30812) @@ -99,4 +99,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "set dist group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); + pxmagicAddArguments(updaterunArgs); // -setimfiletoupdate Index: /branches/eam_branches/ipp-20110213/ippTools/src/diffphottool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/diffphottool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/diffphottool.c (revision 30812) @@ -228,5 +228,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "diffPhotRun", "diff_phot_id", "diffPhotSkyfile", true); + bool result = pxUpdateRun(config, where, &query, "diffPhotRun", "diff_phot_id", "diffPhotSkyfile", true, false); psFree(query); Index: /branches/eam_branches/ipp-20110213/ippTools/src/difftool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/difftool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/difftool.c (revision 30812) @@ -224,5 +224,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true); + bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true, true); psFree(query); Index: /branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/difftoolConfig.c (revision 30812) @@ -73,4 +73,5 @@ psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_diff_mode", 0, "specify type of difference (WW=1,WS=2,SW=3,SS=4)", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + pxmagicAddArguments(updaterunArgs); // -addinputskyfile Index: /branches/eam_branches/ipp-20110213/ippTools/src/faketool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/faketool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/faketool.c (revision 30812) @@ -348,5 +348,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true); + bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true, false); if (!result) { psError(psErrorCodeLast(), false, "pxUpdateRun failed"); Index: /branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/magicdstool.c (revision 30812) @@ -1395,4 +1395,5 @@ (strcmp(state, "full") == 0) || (strcmp(state, "drop") == 0) || + (strcmp(state, "wait") == 0) || (strcmp(state, "failed_revert") == 0) || (strcmp(state, "error_cleaned") == 0) || Index: /branches/eam_branches/ipp-20110213/ippTools/src/pubtool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/pubtool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/pubtool.c (revision 30812) @@ -37,4 +37,5 @@ static bool addMode(pxConfig *config); static bool revertMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -63,4 +64,5 @@ MODECASE(PUBTOOL_MODE_ADD, addMode); MODECASE(PUBTOOL_MODE_REVERT, revertMode); + MODECASE(PUBTOOL_MODE_UPDATERUN, updaterunMode); default: psAbort("invalid option (this should not happen)"); @@ -459,2 +461,40 @@ } +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + PXOPT_COPY_S64(config->args, where, "-pub_id", "pub_id", "=="); + PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-fault", "fault", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + psString query = NULL; // Query to run + psStringAppend(&query, "UPDATE publishRun LEFT JOIN publishDone using(pub_id) SET state = '%s'", state); + + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "\n WHERE %s", clause); + psFree(clause); + } else { + psError(PS_ERR_UNKNOWN, false, "select arguments are required"); + return false; + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + long numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("pubtool", PS_LOG_INFO, "%ld rows updated.", numUpdated); + + return true; +} Index: /branches/eam_branches/ipp-20110213/ippTools/src/pubtool.h =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/pubtool.h (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/pubtool.h (revision 30812) @@ -31,4 +31,5 @@ PUBTOOL_MODE_ADD, PUBTOOL_MODE_REVERT, + PUBTOOL_MODE_UPDATERUN, } pubtoolMode; Index: /branches/eam_branches/ipp-20110213/ippTools/src/pubtoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/pubtoolConfig.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/pubtoolConfig.c (revision 30812) @@ -102,4 +102,13 @@ psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on label", NULL); + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-pub_id", 0, "search on pub_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault", NULL); + psMetadata *argSets = psMetadataAlloc(); @@ -112,4 +121,5 @@ PXOPT_ADD_MODE("-add", "", PUBTOOL_MODE_ADD, addArgs); PXOPT_ADD_MODE("-revert", "", PUBTOOL_MODE_REVERT, revertArgs); + PXOPT_ADD_MODE("-updaterun", "", PUBTOOL_MODE_UPDATERUN, updaterunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branches/ipp-20110213/ippTools/src/pxtools.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/pxtools.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/pxtools.c (revision 30812) @@ -278,5 +278,5 @@ // shared code for updating the various strings for a Run -bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group) +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group, bool has_magicked) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -360,4 +360,7 @@ if (fileWhere) { psStringAppend(pQuery, "%s", fileWhere); + } + if (has_magicked) { + pxmagicAddWhere(config, pQuery, runTable); } Index: /branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/pxtools.h (revision 30812) @@ -84,5 +84,5 @@ bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); -bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group); +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group, bool has_magicked); #define PXOPT_ADD_MODE(option, comment, modeval, argset) \ Index: /branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/stacktool.c (revision 30812) @@ -750,5 +750,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true); + bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true, false); psFree(query); Index: /branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/staticskytool.c (revision 30812) @@ -360,5 +360,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true); + bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false); psFree(query); psFree(where); Index: /branches/eam_branches/ipp-20110213/ippTools/src/warptool.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/warptool.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/warptool.c (revision 30812) @@ -435,5 +435,5 @@ // pxUpdateRun gets parameters from config->args and updates - bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true); + bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true, true); psFree(query); Index: /branches/eam_branches/ipp-20110213/ippTools/src/warptoolConfig.c =================================================================== --- /branches/eam_branches/ipp-20110213/ippTools/src/warptoolConfig.c (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippTools/src/warptoolConfig.c (revision 30812) @@ -141,5 +141,4 @@ psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_begin", 0, "search for runs by registration time (>=)", NULL); psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-registered_end", 0, "search for runs by registration time (<)", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); @@ -149,4 +148,5 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + pxmagicAddArguments(updaterunArgs); // -exp Index: /branches/eam_branches/ipp-20110213/ippconfig/gpc1/format_20100228.config =================================================================== --- /branches/eam_branches/ipp-20110213/ippconfig/gpc1/format_20100228.config (revision 30812) +++ /branches/eam_branches/ipp-20110213/ippconfig/gpc1/format_20100228.config (revision 30812) @@ -0,0 +1,654 @@ +# Refurbished GPC1 +# Change effective from 22 January 2010 (TJD=5219): new CCD temperature keywords + +# How to identify this type +RULE METADATA + ORIGIN STR PS1 + TELESCOP STR PS1 + INSTRUME STR gpc1 + CONTROLR STR STARGRASP + NAMPS S32 64 + MJD-OBS F64 55219.0 # OP: >= +END + +# How to read this data +FILE METADATA + PHU STR CHIP # The FITS file represents a single chip + EXTENSIONS STR CELL # The extensions represent cells + FPA.OBS STR FILENAME # A PHU keyword for unique identifier within the hierarchy level + CONTENT STR DETECTOR # How to determine content of FITS file + CONTENT.RULE STR {CHIP.ID} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID58-2-15a1 STR XY01:GPCChip # chip 00 + CCID58-2-18a1 STR XY02:GPCChip # chip 01 + CCID58-2-04a2 STR XY03:GPCChip # chip 02 + CCID58-1-01b1 STR XY04:GPCChip # chip 03 + CCID58-1-09b1 STR XY05:GPCChip # chip 04 + CCID58-2-09b1 STR XY06:GPCChip # chip 05 + CCID58-2-22a1 STR XY10:GPCChip # chip 06 + CCID58-2-01a1 STR XY11:GPCChip # chip 07 + CCID58-3-11a1 STR XY12:GPCChip # chip 08 + CCID58-3-06a1 STR XY13:GPCChip # chip 09 + CCID58-3-05b1 STR XY14:GPCChip # chip 10 + CCID58-3-08b1 STR XY15:GPCChip # chip 11 + CCID58-1-02b1 STR XY16:GPCChip # chip 12 + CCID58-2-07b2 STR XY17:GPCChip # chip 13 + CCID58-1-19a1 STR XY20:GPCChip # chip 14 + CCID58-1-04a1 STR XY21:GPCChip # chip 15 + CCID58-2-09a2 STR XY22:GPCChip # chip 16 + CCID58-1-02a1 STR XY23:GPCChip # chip 17 + CCID58-3-10b1 STR XY24:GPCChip # chip 18 + CCID58-3-05b2 STR XY25:GPCChip # chip 19 + CCID58-3-04b1 STR XY26:GPCChip # chip 20 + CCID58-1-18b1 STR XY27:GPCChip # chip 21 + CCID58-3-13a2 STR XY30:GPCChip # chip 22 + CCID58-3-05a1 STR XY31:GPCChip # chip 23 + CCID58-2-01a2 STR XY32:GPCChip # chip 24 + CCID58-1-02a2 STR XY33:GPCChip # chip 25 + CCID58-3-03b1 STR XY34:GPCChip # chip 26 + CCID58-1-01b2 STR XY35:GPCChip # chip 27 + CCID58-3-14b1 STR XY36:GPCChip # chip 28 + CCID58-2-23b2 STR XY37:GPCChip # chip 29 + CCID58-3-02b1 STR XY40:GPCChip # chip 30 + CCID58-1-03b2 STR XY41:GPCChip # chip 31 + CCID58-1-07b1 STR XY42:GPCChip # chip 32 + CCID58-2-11b2 STR XY43:GPCChip # chip 33 + CCID58-2-09a1 STR XY44:GPCChip # chip 34 + CCID58-3-04a2 STR XY45:GPCChip # chip 35 + CCID58-3-02a1 STR XY46:GPCChip # chip 36 + CCID58-3-14a1 STR XY47:GPCChip # chip 37 + CCID58-1-17b1 STR XY50:GPCChip # chip 38 + CCID58-2-09b2 STR XY51:GPCChip # chip 39 + CCID58-1-04b1 STR XY52:GPCChip # chip 40 + CCID58-1-05b2 STR XY53:GPCChip # chip 41 + CCID58-3-05a2 STR XY54:GPCChip # chip 42 + CCID58-3-03a2 STR XY55:GPCChip # chip 43 + CCID58-3-02a2 STR XY56:GPCChip # chip 44 + CCID58-3-11a2 STR XY57:GPCChip # chip 45 + CCID58-2-16b1 STR XY60:GPCChip # chip 46 + CCID58-1-03b1 STR XY61:GPCChip # chip 47 + CCID58-3-03b2 STR XY62:GPCChip # chip 48 + CCID58-1-02b2 STR XY63:GPCChip # chip 49 + CCID58-1-07a1 STR XY64:GPCChip # chip 50 + CCID58-2-04a1 STR XY65:GPCChip # chip 51 + CCID58-3-10a1 STR XY66:GPCChip # chip 52 + CCID58-1-09a1 STR XY67:GPCChip # chip 53 + CCID58-2-13b1 STR XY71:GPCChip # chip 54 + CCID58-1-14b1 STR XY72:GPCChip # chip 55 + CCID58-3-11b2 STR XY73:GPCChip # chip 56 + CCID58-1-05a2 STR XY74:GPCChip # chip 57 + CCID58-1-21a1 STR XY75:GPCChip # chip 58 + CCID58-2-05a1 STR XY76:GPCChip # chip 59 +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + xy01 STR xy01:GPCCell + xy11 STR xy11:GPCCell + xy21 STR xy21:GPCCell + xy31 STR xy31:GPCCell + xy41 STR xy41:GPCCell + xy51 STR xy51:GPCCell + xy61 STR xy61:GPCCell + xy71 STR xy71:GPCCell + xy02 STR xy02:GPCCell + xy12 STR xy12:GPCCell + xy22 STR xy22:GPCCell + xy32 STR xy32:GPCCell + xy42 STR xy42:GPCCell + xy52 STR xy52:GPCCell + xy62 STR xy62:GPCCell + xy72 STR xy72:GPCCell + xy03 STR xy03:GPCCell + xy13 STR xy13:GPCCell + xy23 STR xy23:GPCCell + xy33 STR xy33:GPCCell + xy43 STR xy43:GPCCell + xy53 STR xy53:GPCCell + xy63 STR xy63:GPCCell + xy73 STR xy73:GPCCell + xy04 STR xy04:GPCCell + xy14 STR xy14:GPCCell + xy24 STR xy24:GPCCell + xy34 STR xy34:GPCCell + xy44 STR xy44:GPCCell + xy54 STR xy54:GPCCell + xy64 STR xy64:GPCCell + xy74 STR xy74:GPCCell + xy05 STR xy05:GPCCell + xy15 STR xy15:GPCCell + xy25 STR xy25:GPCCell + xy35 STR xy35:GPCCell + xy45 STR xy45:GPCCell + xy55 STR xy55:GPCCell + xy65 STR xy65:GPCCell + xy75 STR xy75:GPCCell + xy06 STR xy06:GPCCell + xy16 STR xy16:GPCCell + xy26 STR xy26:GPCCell + xy36 STR xy36:GPCCell + xy46 STR xy46:GPCCell + xy56 STR xy56:GPCCell + xy66 STR xy66:GPCCell + xy76 STR xy76:GPCCell + xy07 STR xy07:GPCCell + xy17 STR xy17:GPCCell + xy27 STR xy27:GPCCell + xy37 STR xy37:GPCCell + xy47 STR xy47:GPCCell + xy57 STR xy57:GPCCell + xy67 STR xy67:GPCCell + xy77 STR xy77:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + FPA.RADECSYS STR EQUINOX + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.OBS.MODE STR OBS_MODE + FPA.COMMENT STR CMMTOBS + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.ROTANGLE STR ROT + FPA.FOCUS STR M2Z + FPA.TIME STR SHUTOUTC + FPA.ALT STR ALT + FPA.AZ STR AZ + # FPA.TEMP STR DETTEM + FPA.M1X STR M1X + FPA.M1Y STR M1Y + FPA.M1Z STR M1Z + FPA.M1TIP STR M1TIP + FPA.M1TILT STR M1TILT + FPA.M2X STR M2X + FPA.M2Y STR M2Y + FPA.M2Z STR M2Z + FPA.M2TIP STR M2TIP + FPA.M2TILT STR M2TILT + FPA.ENV.TEMP STR ENVTEM + FPA.ENV.HUMID STR ENVHUM + FPA.ENV.WIND STR ENVWIN + FPA.ENV.DIR STR ENVDIR + + FPA.TELTEMP.M1 STR TELTEMM1 # Primary mirror temps (C) + FPA.TELTEMP.M1CELL STR TELTEMMS # Primary mirror support temps (C) + FPA.TELTEMP.M2 STR TELTEMM2 # Secondary mirror temps (C + FPA.TELTEMP.SPIDER STR TELTEMSP # Spider temperatures (C) + FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C + FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE + + FPA.PON.TIME STR PONTIME + + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR ATM1_1 + CELL.YPARITY STR ATM2_2 + CHIP.TEMP STR CDETTEM + CHIP.TEMPERATURE STR DETTEM + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera + CELL.TIME STR SHUTOUTC # Observation time + CELL.GAIN STR GAIN # Cell gain (e/ADU) + CELL.READNOISE STR RDNOISE # Cell read noise (ADU) + CELL.SATURATION STR MAXLIN # Saturation point (ADU) +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS1 + FPA.INSTRUMENT STR GPC1 + FPA.LONGITUDE STR 10:25:01.417 # West Longitude in Hours 10.4170608521 + FPA.LATITUDE STR 20:42:25.558 # North Latitude in Degrees 20.7070999146 + FPA.ELEVATION F32 3048.0 # altitude in meters + FPA.DETECTOR STR GPC1 + FPA.TIMESYS STR UTC + CHIP.XSIZE S32 4846 + CHIP.YSIZE S32 4868 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 # should be 19872 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 # should be 20029 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 # should be 25755 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 # should be 15707 + XY44 S32 20878 # should be 20890 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.READDIR S32 1 + CELL.BAD S32 0 + CELL.XSIZE S32 590 + CELL.YSIZE S32 598 + CELL.TIMESYS STR UTC +END + +# How to translation PS concepts into database lookups +DATABASE METADATA + # this rule is fragile : does not match the camera + FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.LONGITUDE STR HOURS + FPA.LATITUDE STR DEGREES + FPA.TIME STR YEAR.FIRST + CELL.TIME STR YEAR.FIRST + CELL.BINNING STR TOGETHER + # CELL.X0 STR FORTRAN + # CELL.Y0 STR FORTRAN + CELL.READNOISE STR ADU +END + +# Pattern correction information: + +PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL 0011100100000000000000000000000000000000000000000000000000000000 + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 + XY46 BOOL FALSE + XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 + XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 + XY75 BOOL FALSE + XY76 BOOL FALSE +END + +PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 STR 0000000000000000000000000000000000000000000000010000000000101101 + XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 + XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 + XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 + XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 + XY35 BOOL FALSE + XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 + XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 + XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 + XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END Index: /branches/eam_branches/ipp-20110213/ippconfig/gpc1/format_20100723.config =================================================================== --- /branches/eam_branches/ipp-20110213/ippconfig/gpc1/format_20100723.config (revision 30812) +++ /branches/eam_branches/ipp-20110213/ippconfig/gpc1/format_20100723.config (revision 30812) @@ -0,0 +1,654 @@ +# Refurbished GPC1 +# Change effective from 22 January 2010 (TJD=5219): new CCD temperature keywords + +# How to identify this type +RULE METADATA + ORIGIN STR PS1 + TELESCOP STR PS1 + INSTRUME STR gpc1 + CONTROLR STR STARGRASP + NAMPS S32 64 + MJD-OBS F64 55400.0 # OP: >= +END + +# How to read this data +FILE METADATA + PHU STR CHIP # The FITS file represents a single chip + EXTENSIONS STR CELL # The extensions represent cells + FPA.OBS STR FILENAME # A PHU keyword for unique identifier within the hierarchy level + CONTENT STR DETECTOR # How to determine content of FITS file + CONTENT.RULE STR {CHIP.ID} # How to derive the CONTENT when writing +END + +# What's in the FITS file? +CONTENTS METADATA + # CONTENT = chip name : type + CCID58-2-15a1 STR XY01:GPCChip # chip 00 + CCID58-2-18a1 STR XY02:GPCChip # chip 01 + CCID58-2-04a2 STR XY03:GPCChip # chip 02 + CCID58-1-01b1 STR XY04:GPCChip # chip 03 + CCID58-1-09b1 STR XY05:GPCChip # chip 04 + CCID58-2-09b1 STR XY06:GPCChip # chip 05 + CCID58-2-22a1 STR XY10:GPCChip # chip 06 + CCID58-2-01a1 STR XY11:GPCChip # chip 07 + CCID58-3-11a1 STR XY12:GPCChip # chip 08 + CCID58-3-06a1 STR XY13:GPCChip # chip 09 + CCID58-3-05b1 STR XY14:GPCChip # chip 10 + CCID58-3-08b1 STR XY15:GPCChip # chip 11 + CCID58-1-02b1 STR XY16:GPCChip # chip 12 + CCID58-2-07b2 STR XY17:GPCChip # chip 13 + CCID58-1-19a1 STR XY20:GPCChip # chip 14 + CCID58-1-04a1 STR XY21:GPCChip # chip 15 + CCID58-2-09a2 STR XY22:GPCChip # chip 16 + CCID58-1-02a1 STR XY23:GPCChip # chip 17 + CCID58-3-10b1 STR XY24:GPCChip # chip 18 + CCID58-3-05b2 STR XY25:GPCChip # chip 19 + CCID58-3-04b1 STR XY26:GPCChip # chip 20 + CCID58-1-18b1 STR XY27:GPCChip # chip 21 + CCID58-3-13a2 STR XY30:GPCChip # chip 22 + CCID58-3-05a1 STR XY31:GPCChip # chip 23 + CCID58-2-01a2 STR XY32:GPCChip # chip 24 + CCID58-1-02a2 STR XY33:GPCChip # chip 25 + CCID58-3-03b1 STR XY34:GPCChip # chip 26 + CCID58-1-01b2 STR XY35:GPCChip # chip 27 + CCID58-3-14b1 STR XY36:GPCChip # chip 28 + CCID58-2-23b2 STR XY37:GPCChip # chip 29 + CCID58-3-02b1 STR XY40:GPCChip # chip 30 + CCID58-1-03b2 STR XY41:GPCChip # chip 31 + CCID58-1-07b1 STR XY42:GPCChip # chip 32 + CCID58-2-11b2 STR XY43:GPCChip # chip 33 + CCID58-2-09a1 STR XY44:GPCChip # chip 34 + CCID58-3-04a2 STR XY45:GPCChip # chip 35 + CCID58-3-02a1 STR XY46:GPCChip # chip 36 + CCID58-3-14a1 STR XY47:GPCChip # chip 37 + CCID58-1-17b1 STR XY50:GPCChip # chip 38 + CCID58-2-09b2 STR XY51:GPCChip # chip 39 + CCID58-1-04b1 STR XY52:GPCChip # chip 40 + CCID58-1-05b2 STR XY53:GPCChip # chip 41 + CCID58-3-05a2 STR XY54:GPCChip # chip 42 + CCID58-3-03a2 STR XY55:GPCChip # chip 43 + CCID58-3-02a2 STR XY56:GPCChip # chip 44 + CCID58-3-11a2 STR XY57:GPCChip # chip 45 + CCID58-2-16b1 STR XY60:GPCChip # chip 46 + CCID58-1-03b1 STR XY61:GPCChip # chip 47 + CCID58-3-03b2 STR XY62:GPCChip # chip 48 + CCID58-1-02b2 STR XY63:GPCChip # chip 49 + CCID58-1-07a1 STR XY64:GPCChip # chip 50 + CCID58-2-04a1 STR XY65:GPCChip # chip 51 + CCID58-3-10a1 STR XY66:GPCChip # chip 52 + CCID58-1-09a1 STR XY67:GPCChip # chip 53 + CCID58-2-13b1 STR XY71:GPCChip # chip 54 + CCID58-1-14b1 STR XY72:GPCChip # chip 55 + CCID58-3-11b2 STR XY73:GPCChip # chip 56 + CCID58-1-05a2 STR XY74:GPCChip # chip 57 + CCID58-1-21a1 STR XY75:GPCChip # chip 58 + CCID58-2-05a1 STR XY76:GPCChip # chip 59 +END + +CHIPS METADATA + GPCChip METADATA + # Extension name, cellName:cellType + xy00 STR xy00:GPCCell + xy10 STR xy10:GPCCell + xy20 STR xy20:GPCCell + xy30 STR xy30:GPCCell + xy40 STR xy40:GPCCell + xy50 STR xy50:GPCCell + xy60 STR xy60:GPCCell + xy70 STR xy70:GPCCell + xy01 STR xy01:GPCCell + xy11 STR xy11:GPCCell + xy21 STR xy21:GPCCell + xy31 STR xy31:GPCCell + xy41 STR xy41:GPCCell + xy51 STR xy51:GPCCell + xy61 STR xy61:GPCCell + xy71 STR xy71:GPCCell + xy02 STR xy02:GPCCell + xy12 STR xy12:GPCCell + xy22 STR xy22:GPCCell + xy32 STR xy32:GPCCell + xy42 STR xy42:GPCCell + xy52 STR xy52:GPCCell + xy62 STR xy62:GPCCell + xy72 STR xy72:GPCCell + xy03 STR xy03:GPCCell + xy13 STR xy13:GPCCell + xy23 STR xy23:GPCCell + xy33 STR xy33:GPCCell + xy43 STR xy43:GPCCell + xy53 STR xy53:GPCCell + xy63 STR xy63:GPCCell + xy73 STR xy73:GPCCell + xy04 STR xy04:GPCCell + xy14 STR xy14:GPCCell + xy24 STR xy24:GPCCell + xy34 STR xy34:GPCCell + xy44 STR xy44:GPCCell + xy54 STR xy54:GPCCell + xy64 STR xy64:GPCCell + xy74 STR xy74:GPCCell + xy05 STR xy05:GPCCell + xy15 STR xy15:GPCCell + xy25 STR xy25:GPCCell + xy35 STR xy35:GPCCell + xy45 STR xy45:GPCCell + xy55 STR xy55:GPCCell + xy65 STR xy65:GPCCell + xy75 STR xy75:GPCCell + xy06 STR xy06:GPCCell + xy16 STR xy16:GPCCell + xy26 STR xy26:GPCCell + xy36 STR xy36:GPCCell + xy46 STR xy46:GPCCell + xy56 STR xy56:GPCCell + xy66 STR xy66:GPCCell + xy76 STR xy76:GPCCell + xy07 STR xy07:GPCCell + xy17 STR xy17:GPCCell + xy27 STR xy27:GPCCell + xy37 STR xy37:GPCCell + xy47 STR xy47:GPCCell + xy57 STR xy57:GPCCell + xy67 STR xy67:GPCCell + xy77 STR xy77:GPCCell + END +END + +# Specify the cell data +CELLS METADATA + GPCCell METADATA + CELL.TRIMSEC.SOURCE STR HEADER + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC.SOURCE STR HEADER + CELL.BIASSEC STR BIASSEC + END +END + + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + FPA.FILTERID STR FILTERID + FPA.FILTER STR FILTERID + FPA.RA STR RA + FPA.DEC STR DEC + FPA.RADECSYS STR EQUINOX + FPA.OBSTYPE STR OBSTYPE + FPA.OBJECT STR OBJECT + FPA.OBS.MODE STR OBS_MODE + FPA.COMMENT STR CMMTOBS + FPA.AIRMASS STR AIRMASS + FPA.POSANGLE STR POSANGLE + FPA.ROTANGLE STR ROT + FPA.FOCUS STR M2Z + FPA.TIME STR SHUTOUTC + FPA.ALT STR ALT + FPA.AZ STR AZ + # FPA.TEMP STR DETTEM + FPA.M1X STR M1X + FPA.M1Y STR M1Y + FPA.M1Z STR M1Z + FPA.M1TIP STR M1TIP + FPA.M1TILT STR M1TILT + FPA.M2X STR M2X + FPA.M2Y STR M2Y + FPA.M2Z STR M2Z + FPA.M2TIP STR M2TIP + FPA.M2TILT STR M2TILT + FPA.ENV.TEMP STR ENVTEM + FPA.ENV.HUMID STR ENVHUM + FPA.ENV.WIND STR ENVWIN + FPA.ENV.DIR STR ENVDIR + + FPA.TELTEMP.M1 STR TELTEMM1 # Primary mirror temps (C) + FPA.TELTEMP.M1CELL STR TELTEMMS # Primary mirror support temps (C) + FPA.TELTEMP.M2 STR TELTEMM2 # Secondary mirror temps (C + FPA.TELTEMP.SPIDER STR TELTEMSP # Spider temperatures (C) + FPA.TELTEMP.TRUSS STR TELTEMTR # Mid truss temperatures (C + FPA.TELTEMP.EXTRA STR TELTEMEX # Miscellaneous temperatures (C) + + FPA.BURNTOOL.APPLIED STR BTOOLAPP + CHIP.VIDEOCELL STR CELLMODE + + FPA.PON.TIME STR PONTIME + + CHIP.ID STR DETECTOR + CELL.XBIN STR CCDSUM + CELL.YBIN STR CCDSUM + CELL.X0 STR IMNPIX1 + CELL.Y0 STR IMNPIX2 + CELL.XPARITY STR ATM1_1 + CELL.YPARITY STR ATM2_2 + CHIP.TEMP STR CDETTEM + CHIP.TEMPERATURE STR DETTEM + FPA.EXPOSURE STR EXPREQ # Requested exposure time, presumably camera exposure time + CELL.EXPOSURE STR EXPTIME # Exposure time measured by shutter + CELL.DARKTIME STR DARKTIME # Exposure time for camera + CELL.TIME STR SHUTOUTC # Observation time + CELL.GAIN STR GAIN # Cell gain (e/ADU) + CELL.READNOISE STR RDNOISE # Cell read noise (ADU) + CELL.SATURATION STR MAXLIN # Saturation point (ADU) +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR PS1 + FPA.INSTRUMENT STR GPC1 + FPA.LONGITUDE STR 10:25:01.417 # West Longitude in Hours 10.4170608521 + FPA.LATITUDE STR 20:42:25.558 # North Latitude in Degrees 20.7070999146 + FPA.ELEVATION F32 3048.0 # altitude in meters + FPA.DETECTOR STR GPC1 + FPA.TIMESYS STR UTC + CHIP.XSIZE S32 4846 + CHIP.YSIZE S32 4868 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CHIP.X0.DEPEND STR CHIP.NAME + CHIP.X0 METADATA + XY01 S32 4971 + XY02 S32 4971 + XY03 S32 4971 + XY04 S32 4971 + XY05 S32 4971 + XY06 S32 4971 + XY10 S32 9942 + XY11 S32 9942 + XY12 S32 9942 + XY13 S32 9942 + XY14 S32 9942 + XY15 S32 9942 + XY16 S32 9942 + XY17 S32 9942 + XY20 S32 14913 + XY21 S32 14913 + XY22 S32 14913 + XY23 S32 14913 + XY24 S32 14913 + XY25 S32 14913 + XY26 S32 14913 + XY27 S32 14913 + XY30 S32 19884 + XY31 S32 19884 + XY32 S32 19884 + XY33 S32 19884 # should be 19872 + XY34 S32 19884 + XY35 S32 19884 + XY36 S32 19884 + XY37 S32 19884 + XY40 S32 20041 + XY41 S32 20041 + XY42 S32 20041 + XY43 S32 20041 # should be 20029 + XY44 S32 20041 + XY45 S32 20041 + XY46 S32 20041 + XY47 S32 20041 + XY50 S32 25012 + XY51 S32 25012 + XY52 S32 25012 + XY53 S32 25012 + XY54 S32 25012 + XY55 S32 25012 + XY56 S32 25012 + XY57 S32 25012 + XY60 S32 29983 + XY61 S32 29983 + XY62 S32 29983 + XY63 S32 29983 + XY64 S32 29983 + XY65 S32 29983 + XY66 S32 29983 + XY67 S32 29983 + XY71 S32 34954 + XY72 S32 34954 + XY73 S32 34954 + XY74 S32 34954 + XY75 S32 34954 + XY76 S32 34954 + END + CHIP.Y0.DEPEND STR CHIP.NAME + CHIP.Y0 METADATA + XY01 S32 10286 + XY02 S32 15429 + XY03 S32 20572 + XY04 S32 25715 + XY05 S32 30858 + XY06 S32 36001 + XY10 S32 5143 + XY11 S32 10286 + XY12 S32 15429 + XY13 S32 20572 + XY14 S32 25715 + XY15 S32 30858 + XY16 S32 36001 + XY17 S32 41144 + XY20 S32 5143 + XY21 S32 10286 + XY22 S32 15429 + XY23 S32 20572 + XY24 S32 25715 + XY25 S32 30858 + XY26 S32 36001 + XY27 S32 41144 + XY30 S32 5143 + XY31 S32 10286 + XY32 S32 15429 + XY33 S32 20572 + XY34 S32 25715 # should be 25755 + XY35 S32 30858 + XY36 S32 36001 + XY37 S32 41144 + XY40 S32 306 + XY41 S32 5449 + XY42 S32 10592 + XY43 S32 15735 # should be 15707 + XY44 S32 20878 # should be 20890 + XY45 S32 26021 + XY46 S32 31164 + XY47 S32 36307 + XY50 S32 306 + XY51 S32 5449 + XY52 S32 10592 + XY53 S32 15735 + XY54 S32 20878 + XY55 S32 26021 + XY56 S32 31164 + XY57 S32 36307 + XY60 S32 306 + XY61 S32 5449 + XY62 S32 10592 + XY63 S32 15735 + XY64 S32 20878 + XY65 S32 26021 + XY66 S32 31164 + XY67 S32 36307 + XY71 S32 5449 + XY72 S32 10592 + XY73 S32 15735 + XY74 S32 20878 + XY75 S32 26021 + XY76 S32 31164 + END + CHIP.XPARITY.DEPEND STR CHIP.NAME + CHIP.XPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CHIP.YPARITY.DEPEND STR CHIP.NAME + CHIP.YPARITY METADATA + XY01 S32 -1 + XY02 S32 -1 + XY03 S32 -1 + XY04 S32 -1 + XY05 S32 -1 + XY06 S32 -1 + XY10 S32 -1 + XY11 S32 -1 + XY12 S32 -1 + XY13 S32 -1 + XY14 S32 -1 + XY15 S32 -1 + XY16 S32 -1 + XY17 S32 -1 + XY20 S32 -1 + XY21 S32 -1 + XY22 S32 -1 + XY23 S32 -1 + XY24 S32 -1 + XY25 S32 -1 + XY26 S32 -1 + XY27 S32 -1 + XY30 S32 -1 + XY31 S32 -1 + XY32 S32 -1 + XY33 S32 -1 + XY34 S32 -1 + XY35 S32 -1 + XY36 S32 -1 + XY37 S32 -1 + XY40 S32 1 + XY41 S32 1 + XY42 S32 1 + XY43 S32 1 + XY44 S32 1 + XY45 S32 1 + XY46 S32 1 + XY47 S32 1 + XY50 S32 1 + XY51 S32 1 + XY52 S32 1 + XY53 S32 1 + XY54 S32 1 + XY55 S32 1 + XY56 S32 1 + XY57 S32 1 + XY60 S32 1 + XY61 S32 1 + XY62 S32 1 + XY63 S32 1 + XY64 S32 1 + XY65 S32 1 + XY66 S32 1 + XY67 S32 1 + XY71 S32 1 + XY72 S32 1 + XY73 S32 1 + XY74 S32 1 + XY75 S32 1 + XY76 S32 1 + END + CELL.READDIR S32 1 + CELL.BAD S32 0 + CELL.XSIZE S32 590 + CELL.YSIZE S32 598 + CELL.TIMESYS STR UTC +END + +# How to translation PS concepts into database lookups +DATABASE METADATA + # this rule is fragile : does not match the camera + FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +END + + +# Where there might be some ambiguity, specify the format +FORMATS METADATA + FPA.RA STR DEGREES + FPA.DEC STR DEGREES + FPA.LONGITUDE STR HOURS + FPA.LATITUDE STR DEGREES + FPA.TIME STR YEAR.FIRST + CELL.TIME STR YEAR.FIRST + CELL.BINNING STR TOGETHER + # CELL.X0 STR FORTRAN + # CELL.Y0 STR FORTRAN + CELL.READNOISE STR ADU +END + +# Pattern correction information: + +PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 BOOL FALSE + XY10 BOOL FALSE + XY11 BOOL 0011100100000000000000000000000000000000000000000000000000000000 + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 BOOL FALSE + XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 + XY16 BOOL FALSE + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 BOOL FALSE + XY25 BOOL FALSE + XY26 BOOL FALSE + XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 + XY33 BOOL FALSE + XY34 BOOL FALSE + XY35 BOOL FALSE + XY36 BOOL FALSE + XY37 BOOL FALSE + XY40 BOOL FALSE + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 BOOL FALSE + XY44 BOOL FALSE + XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 + XY46 BOOL FALSE + XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 + XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 + XY61 BOOL FALSE + XY62 BOOL FALSE + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 BOOL FALSE + XY73 BOOL FALSE + XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 + XY75 BOOL FALSE + XY76 BOOL FALSE +END + +PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction + XY01 BOOL FALSE + XY02 BOOL FALSE + XY03 BOOL FALSE + XY04 BOOL FALSE + XY05 BOOL FALSE + XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 + XY10 BOOL FALSE + XY11 BOOL FALSE + XY12 BOOL FALSE + XY13 BOOL FALSE + XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 + XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 + XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 + XY17 BOOL FALSE + XY20 BOOL FALSE + XY21 BOOL FALSE + XY22 BOOL FALSE + XY23 BOOL FALSE + XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 + XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 + XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 + XY27 BOOL FALSE + XY30 BOOL FALSE + XY31 BOOL FALSE + XY32 BOOL FALSE + XY33 BOOL FALSE + XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 + XY35 BOOL FALSE + XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 + XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 + XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 + XY41 BOOL FALSE + XY42 BOOL FALSE + XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 + XY44 BOOL FALSE + XY45 BOOL FALSE + XY46 BOOL FALSE + XY47 BOOL FALSE + XY50 BOOL FALSE + XY51 BOOL FALSE + XY52 BOOL FALSE + XY53 BOOL FALSE + XY54 BOOL FALSE + XY55 BOOL FALSE + XY56 BOOL FALSE + XY57 BOOL FALSE + XY60 BOOL FALSE + XY61 BOOL FALSE + XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 + XY63 BOOL FALSE + XY64 BOOL FALSE + XY65 BOOL FALSE + XY66 BOOL FALSE + XY67 BOOL FALSE + XY71 BOOL FALSE + XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 + XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 + XY74 BOOL FALSE + XY75 BOOL FALSE + XY76 BOOL FALSE +END Index: /branches/eam_branches/ipp-20110213/ippconfig/recipes/nightly_science.config =================================================================== --- /branches/eam_branches/ipp-20110213/ippconfig/recipes/nightly_science.config (revision 30811) +++ /branches/eam_branches/ipp-20110213/ippconfig/recipes/nightly_science.config (revision 30812) @@ -2,15 +2,15 @@ CLEAN_MODES METADATA MODE STR CHIP - COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ + COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked RETENTION_TIME U16 21 END CLEAN_MODES METADATA MODE STR WARP - COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ + COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked RETENTION_TIME U16 7 END CLEAN_MODES METADATA MODE STR DIFF - COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ + COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked RETENTION_TIME S16 21 END Index: /branches/eam_branches/ipp-20110213/pstamp/scripts/pstamp_checkdependent.pl =================================================================== --- /branches/eam_branches/ipp-20110213/pstamp/scripts/pstamp_checkdependent.pl (revision 30811) +++ /branches/eam_branches/ipp-20110213/pstamp/scripts/pstamp_checkdependent.pl (revision 30812) @@ -27,5 +27,5 @@ my $IPP_DIFF_MODE_STACK_STACK = 4; -my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count); +my ($dep_id, $stage, $stage_id, $component, $imagedb, $rlabel, $need_magic, $fault_count, $max_fault_count, $logfile); my ($dbname, $ps_dbserver, $verbose, $save_temps, $no_update); @@ -40,4 +40,5 @@ 'fault_count=i' => \$fault_count, 'max_fault_count=i' => \$max_fault_count, + 'logfile=s' => \$logfile, 'dbname=s' => \$dbname, # postage stamp server dbname 'dbserver=s' => \$ps_dbserver, # postage stamp server dbserver @@ -68,4 +69,8 @@ my $ipprc = PS::IPP::Config->new(); +if ($logfile) { + $ipprc->redirect_output($logfile); +} + if (!$ps_dbserver) { $ps_dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); @@ -314,4 +319,10 @@ my $data_state = $metadata->{data_state}; + # The update system currently requires that data that has been magicked be destreaked + # at chip stage so it needs magic even if the customer doesn't. + if ($metadata->{magicked} != 0) { + $need_magic = 1; + } + if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop') or ($data_state =~ /error/) or ($data_state =~ /purged/) or ($data_state =~ /scrubbed/) or ($data_state eq 'drop')) { @@ -350,5 +361,17 @@ foreach my $chip (@$data) { $chip_id = $chip->{chip_id}; - if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) { + + # if chip has been magicked before require it to be magicked again + # because the warp pending query requires it. + if ($chip->{magicked} < 0) { + print "Input has been destreaked so we must destreak before warping\n"; + $need_magic = 1 + } + + if ($need_magic and ($chip->{magicked} eq 0)) { + my_die("Client requires magic, but chip never magicked. How did this dependent get queued?", $PS_EXIT_PROG_ERROR); + } + + if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} < 0))) { $chips_ready = 0; $chip->{fault} = $chip->{chip_fault}; @@ -399,4 +422,10 @@ my $skycell = $metadata; my $skycell_id = $skycell->{skycell_id}; + + # The update system currently requires that data that has been magicked be destreaked + # at chip stage so it needs magic even if the customer doesn't. + if ($metadata->{magicked} != 0) { + $need_magic = 1; + } if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) { Index: /branches/eam_branches/ipp-20110213/pstamp/scripts/pstampparse.pl =================================================================== --- /branches/eam_branches/ipp-20110213/pstamp/scripts/pstampparse.pl (revision 30811) +++ /branches/eam_branches/ipp-20110213/pstamp/scripts/pstampparse.pl (revision 30812) @@ -31,4 +31,7 @@ my $no_update; my $dest_requires_magic; + +# set this to true to disable update processing +my $no_updates_allowed = 0; GetOptions( @@ -424,4 +427,12 @@ # and this user's data store destination is allowed uncensored stamps, so accept the request $need_magic = 0; + + # Since user can get unmagicked data "by coordinate" requests can go back in time + # to dredge unusable data from the "dark days"... + if ($req_type eq 'bycoord' and $mjd_min eq 0) { + # ... so unless the user sets mjd_min clamp it to 2009-04-01 + # XXX: This value should live in the pstampProject table not be hardcoded here + $mjd_min = 54922; + } } else { print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n"; @@ -465,10 +476,11 @@ print "Time to locate_images for row $rownum $dtime_locate\n"; - # handle this XXX: what did I mean by this comment + # handle this + # XXX: what did I mean by that comment? $row->{need_magic} = $need_magic; $num_jobs += queueJobs($mode, \@rowList, $imageList); - # if a row slipped through with no jobs add one + # if a row slipped through with no jobs add a faulted one foreach my $row (@rowList) { if ($row->{job_num} == 0) { @@ -495,4 +507,6 @@ my $option_mask = $row->{OPTION_MASK}; my $components = $row->{components}; + + $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed; my $roi_string; @@ -668,4 +682,6 @@ my $rownum = $row->{ROWNUM}; my $option_mask = $row->{OPTION_MASK}; + + $option_mask |= $PSTAMP_NO_WAIT_FOR_UPDATE if $no_updates_allowed; # For dist_bundle we need Index: /branches/eam_branches/ipp-20110213/pstamp/scripts/pstampstopfaulted =================================================================== --- /branches/eam_branches/ipp-20110213/pstamp/scripts/pstampstopfaulted (revision 30811) +++ /branches/eam_branches/ipp-20110213/pstamp/scripts/pstampstopfaulted (revision 30812) @@ -1,15 +1,20 @@ +#!/bin/bash +# +# script executed by the task pstamp.stopfaulted to stop proceessing of jobs +# and dependents that have fault_count >= some maximum value. That value is expected +# to be passed in as an argument otherwise the default in pstamptool is used +# All arguments are passed to pstamptool -# script executed by the task pstamp.stopfaulted to stop proceessing of jobs -# and dependents that have faulted 5 or more times -# all arguments to this script are passed to pstamptool - -# 25 is PSTAMP_NOT_AVAILABLE +# note: 25 is PSTAMP_NOT_AVAILABLE fault_code=25 -fault_count=3 date echo stopping faulted dependent jobs -pstamptool -stopdependentjob -set_fault $fault_code -fault_count $fault_count $* +cmd="pstamptool -stopdependentjob -set_fault $fault_code $*" +echo $cmd +$cmd echo stopping faulted jobs -pstamptool -updatejob -state run -set_state stop -set_fault $fault_code -fault_count $fault_count $* +cmd="pstamptool -updatejob -state run -set_state stop -set_fault $fault_code $*" +echo $cmd +$cmd Index: /branches/eam_branches/ipp-20110213/tools/czartool/Config.pm =================================================================== --- /branches/eam_branches/ipp-20110213/tools/czartool/Config.pm (revision 30811) +++ /branches/eam_branches/ipp-20110213/tools/czartool/Config.pm (revision 30812) @@ -66,6 +66,5 @@ $self->{roboczaremail} = $xc->findvalue('//roboczar/email'); $self->{roboczarserverinterval} = $xc->findvalue('//roboczar/serverinterval'); - - + $self->{roboczarinterestedservers} = $xc->findvalue('//roboczar/interestedservers'); } @@ -126,4 +125,17 @@ ########################################################################### # +# Returns array of servers we are interested in for roboczar +# +########################################################################### +sub getRoboczarInterestedServers { + my ($self) = @_; + + my @servers = split('\s+', $self->{roboczarinterestedservers}); + + return @servers; +} + +########################################################################### +# # Returns the email to which roboczar should send its warnings # @@ -136,5 +148,5 @@ ########################################################################### # -# Returns +# Returns font for gnuplot # ########################################################################### @@ -146,5 +158,5 @@ ########################################################################### # -# Returns +# Returns font size for gnuplot # ########################################################################### @@ -156,5 +168,5 @@ ########################################################################### # -# Returns +# Returns save location for metrics # ########################################################################### @@ -166,5 +178,5 @@ ########################################################################### # -# Returns +# Returns daily start time for metrics # ########################################################################### @@ -176,5 +188,5 @@ ########################################################################### # -# Returns +# Returns name for gpc1 Db # ########################################################################### @@ -186,5 +198,5 @@ ########################################################################### # -# Returns +# Returns host for gpc1 Db # ########################################################################### @@ -196,5 +208,5 @@ ########################################################################### # -# Returns +# Returns user for gpc1 Db # ########################################################################### @@ -206,5 +218,5 @@ ########################################################################### # -# Returns +# Returns password for gpc1 Db # ########################################################################### @@ -216,5 +228,5 @@ ########################################################################### # -# Returns +# Returns czar Db name # ########################################################################### @@ -226,5 +238,5 @@ ########################################################################### # -# Returns +# Returns host for czar Db # ########################################################################### @@ -236,5 +248,5 @@ ########################################################################### # -# Returns +# Returns username for czar Db # ########################################################################### @@ -246,5 +258,5 @@ ########################################################################### # -# Returns +# Returns password for czar Db # ########################################################################### Index: /branches/eam_branches/ipp-20110213/tools/czartool/czarconfig.xml =================================================================== --- /branches/eam_branches/ipp-20110213/tools/czartool/czarconfig.xml (revision 30811) +++ /branches/eam_branches/ipp-20110213/tools/czartool/czarconfig.xml (revision 30812) @@ -1,6 +1,9 @@ + + + /usr/share/fonts/corefonts/arial.ttf @@ -8,4 +11,5 @@ + /data/ipp004.0/ipp/ippMetrics @@ -13,4 +17,5 @@ + gpc1 @@ -20,4 +25,5 @@ + czardb @@ -28,7 +34,13 @@ + ps-ipp-ops@ifa.hawaii.edu + 20 MINUTE + + + pstamp + Index: /branches/eam_branches/ipp-20110213/tools/roboczar.pl =================================================================== --- /branches/eam_branches/ipp-20110213/tools/roboczar.pl (revision 30811) +++ /branches/eam_branches/ipp-20110213/tools/roboczar.pl (revision 30812) @@ -22,6 +22,6 @@ my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO get from Pantasks -my @serversWeCareAbout = ("stdscience", "distribution", "summitcopy", "registration", "pstamp"); +my @interestedServers = $config->getRoboczarInterestedServers(); my $stuckMessage; @@ -115,5 +115,5 @@ ${$message} = "Pantasks servers:\n"; - foreach $server (@serversWeCareAbout) { + foreach $server (@interestedServers) { # is server alice?