Changeset 38441 for trunk/Ohana/src/addstar

Timestamp:

Jun 12, 2015, 6:18:23 PM (11 years ago)

Author:

eugene

Message:

merge changes from EAM dev branch ohana.20150429

Location:

trunk/Ohana

Files:

• . (modified) (1 prop)
• src/addstar/include/addstar.h (modified) (2 diffs)
• src/addstar/src/ConfigInit.c (modified) (1 diff)
• src/addstar/src/ReadStarsFITS.c (modified) (11 diffs)
• src/addstar/src/SEDfit.c (modified) (1 diff)
• src/addstar/src/UpdateDatabase_Image.c (modified) (5 diffs)
• src/addstar/src/UpdateDatabase_Refcat.c (modified) (3 diffs)
• src/addstar/src/UpdateDatabase_Reflist.c (modified) (3 diffs)
• src/addstar/src/UpdateImageIDs.c (modified) (1 diff)
• src/addstar/src/addstar.c (modified) (5 diffs)
• src/addstar/src/addstar_create_ID.c (modified) (1 diff)
• src/addstar/src/args.c (modified) (1 diff)
• src/addstar/src/args_parallel_client.c (modified) (1 diff)
• src/addstar/src/dumpskycells.c (modified) (1 diff)
• src/addstar/src/find_matches.c (modified) (2 diffs)
• src/addstar/src/find_matches_ICRF.c (modified) (1 diff)
• src/addstar/src/find_matches_closest.c (modified) (2 diffs)
• src/addstar/src/find_matches_closest_refstars.c (modified) (2 diffs)
• src/addstar/src/find_matches_refstars.c (modified) (2 diffs)
• src/addstar/src/find_matches_starpar.c (modified) (1 diff)
• src/addstar/src/findskycell.c (modified) (4 diffs)
• src/addstar/src/load2mass_as_rawdata.c (modified) (1 diff)
• src/addstar/src/load2mass_catalog.c (modified) (1 diff)
• src/addstar/src/loadICRF_catalog.c (modified) (1 diff)
• src/addstar/src/loadICRF_save_remote.c (modified) (2 diffs)
• src/addstar/src/loadICRF_table.c (modified) (1 diff)
• src/addstar/src/loadstarpar_catalog.c (modified) (1 diff)
• src/addstar/src/loadstarpar_save_remote.c (modified) (3 diffs)
• src/addstar/src/loadstarpar_table.c (modified) (1 diff)
• src/addstar/src/loadsupercos.c (modified) (1 diff)
• src/addstar/src/loadsupercos_rawdata.c (modified) (1 diff)
• src/addstar/src/loadwise_prelim_rawdata.c (modified) (1 diff)
• src/addstar/src/loadwise_rawdata.c (modified) (1 diff)
• src/addstar/src/mkcmf.c (modified) (6 diffs)
• src/addstar/src/resort_catalogs.c (modified) (1 diff)
• src/addstar/src/resort_threaded.c (modified) (2 diffs)
• src/addstar/src/resort_unthreaded.c (modified) (2 diffs)
• src/addstar/src/sedstar.c (modified) (3 diffs)
• src/addstar/src/skycells.c (modified) (1 diff)
• src/addstar/src/skycells_to_mdc.c (modified) (1 diff)
• src/addstar/test (modified) (2 props)
• src/addstar/test/GSCregions.tbl (copied) (copied from branches/eam_branches/ohana.20150429/src/addstar/test/GSCregions.tbl )
• src/addstar/test/compress.dvo (copied) (copied from branches/eam_branches/ohana.20150429/src/addstar/test/compress.dvo )
• src/addstar/test/dvo.config (copied) (copied from branches/eam_branches/ohana.20150429/src/addstar/test/dvo.config )
• src/addstar/test/dvo.layout (copied) (copied from branches/eam_branches/ohana.20150429/src/addstar/test/dvo.layout )
• src/addstar/test/relphot.dvo (modified) (2 diffs)
• src/addstar/test/relphot.flatcorr.dvo (modified) (19 diffs)
• src/addstar/test/simple.dvo (modified) (15 diffs)
• src/addstar/test/tap.dvo (modified) (2 diffs)

trunk/Ohana/src/addstar/include/addstar.h

@@ -84 +84 @@
 char   GSCFILE[256];
 char  *CATDIR;
-char   CATMODE[16];    /* raw, mef, split, mysql */
-char   CATFORMAT[16];  /* internal, elixir, loneos, panstarrs */
+char   CATMODE[16];     /* raw, mef, split, mysql */
+char   CATFORMAT[16];   /* internal, elixir, loneos, panstarrs */
+char   CATCOMPRESS[16]; /* GZIP_1, NONE */
 char   TWO_MASS_DIR_AS[256];
 char   TWO_MASS_DIR_DR2[256];
@@ -326 +327 @@
 void *memrchr(const void *s, int c, size_t n);
 int addstar_create_ID ();
-int strextend (char *input, char *format,...);
 
 void initMosaicCoords ();

trunk/Ohana/src/addstar/src/ConfigInit.c

@@ -134 +134 @@
   ScanConfig (config, "CATMODE",                "%s",  0, CATMODE);
   ScanConfig (config, "CATFORMAT",              "%s",  0, CATFORMAT);
+  if (!ScanConfig (config, "CATCOMPRESS",       "%s",  0, CATCOMPRESS)) {
+    strcpy (CATCOMPRESS, "NONE");
+  }
   if (!ScanConfig (config, "SKY_DEPTH",         "%d",  0, &SKY_DEPTH)) {
     SKY_DEPTH = SKY_DEPTH_HST;

trunk/Ohana/src/addstar/src/ReadStarsFITS.c

@@ -153 +153 @@
   off_t Nstars;
   unsigned int i;
-  char swapped;
   double ZeroPt;
   Stars *stars = NULL;
   SMPData *smpdata = NULL;
 
-  swapped = FALSE;
-  smpdata = gfits_table_get_SMPData (table, &Nstars, &swapped);
+  smpdata = gfits_table_get_SMPData (table, &Nstars, NULL, NULL);
   if (!smpdata) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -229 +227 @@
   PS1_DEV_0 *ps1data;
 
-  ps1data = gfits_table_get_PS1_DEV_0 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_PS1_DEV_0 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -300 +298 @@
   PS1_DEV_1 *ps1data;
 
-  ps1data = gfits_table_get_PS1_DEV_1 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_PS1_DEV_1 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -383 +381 @@
   }
 
-  ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -545 +543 @@
   CMF_PS1_V2 *ps1data;
 
-  ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -622 +620 @@
   CMF_PS1_V3 *ps1data;
 
-  ps1data = gfits_table_get_CMF_PS1_V3 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_V3 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -706 +704 @@
   CMF_PS1_V4 *ps1data;
 
-  ps1data = gfits_table_get_CMF_PS1_V4 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_V4 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -1259 +1257 @@
   }
 
-  ps1data = gfits_table_get_CMF_PS1_SV1 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_SV1 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -1423 +1421 @@
   CMF_PS1_SV2 *ps1data;
 
-  ps1data = gfits_table_get_CMF_PS1_SV2 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_SV2 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -1753 +1751 @@
   CMF_PS1_DV4 *ps1data;
 
-  ps1data = gfits_table_get_CMF_PS1_DV4 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_DV4 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");
@@ -1839 +1837 @@
   CMF_PS1_DV5 *ps1data;
 
-  ps1data = gfits_table_get_CMF_PS1_DV5 (table, &Nstars, NULL);
+  ps1data = gfits_table_get_CMF_PS1_DV5 (table, &Nstars, NULL, NULL);
   if (!ps1data) {
     fprintf (stderr, "skipping inconsistent entry\n");

trunk/Ohana/src/addstar/src/SEDfit.c

@@ -198 +198 @@
   outcat[0].Naverage  = Nave;
   outcat[0].Nmeasure  = Nmeas;
-  outcat[0].Nsecf_mem = Nave*Nsec;
+  outcat[0].Nsecfilt_mem = Nave*Nsec;
   
   free (sourceValue.mags);

trunk/Ohana/src/addstar/src/UpdateDatabase_Image.c

@@ -12 +12 @@
   }
 
-  catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
-  if (options[0].update) catalog.catflags = LOAD_AVES | LOAD_MISS | LOAD_SECF;
+  catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
+  if (options[0].update) catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
   
   // XXX this is probably not needed anymore
@@ -47 +47 @@
     catalog.filename  = skylist[0].filename[i];
     catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
-    if (options[0].update) catalog.catflags = LOAD_AVES | LOAD_MISS | LOAD_SECF;
+    catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
+    if (options[0].update) catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
 
     // an error exit status here is a significant error
@@ -56 +56 @@
     }
 
-    // Naves_disk == 0 implies an empty catalog file
+    // Naverage_disk == 0 implies an empty catalog file
     // for only_match, skip empty catalogs 
-    if ((catalog.Naves_disk == 0) && options[0].only_match) {
+    if ((catalog.Naverage_disk == 0) && options[0].only_match) {
       dvo_catalog_unlock (&catalog);
       dvo_catalog_free (&catalog);
@@ -73 +73 @@
       SetProtect (TRUE);
       if (options[0].update) {
-        catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         dvo_catalog_update (&catalog, VERBOSE);
       } else {
@@ -92 +92 @@
     /*** update the image table ***/
     /* setup image table format and lock */
+    gfits_db_init (&db);
     db.mode   = dvo_catalog_catmode (CATMODE);
     db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/UpdateDatabase_Refcat.c

@@ -36 +36 @@
     catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
     catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
-    if (options[0].update) catalog.catflags = LOAD_AVES | LOAD_MISS | LOAD_SECF;
+    catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
+    if (options[0].update) catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
   
     // an error exit status here is a significant error
@@ -45 +45 @@
     }
 
-    // Naves_disk == 0 implies an empty catalog file
+    // Naverage_disk == 0 implies an empty catalog file
     // for only_match, skip empty catalogs
-    if ((catalog.Naves_disk == 0) && options[0].only_match) {
+    if ((catalog.Naverage_disk == 0) && options[0].only_match) {
       dvo_catalog_unlock (&catalog);
       dvo_catalog_free (&catalog);
@@ -63 +63 @@
       SetProtect (TRUE);
       if (options[0].update) {
-        catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         dvo_catalog_update (&catalog, VERBOSE);
       } else {

trunk/Ohana/src/addstar/src/UpdateDatabase_Reflist.c

@@ -36 +36 @@
     catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
     catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
-    if (options[0].update) catalog.catflags = LOAD_AVES | LOAD_MISS | LOAD_SECF;
+    catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
+    if (options[0].update) catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
 
     // an error exit status here is a significant error
@@ -46 +46 @@
 
     /* for only_match, skip empty catalogs XXX EAM : this leaves behind empty files */
-    if ((catalog.Naves_disk == 0) && options[0].only_match) {
+    if ((catalog.Naverage_disk == 0) && options[0].only_match) {
       dvo_catalog_unlock (&catalog);
       dvo_catalog_free (&catalog);
@@ -59 +59 @@
       SetProtect (TRUE);
       if (options[0].update) {
-        catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         dvo_catalog_update (&catalog, VERBOSE);
       } else {

trunk/Ohana/src/addstar/src/UpdateImageIDs.c

@@ -9 +9 @@
   /*** update the image table ***/
   /* setup image table format and lock */
+  gfits_db_init (&db);
   db.mode   = dvo_catalog_catmode (CATMODE);
   db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/addstar.c

@@ -104 +104 @@
 
     // set the parameters which guide catalog open/load/create
-    catalog.filename  = skylist[0].filename[i];
-    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF | LOAD_LENSING;
-    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+    catalog.filename    = skylist[0].filename[i];
+    catalog.catmode     = dvo_catalog_catmode (CATMODE);         // set the default catmode from config data
+    catalog.catformat   = dvo_catalog_catformat (CATFORMAT);     // set the default catformat from config data
+    catalog.catcompress = dvo_catalog_catcompress (CATCOMPRESS); // set the default catcompress from config data
+    catalog.catflags    = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING;
+    catalog.Nsecfilt    = GetPhotcodeNsecfilt ();
 
     // XXX need to do something to enforce consistency.  we can change options.update
@@ -117 +118 @@
     if (catalog.catmode != DVO_MODE_SPLIT) options.update = FALSE;
 
-    if (options.update) catalog.catflags = LOAD_AVES | LOAD_MISS | LOAD_SECF;
+    if (options.update) catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
 
     // open as read or write, depending on desire
@@ -134 +135 @@
     if (VERBOSE) MARKTIME ("load cpt: %f sec\n", dtime); RESETTIME; 
 
-    // Naves_disk == 0 implies an empty catalog file
+    // Naverage_disk == 0 implies an empty catalog file
     // for only_match, skip empty catalogs
-    if ((catalog.Naves_disk == 0) && options.only_match) {
+    if ((catalog.Naverage_disk == 0) && options.only_match) {
       dvo_catalog_unlock (&catalog);
       dvo_catalog_free (&catalog);
@@ -174 +175 @@
       SetProtect (TRUE);
       if (options.update) {
-        catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF | LOAD_LENSING;
+        catalog.catflags  = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT | DVO_LOAD_LENSING;
         dvo_catalog_update (&catalog, VERBOSE);
       } else {
@@ -193 +194 @@
   /*** update the image table ***/
   /* setup image table format and lock */
+  gfits_db_init (&db);
   db.mode   = dvo_catalog_catmode (CATMODE);
   db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/addstar_create_ID.c

@@ -10 +10 @@
   /*** update the image table ***/
   /* setup image table format and lock */
-  // gfits_db_init (&db);
+  gfits_db_init (&db);
   db.mode   = dvo_catalog_catmode (CATMODE);
   db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/args.c

@@ -480 +480 @@
 }
 
-int strextend (char *input, char *format,...) {
-
-  char tmpextra[1024], tmpline[1024];
-  va_list argp;
-
-  va_start (argp, format);
-  vsnprintf (tmpextra, 1024, format, argp);
-  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
-  strcpy (input, tmpline);
-
-  return TRUE;
-}
-
 /** addstar modes:
  

trunk/Ohana/src/addstar/src/args_parallel_client.c

@@ -125 +125 @@
 }
 
-int strextend (char *input, char *format,...) {
-
-  char tmpextra[1024], tmpline[1024];
-  va_list argp;
-
-  va_start (argp, format);
-  vsnprintf (tmpextra, 1024, format, argp);
-  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
-  strcpy (input, tmpline);
-
-  return TRUE;
-}
-

trunk/Ohana/src/addstar/src/dumpskycells.c

@@ -46 +46 @@
   /*** open the image table ***/
   /* setup image table format and lock */
+  gfits_db_init (&db);
   db.mode   = dvo_catalog_catmode (CATMODE);
   db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/find_matches.c

@@ -91 +91 @@
 
   /* set up pointers for linked list of measure */
-  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeas_disk)) {
+  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeasure_disk)) {
     next_meas = init_measure_links (catalog[0].average, Nave, catalog[0].measure, Nmeas);
   } else {
@@ -320 +320 @@
   catalog[0].Naverage = Nave;
   catalog[0].Nmeasure = Nmeas;
-  catalog[0].Nsecf_mem = Nave*Nsecfilt;
+  catalog[0].Nsecfilt_mem = Nave*Nsecfilt;
   if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n",  Nstars,  Nave,  Nmeas,  Nmatch);
 

trunk/Ohana/src/addstar/src/find_matches_ICRF.c

@@ -164 +164 @@
   catalog[0].Naverage  = Nave;
   catalog[0].Nmeasure  = Nmeas;
-  catalog[0].Nsecf_mem = Nave*Nsecfilt;
+  catalog[0].Nsecfilt_mem = Nave*Nsecfilt;
   if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %d "OFF_T_FMT" "OFF_T_FMT" ("OFF_T_FMT" matches)\n",  Nstars, Nave, Nmeas, Nmatch);
 

trunk/Ohana/src/addstar/src/find_matches_closest.c

@@ -91 +91 @@
 
   /* set up pointers for linked list of measure */
-  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeas_disk)) {
+  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeasure_disk)) {
     // this version is only valid if we have done a full catalog load, and if the catalog
     // is sorted while processed
@@ -381 +381 @@
   catalog[0].Nmeasure = Nmeas;
   catalog[0].Nlensing = Nlens;
-  catalog[0].Nsecf_mem = Nave*Nsecfilt;
+  catalog[0].Nsecfilt_mem = Nave*Nsecfilt;
   if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas, Nlens: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n",  Nstars,  Nave,  Nmeas,  Nlens, Nmatch);
 

trunk/Ohana/src/addstar/src/find_matches_closest_refstars.c

@@ -85 +85 @@
 
   /* set up pointers for linked list of measure */
-  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeas_disk)) {
+  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeasure_disk)) {
     // this version is only valid if we have done a full catalog load, and if the catalog
     // is sorted while processed
@@ -322 +322 @@
   catalog[0].Naverage = Nave;
   catalog[0].Nmeasure = Nmeas;
-  catalog[0].Nsecf_mem = Nave*Nsecfilt;
+  catalog[0].Nsecfilt_mem = Nave*Nsecfilt;
   if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: "OFF_T_FMT" "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n",  Nstars,  Nave,  Nmeas,  Nmatch);
 

trunk/Ohana/src/addstar/src/find_matches_refstars.c

@@ -70 +70 @@
 
   /* set up pointers for linked list of measurements */
-  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeas_disk)) {
+  if (catalog[0].sorted && (catalog[0].Nmeasure == catalog[0].Nmeasure_disk)) {
     // this version is only valid if we have done a full catalog load, and if the catalog
     // is sorted while processed
@@ -298 +298 @@
   catalog[0].Naverage = Nave;
   catalog[0].Nmeasure = Nmeas;
-  catalog[0].Nsecf_mem = Nave*Nsecfilt;
+  catalog[0].Nsecfilt_mem = Nave*Nsecfilt;
   if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nmeas: %d "OFF_T_FMT" "OFF_T_FMT", ("OFF_T_FMT" matches)\n",  Nstars,  Nave,  Nmeas,  Nmatch);
 

trunk/Ohana/src/addstar/src/find_matches_starpar.c

@@ -205 +205 @@
   catalog[0].Naverage  = Nave;
   catalog[0].Nstarpar  = Nstarpar;
-  catalog[0].Nsecf_mem = Nave*Nsecfilt;
+  catalog[0].Nsecfilt_mem = Nave*Nsecfilt;
   if (VERBOSE) fprintf (stderr, "Nstars, Nave, Nstarpar: %d "OFF_T_FMT" "OFF_T_FMT" ("OFF_T_FMT" matches)\n",  Nstars, Nave, Nstarpar, Nmatch);
 

trunk/Ohana/src/addstar/src/findskycell.c

@@ -173 +173 @@
   snprintf (imagefile, DVO_MAX_PATH, "%s/Images.dat", catdir);
 
+  gfits_db_init (&db);
   status = dvo_image_lock (&db, imagefile, 2.0, LCK_XCLD);
   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
@@ -183 +184 @@
   // convert database table to internal structure (binary to Image)
   // 'image' points to the same memory as db->ftable->buffer
-  image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped);
+  image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder);
   if (!image) {
       fprintf (stderr, "ERROR: failed to read images\n");
@@ -557 +558 @@
   snprintf (imagefile, DVO_MAX_PATH, "%s/Images.dat", catdir);
 
+  gfits_db_init (&db);
   status = dvo_image_lock (&db, imagefile, 2.0, LCK_XCLD);
   if (!status) Shutdown ("ERROR: failure to lock image catalog %s", db.filename);
@@ -567 +569 @@
   // convert database table to internal structure (binary to Image)
   // 'image' points to the same memory as db->ftable->buffer
-  image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped);
+  image = gfits_table_get_Image (&db.ftable, &Nimage, &db.scaledValue, &db.nativeOrder);
   if (!image) {
     fprintf (stderr, "ERROR: failed to read images\n");

trunk/Ohana/src/addstar/src/load2mass_as_rawdata.c

@@ -181 +181 @@
         catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
         catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-        catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
 

trunk/Ohana/src/addstar/src/load2mass_catalog.c

@@ -56 +56 @@
   catalog[0].Naverage = Nave;
   catalog[0].Nmeasure = Nmeas;
-  catalog[0].Nsecf_mem = Nave*Nsec;
+  catalog[0].Nsecfilt_mem = Nave*Nsec;
   return (TRUE);
 }

trunk/Ohana/src/addstar/src/loadICRF_catalog.c

@@ -10 +10 @@
   catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
   catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-  catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_MEAS;
+  catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_MEASURE;
   catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
     

trunk/Ohana/src/addstar/src/loadICRF_save_remote.c

@@ -2 +2 @@
 # include "loadICRF.h"
 # define DEBUG 1
-
-int strextend (char *input, char *format,...) {
-
-  char tmpextra[1024], tmpline[1024];
-  va_list argp;
-
-  va_start (argp, format);
-  vsnprintf (tmpextra, 1024, format, argp);
-  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
-  strcpy (input, tmpline);
-
-  return TRUE;
-}
 
 int loadICRF_save_remote (ICRF_Stars *stars, int Nstars, HostTable *hosts, SkyRegion *region, char *fullname, AddstarClientOptions *options) {
@@ -64 +51 @@
 
     // need to generate the remote command
-    char command[1024];
-    snprintf (command, 1024, "loadICRF_client");
-    strextend (command, "-hostID %d", host->hostID);
-    strextend (command, "-D CATDIR %s", CATDIR);
-    strextend (command, "-hostdir %s", host->pathname);
-    strextend (command, "-cpt %s", region->name);
-    strextend (command, "-input %s", filename);
-    strextend (command, "-D ADDSTAR_RADIUS %f", options->radius);
+    char *command = NULL;
+    strextend (&command, "loadICRF_client");
+    strextend (&command, "-hostID %d", host->hostID);
+    strextend (&command, "-D CATDIR %s", CATDIR);
+    strextend (&command, "-hostdir %s", host->pathname);
+    strextend (&command, "-cpt %s", region->name);
+    strextend (&command, "-input %s", filename);
+    strextend (&command, "-D ADDSTAR_RADIUS %f", options->radius);
 
     fprintf (stderr, "command: %s\n", command);

trunk/Ohana/src/addstar/src/loadICRF_table.c

@@ -1 +1 @@
 # include "addstar.h"
 # include "loadICRF.h"
-
-# define GET_COLUMN(OUT,NAME,TYPE) \
-  TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \
-  myAssert (!strcmp(type, #TYPE), "wrong column type");
 
 int loadICRF_table (SkyList *skylistInput, HostTable *hosts, char *filename, AddstarClientOptions *options) {

trunk/Ohana/src/addstar/src/loadstarpar_catalog.c

@@ -10 +10 @@
   catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
   catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-  catalog.catflags = LOAD_AVES | LOAD_SECF | LOAD_STARPAR;
+  catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT | DVO_LOAD_STARPAR;
   catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
     

trunk/Ohana/src/addstar/src/loadstarpar_save_remote.c

@@ -2 +2 @@
 # include "loadstarpar.h"
 # define DEBUG 1
-
-int strextend (char *input, char *format,...) {
-
-  char tmpextra[1024], tmpline[1024];
-  va_list argp;
-
-  va_start (argp, format);
-  vsnprintf (tmpextra, 1024, format, argp);
-  snprintf (tmpline, 1024, "%s %s", input, tmpextra);
-  strcpy (input, tmpline);
-
-  return TRUE;
-}
 
 int loadstarpar_save_remote (StarPar_Stars *stars, int Nstars, HostTable *hosts, SkyRegion *region, char *fullname, AddstarClientOptions *options) {
@@ -62 +49 @@
 
     // need to generate the remote command
-    char command[1024];
-    snprintf (command, 1024, "loadstarpar_client");
-    strextend (command, "-hostID %d", host->hostID);
-    strextend (command, "-D CATDIR %s", CATDIR);
-    strextend (command, "-hostdir %s", host->pathname);
-    strextend (command, "-cpt %s", region->name);
-    strextend (command, "-input %s", filename);
+    char *command = NULL;
+    strextend (&command, "loadstarpar_client");
+    strextend (&command, "-hostID %d", host->hostID);
+    strextend (&command, "-D CATDIR %s", CATDIR);
+    strextend (&command, "-hostdir %s", host->pathname);
+    strextend (&command, "-cpt %s", region->name);
+    strextend (&command, "-input %s", filename);
 
     // launch the job on the remote machine (no handshake)
@@ -78 +65 @@
     }
     host->pid = pid; // save for future reference
+    free (command);
     
     save_remote_host (host);

trunk/Ohana/src/addstar/src/loadstarpar_table.c

@@ -1 +1 @@
 # include "addstar.h"
 # include "loadstarpar.h"
-
-# define GET_COLUMN(OUT,NAME,TYPE) \
-  TYPE *OUT = gfits_get_bintable_column_data (&theader, &ftable, NAME, type, &Nrow, &Ncol); \
-  myAssert (!strcmp(type, #TYPE), "wrong column type");
 
 int loadstarpar_table (SkyList *skylistInput, HostTable *hosts, char *filename, AddstarClientOptions *options) {

trunk/Ohana/src/addstar/src/loadsupercos.c

@@ -47 +47 @@
 
   // attempt to open the existing images table.  if it does not exist, we will save the loaded images
+  gfits_db_init (&db);
   db.mode   = dvo_catalog_catmode (CATMODE);
   db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/loadsupercos_rawdata.c

@@ -168 +168 @@
         catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
         catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-        catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
 

trunk/Ohana/src/addstar/src/loadwise_prelim_rawdata.c

@@ -186 +186 @@
         catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
         catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-        catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
 

trunk/Ohana/src/addstar/src/loadwise_rawdata.c

@@ -198 +198 @@
         catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
         catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-        catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+        catalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
         catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
 

trunk/Ohana/src/addstar/src/mkcmf.c

@@ -500 +500 @@
   }
 
-  gfits_table_set_PS1_DEV_0 (ftable, stars, Nstars);
+  gfits_table_set_PS1_DEV_0 (ftable, stars, Nstars, TRUE);
   gfits_modify (ftable->header, "EXTTYPE",   "%s", 1, "PS1_DEV_0");
 }
@@ -547 +547 @@
   }
 
-  gfits_table_set_PS1_DEV_1 (ftable, stars, Nstars);
+  gfits_table_set_PS1_DEV_1 (ftable, stars, Nstars, TRUE);
   gfits_modify (ftable->header, "EXTTYPE",   "%s", 1, "PS1_DEV_1");
 }
@@ -596 +596 @@
   }
 
-  gfits_table_set_CMF_PS1_V1 (ftable, stars, Nstars);
+  gfits_table_set_CMF_PS1_V1 (ftable, stars, Nstars, TRUE);
   gfits_modify (ftable->header, "EXTTYPE",   "%s", 1, "PS1_V1");
 }
@@ -651 +651 @@
   }
 
-  gfits_table_set_CMF_PS1_V2 (ftable, stars, Nstars);
+  gfits_table_set_CMF_PS1_V2 (ftable, stars, Nstars, TRUE);
   gfits_modify (ftable->header, "EXTTYPE",   "%s", 1, "PS1_V2");
 }
@@ -710 +710 @@
   }
 
-  gfits_table_set_CMF_PS1_V3 (ftable, stars, Nstars);
+  gfits_table_set_CMF_PS1_V3 (ftable, stars, Nstars, TRUE);
   gfits_modify (ftable->header, "EXTTYPE",   "%s", 1, "PS1_V3");
 }
@@ -778 +778 @@
   }
 
-  gfits_table_set_CMF_PS1_V4 (ftable, stars, Nstars);
+  gfits_table_set_CMF_PS1_V4 (ftable, stars, Nstars, TRUE);
   gfits_modify (ftable->header, "EXTTYPE",   "%s", 1, "PS1_V4");
 }

trunk/Ohana/src/addstar/src/resort_catalogs.c

@@ -74 +74 @@
     table->hosts[i].pathname = tmppath;
 
-    char command[1024];
-    snprintf (command, 1024, "addstar_client -resort");
-    strextend (command, "-D CATDIR %s", CATDIR);
-    strextend (command, "-hostID %d", table->hosts[i].hostID);
-    strextend (command, "-hostdir %s", table->hosts[i].pathname);
-    strextend (command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
+    char *command = NULL;
+    strextend (&command, "addstar_client -resort");
+    strextend (&command, "-D CATDIR %s", CATDIR);
+    strextend (&command, "-hostID %d", table->hosts[i].hostID);
+    strextend (&command, "-hostdir %s", table->hosts[i].pathname);
+    strextend (&command, "-region %f %f %f %f", UserPatch.Rmin, UserPatch.Rmax, UserPatch.Dmin, UserPatch.Dmax);
 
-    if (VERBOSE)                   strextend (command, "%s -v");
-    if (NTHREADS)                  strextend (command, "%s -threads %d", NTHREADS);
-    if (OLD_RESORT)                strextend (command, "%s -old-resort");
-    if (options->nosort == 3)      strextend (command, "%s -force-sort");
-    if (options->existing_regions) strextend (command, "%s -existing-regions");
+    if (VERBOSE)                   strextend (&command, "-v");
+    if (NTHREADS)                  strextend (&command, "-threads %d", NTHREADS);
+    if (OLD_RESORT)                strextend (&command, "-old-resort");
+    if (options->nosort == 3)      strextend (&command, "-force-sort");
+    if (options->existing_regions) strextend (&command, "-existing-regions");
 
     fprintf (stderr, "command: %s\n", command);

trunk/Ohana/src/addstar/src/resort_threaded.c

@@ -43 +43 @@
 
     // set the parameters which guide catalog open/load/create
-    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_LENSING;
-    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+    catalog.catformat   = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
+    catalog.catmode     = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
+    catalog.catcompress = dvo_catalog_catcompress (CATCOMPRESS); // set the default catcompress from config data
+    catalog.catflags    = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING;
+    catalog.Nsecfilt    = GetPhotcodeNsecfilt ();
   
     // an error exit status here is a significant error (disk I/O or file access)
@@ -57 +58 @@
     }
 
-    // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
-    if (catalog.Naves_disk == 0) {
+    // Naverage_disk == 0 implies an empty catalog file, skip empty catalogs
+    if (catalog.Naverage_disk == 0) {
       dvo_catalog_unlock (&catalog);
       dvo_catalog_free (&catalog);

trunk/Ohana/src/addstar/src/resort_unthreaded.c

@@ -21 +21 @@
 
     // set the parameters which guide catalog open/load/create
-    catalog.catformat = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
-    catalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
-    catalog.catflags  = LOAD_AVES | LOAD_MEAS | LOAD_LENSING;
-    catalog.Nsecfilt  = GetPhotcodeNsecfilt ();
+    catalog.catformat   = dvo_catalog_catformat (CATFORMAT);  // set the default catformat from config data
+    catalog.catmode     = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
+    catalog.catcompress = dvo_catalog_catcompress (CATCOMPRESS); // set the default catcompress from config data
+    catalog.catflags    = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_LENSING;
+    catalog.Nsecfilt    = GetPhotcodeNsecfilt ();
 
     // an error exit status here is a significant error (disk I/O or file access)
@@ -32 +33 @@
     }
 
-    // Naves_disk == 0 implies an empty catalog file, skip empty catalogs
-    if (catalog.Naves_disk == 0) {
+    // Naverage_disk == 0 implies an empty catalog file, skip empty catalogs
+    if (catalog.Naverage_disk == 0) {
       dvo_catalog_unlock (&catalog);
       dvo_catalog_free (&catalog);

trunk/Ohana/src/addstar/src/sedstar.c

@@ -26 +26 @@
   for (i = 0; i < skylist[0].Nregions; i++) {
     incatalog.filename = skylist[0].filename[i];
-    incatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
+    incatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT;
 
     // an error exit status here is a significant error
@@ -34 +34 @@
     }
 
-    // Naves_disk == 0 implies an empty catalog file
-    if ((incatalog.Naves_disk == 0) && options.only_match) {
+    // Naverage_disk == 0 implies an empty catalog file
+    if ((incatalog.Naverage_disk == 0) && options.only_match) {
       if (VERBOSE) fprintf (stderr, "skipping empty region\n");
       dvo_catalog_unlock (&incatalog);
@@ -54 +54 @@
     outcatalog.catmode   = dvo_catalog_catmode (CATMODE);      // set the default catmode from config data
     outcatalog.Nsecfilt  = GetPhotcodeNsecfilt ();
-    outcatalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF;
+    outcatalog.catflags = DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT;
 
     // an error exit status here is a significant error

trunk/Ohana/src/addstar/src/skycells.c

@@ -12 +12 @@
   /*** update the image table ***/
   /* setup image table format and lock */
+  gfits_db_init (&db);
   db.mode   = dvo_catalog_catmode (CATMODE);
   db.format = dvo_catalog_catformat (CATFORMAT);

trunk/Ohana/src/addstar/src/skycells_to_mdc.c

@@ -5 +5 @@
     // Convert FITS_DB to images structure
     off_t NdbImages;
-    Image *dbImages = gfits_table_get_Image(&db->ftable, &NdbImages, &db->swapped);
+    Image *dbImages = gfits_table_get_Image(&db->ftable, &NdbImages, &db->scaledValue, &db->nativeOrder);
 
     // If tess_id was not supplied assume that it is the basename of the CATDIR

trunk/Ohana/src/addstar/test

@@ -17 +17 @@
 relphot.Mcal.dMcal.png
 catdir.wise.test
+dvo.photcodes
+ptolemy.rc
+tap.log

@@ -17 +17 @@
 relphot.Mcal.dMcal.png
 catdir.wise.test
+dvo.photcodes
+ptolemy.rc
+tap.log

trunk/Ohana/src/addstar/test/relphot.dvo

@@ -1 +1 @@
 # -*-sh-*-
+if (not($?VERBOSE)) set VERBOSE = 0
 
 input tap.dvo
@@ -41 +42 @@
   end
 
-  echo relphot -D CATDIR catdir.test r -region {$RA-1.0} {$RA+1.0} {$DEC-1.0} {$DEC+1.0} -update -nloop 10.0
-  exec relphot -D CATDIR catdir.test r -region {$RA-1.0} {$RA+1.0} {$DEC-1.0} {$DEC+1.0} -update -nloop 10.0 >& /dev/null
+  $output = >& /dev/null
+  if ($VERBOSE > 2) 
+    $output = 
+  end
+  $command = relphot -D CATDIR catdir.test -region {$RA-1.0} {$RA+1.0} {$DEC-1.0} {$DEC+1.0} -update -nloop 10.0 -images r
+  echo $command
+  exec $command $output
 
   catdir catdir.test

trunk/Ohana/src/addstar/test/relphot.flatcorr.dvo

@@ -14 +14 @@
 # * no other mechanism to tell DVO about a flat-field correction
 
+$TAP_BREAK = 1
+$TAP_VERBOSE = 1
 input tap.dvo
 
 # set globals
-if (not($?PLOT)) set PLOT = 0 
+if (not($?PLOT)) set PLOT = 0
+
+$USE_CELL_OFFSETS = 1; # turn on non-zero cell offsets
+$SET_UC_RESID = 1; # write a non-zero resid value to the ubercal table (should not have an impact)
+$DO_2MASS = 1
+$DO_SYNTH = 1
+$DO_TYCHO = 1
+$NC_CLOUDS = 0.08
+$UC_CLOUDS = 0.02
 
 # set various global variables
 macro init
+  # generate a fake ptolemy.rc (to make this a contained test)
+  exec rm -f ptolemy.rc
+  output ptolemy.rc
+    echo "\#"
+    echo PHOTCODE_FILE dvo.photcodes
+    echo GSCFILE GSCregions.tbl
+    echo ZERO_PT 25.0
+    echo ADDSTAR_RADIUS 1.0
+    echo CATMODE                 SPLIT
+    echo CATFORMAT               PS1_V5
+    # echo CATFORMAT               PS1_V2 -- must be at least PS1_V3 for ubercaldist to work
+    echo SKY_DEPTH               4
+    echo CAMERA                  gpc1
+    echo CAMERA_CONFIG           dvo.layout
+    echo "input dvo.config"
+  output stdout
+
+  # we need to define a photcode table consistent with the fake construction
+  $zpt_nominal:g = 24.0
+  $zpt_nominal:r = 24.5
+
+  # klam is negative, so klam*(airmass - 1) increases the zero point
+  $klam_nominal:g = -0.15
+  $klam_nominal:r = -0.10
+
+  # generate a fake dvo.photcode
+  exec rm -f dvo.photcodes
+  output dvo.photcodes
+    echo "\# code  name                type    zero  slope offset c1    c2   slope   zero  equiv  sys scale   scale  sys     poor   bad     poor   bad"
+    echo "1     g                    sec   0.000  0.000 0.000     1     3 0.0000     0  1051   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000"
+    echo "2     r                    sec   0.000  0.000 0.000     2     3 0.0000     0  1052   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000"
+    echo "10000 GPC1.g.XY00          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10001 GPC1.g.XY01          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10010 GPC1.g.XY10          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10011 GPC1.g.XY11          dep  $zpt_nominal:g $klam_nominal:g 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10100 GPC1.r.XY00          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10101 GPC1.r.XY01          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10110 GPC1.r.XY10          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "10111 GPC1.r.XY11          dep  $zpt_nominal:r $klam_nominal:r 0.000     -     - 0.0000     0     1   0.015 0.000 1.000  0.000   0x0000 0x0000  0xe0440130 0x1003bc88"
+    echo "2011  2MASS_J              ref   0.000  0.000 0.000     -     - 0.0000     0     6   0.080 0.000 2.000  0.000   0x0000 0x0000  0x0000 0x0000"
+    echo "3001  SYNTH.g              ref   0.000  0.000 0.000     -     - 0.0000     0     1     nan 0.000 2.000  0.600   0x0000 0x0000  0x0000 0x0000"
+    echo "2020  TYCHO_B              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000"
+  output stdout      
+
   $RA_CENTER = 10.0
   $DEC_CENTER = 20.0
@@ -32 +86 @@
   $CHIP_DX = 1000
   $CHIP_DY = 1000
-
-  # images are loaded into dvo with GPC1 photcodes, so we need to get the nominal zps for those filters
-  $zpt_nominal:g = 24.58
-  $zpt_nominal:r = 24.80
-  $zpt_nominal:i = 24.74
-  $zpt_nominal:z = 24.26
-  $zpt_nominal:y = 23.41
-
-  # klam is negative, so klam*(airmass - 1) increase the zero point
-  $klam_nominal:g = -0.15
-  $klam_nominal:r = -0.10
-  $klam_nominal:i = -0.04
-  $klam_nominal:z = -0.03
-  $klam_nominal:y = -0.03
+  $DEFECT_FRAC = 0.05
+  $OFFSET_FRAC_UC = 0.0
+  $OFFSET_FRAC_NC = 0.5
 
   # we have two sets of images: ubercaled and not-ubercaled
 
-  # sequence to count images (only used in this function)
-  create tmpseq 0 9
-
-  $filt_uc:n = 9
-  $filt_uc:0 = g
-  $filt_uc:1 = g
-  $filt_uc:2 = g
-  $filt_uc:3 = r
-  $filt_uc:4 = r
-  $filt_uc:5 = r
-  $filt_uc:6 = i
-  $filt_uc:7 = i
-  $filt_uc:8 = i
-
-  # airmass slopes for these sequnece
+  # the number of exposures is defined by filt_uc:n
+  list filt_uc -split g g g g r r r r
+  $filt_uc:n = 4
+
+  # tmpseq is used to generate vectors in this function
+  create tmpseq 0 $filt_uc:n
+
+  # exptime and airmasses for uc images
+  set exptime_uc = 10.0 + zero(tmpseq)
+  set airmass_uc =  1.3 + zero(tmpseq)
   set klam_uc = zero(tmpseq)
-  for i 0 tmpseq[]
-    klam_uc[$i] = $klam_nominal:$filt_uc:$i
-  end    
-
-  # mjd and zpt values for ubercal'ed imaged
-  set exptime_uc = 10.0 + zero(tmpseq)
-  set airmass_uc = 1.3 + zero(tmpseq)
+  set zpt_uc = zero(tmpseq)
 
   # ubercal zero points are defined as ZP_nominal + 2.5log(exptime) + K*(airmass - 1.0)
   # note that K is defined as a negative value (is this sensible?)
-  set zpt_uc = 25.0 + 2.5*log(exptime_uc) + klam_uc*(airmass_uc - 1.0) + tmpseq*0.0050 - 0.0025
-
-  set mjd_uc = zero(zpt_uc)
-  mjd_uc[0] = 55000.01
-  mjd_uc[1] = 55000.02
-  mjd_uc[2] = 55000.03
-  mjd_uc[3] = 55015.01
-  mjd_uc[4] = 55015.02
-  mjd_uc[5] = 55015.03
-  mjd_uc[6] = 55025.01
-  mjd_uc[7] = 55025.02
-  mjd_uc[8] = 55025.03
+
+  # zero points and airmass slopes for these sequnece
+  for i 0 $filt_uc:n
+    klam_uc[$i] = $klam_nominal:$filt_uc:$i 
+    zpt_uc[$i]  = $zpt_nominal:$filt_uc:$i + 2.5*log(exptime_uc[$i]) + klam_uc[$i]*(airmass_uc[$i] - 1.0) + $UC_CLOUDS*(rnd(klam_uc[$i]) - 0.5)
+  end    
+  set mjd_uc = 55000.00 + 0.01*(tmpseq % 3) + 10.0*int(tmpseq / 3)
 
   # mjd and zpt values for not-ubercal'ed images
-  $filt_nc:n = 9
-  $filt_nc:0 = g
-  $filt_nc:1 = g
-  $filt_nc:2 = g
-  $filt_nc:3 = r
-  $filt_nc:4 = r
-  $filt_nc:5 = r
-  $filt_nc:6 = i
-  $filt_nc:7 = i
-  $filt_nc:8 = i
+  list filt_nc -split g g g g r r r r
+  $filt_nc:n = 4
+
+  # tmpseq is used to generate vectors in this function
+  create tmpseq 0 $filt_nc:n
+
+  # place this within a valid season (55000.0 - 55010.0 - 55020.0 - 55030.0)
+  set exptime_nc = 10.0 + zero(tmpseq)
+  set airmass_nc =  1.6 + zero(tmpseq)
+  set klam_nc = zero(tmpseq)
+  set zpt_nc = zero(tmpseq)
 
   # airmass slopes for these sequnece
-  set klam_nc = zero(tmpseq)
-  for i 0 tmpseq[]
-    klam_nc[$i] = $klam_nominal:$filt_nc:$i
+  for i 0 $filt_nc:n
+    klam_nc[$i] = $klam_nominal:$filt_nc:$i 
+    zpt_nc[$i]  = $zpt_nominal:$filt_nc:$i + 2.5*log(exptime_nc[$i]) + klam_nc[$i]*(airmass_nc[$i] - 1.0) + $NC_CLOUDS*(rnd(klam_nc[$i]) - 0.5)
   end    
-
-  # place this within a valid season (55000.0 - 55010.0 - 55020.0 - 55030.0)
-  set exptime_nc = 15.0 + zero(tmpseq)
-  set airmass_nc = 1.6 + zero(tmpseq)
-
-  # ubercal zero points are defined as ZP_nominal + 2.5log(exptime) + K*(airmass - 1.0)
-  set zpt_nc = 25.0 + 2.5*log(exptime_nc) + klam_nc*(airmass_nc - 1.0) - tmpseq*0.0050 + 0.0025
-
-  set mjd_nc = zero(zpt_nc)
-  mjd_nc[0] = 55000.11
-  mjd_nc[1] = 55000.12
-  mjd_nc[2] = 55000.13
-  mjd_nc[3] = 55015.11
-  mjd_nc[4] = 55015.12
-  mjd_nc[5] = 55015.13
-  mjd_nc[6] = 55025.11
-  mjd_nc[7] = 55025.12
-  mjd_nc[8] = 55025.13
+  set mjd_nc = 55000.10 + 0.01*(tmpseq % 3) + 10.0*int(tmpseq / 3)
+
+  delete tmpseq
 end
 
@@ -139 +152 @@
   mkdir $rootdir
 
+  break -auto off
   exec rm -rf $catdir
+  exec rm -rf $fileroot*
+  break -auto on
 
   init
   mkzptfile
-  mkstars 500
+  mkstars 1000
+
+  catdir $catdir
+
+  mk2mass $fileroot $catdir
+  ck2mass addstar
+  
+  mksynth $fileroot $catdir
+  cksynth addstar
+
+  # to add a valid tycho test, I need to generate a zpt table for tycho as well
+  mktycho $fileroot $catdir
+  cktycho addstar
 
   tapPLAN {((mjd_uc[]*4 + mjd_nc[]*4) + (mjd_uc[] + mjd_nc[]) + (mjd_uc[] + mjd_nc[])) / 3}
@@ -149 +177 @@
   # generate the basic images and check they were correctly ingested by dvo
   mksequence $fileroot $catdir
-  for i 0 mjd_uc[] 3
-    ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i raw
-  end
-  for i 0 mjd_nc[] 3
-    ckexposure catdir.test mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i raw
+  for i 0 mjd_uc[]
+    ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i raw
+  end
+  for i 0 mjd_nc[]
+    ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i raw
   end
 
   # run setphot on the db and check that the images now match the expected values
-  exec setphot -update -ubercal testzpt.fits -D CATDIR $catdir
-  for i 0 mjd_uc[] 3
-    ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i setphot_uc
-  end
-  for i 0 mjd_nc[] 3
-    ckexposure catdir.test mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i setphot_nc
+  tapEXEC setphot -update -ubercal testzpt.fits -D CATDIR $catdir
+  for i 0 mjd_uc[]
+    ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i setphot_uc
+  end
+  for i 0 mjd_nc[]
+    ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i setphot_nc
   end
 
   # run relphot on the db and check that the images now match the expected values
-  exec relphot g,r,i -v -region 9.5 10.5 19.5 20.5 -D CATDIR catdir.test -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -keep-ubercal -D IMAGE_OFFSET 0.5 -update >& tmp.log
-  for i 0 mjd_uc[] 3
-    ckexposure catdir.test mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot
-  end
-  for i 0 mjd_nc[] 3
-    ckexposure catdir.test mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot
-  end
+  tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -D IMAGE_OFFSET 0.5 -update -synthphot_zpts $fileroot.synthzpts.fits
+  for i 0 mjd_uc[]
+    ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc
+  end
+  for i 0 mjd_nc[]
+    ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc
+  end
+  ck2mass relphot
+  cksynth relphot
+  cktycho relphot
 
   tapDONE
+end
+
+macro try.relphot
+  if ($0 != 3)
+    echo "go (fileroot) (catdir)"
+    break
+  end
+
+  local i fileroot rootdir catdir
+
+  $fileroot = $1
+  $catdir = $2
+
+  # run relphot on the db and check that the images now match the expected values
+  tapEXEC relphot -images g,r -v -region 9.5 10.5 19.5 20.5 -D CATDIR $catdir -D STAR_TOOFEW 1 -D SIGMA_LIM 0.07 -statmode WT_MEAN -cloud-limit 0.5 -D IMAGE_OFFSET 0.5 -update -synthphot_zpts $fileroot.synthzpts.fits -nloop 24
+
+  for i 0 mjd_uc[]
+    # ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc
+      ckexposure $catdir mjd_uc[$i] zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] $filt_uc:$i relphot_uc
+  end
+  for i 0 mjd_nc[]
+    # ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc
+      ckexposure $catdir mjd_nc[$i] zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] $filt_nc:$i relphot_nc
+  end
 end
 
@@ -215 +270 @@
   $AIRMASS     = $5
   $FILTER      = $6
-  $MODE        = $7
+  $FUNC_MODE   = $7
 
   # XXX need a function to extract the nominal zpt for a given filter / photcode from the db
@@ -235 +290 @@
 
   skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2} 
-  mextract ra dec mag xccd yccd where (abs(time - $MJD_IMAGE) < 0.0001)
+  mextract ra dec mag xccd yccd dbflags photflags where (abs(time - $MJD_IMAGE) < 0.0001)
  
   match2d -closest ra dec stars_ra stars_dec 0.001 -index1 index1 -index2 index2
@@ -246 +301 @@
   set dm = mag - stars_mag_m
 
+  set good_photom = (not(photflags & 0x02))
+  foreach field dm xccd yccd
+    subset $field\_good = $field if good_photom
+  end
+
   if ($PLOT) 
     dev -n 1
-    lim mag dm; clear; box; plot mag dm
-  end
-
-  set cell_xbin = int(xccd / ($CHIP_DX / $NCELL_X))
-  set cell_ybin = int(yccd / ($CHIP_DY / $NCELL_Y))
-  subset dm00 = dm if (cell_xbin == 0) && (cell_ybin == 0)
-  subset dm01 = dm if (cell_xbin == 0) && (cell_ybin == 1)
-  subset dm10 = dm if (cell_xbin == 1) && (cell_ybin == 0)
-  subset dm11 = dm if (cell_xbin == 1) && (cell_ybin == 1)
+    lim mag dm; clear; box; plot mag dm; plot -c red -pt 7 mag dm where (photflags & 0x00000002)
+    dev -n 2
+    lim xccd dm; clear; box; plot xccd dm; plot -c red -pt 7 xccd dm where (photflags & 0x00000002)
+    dev -n 3
+    lim ra dm; clear; box; plot ra dm; plot -c red -pt 7 ra dm where (photflags & 0x00000002)
+    echo "red points are known bad photometry"
+  end
+
+  # reproduce eddie's esoteric padding
+  set cell_xbin = max(0 , min(1 , int((xccd_good +  8) / ($CHIP_DX / $NCELL_X))))
+  set cell_ybin = max(0 , min(1 , int((yccd_good + 10) / ($CHIP_DY / $NCELL_Y))))
+  subset dm00 = dm_good if (cell_xbin == 0) && (cell_ybin == 0)
+  subset dm01 = dm_good if (cell_xbin == 0) && (cell_ybin == 1)
+  subset dm10 = dm_good if (cell_xbin == 1) && (cell_ybin == 0)
+  subset dm11 = dm_good if (cell_xbin == 1) && (cell_ybin == 1)
 
   # uncorrected values behave like this:
@@ -273 +339 @@
   $ZPT_REAL_NORM = $ZPT_REAL - 2.5*log($EXPTIME) - $KLAM_NOMINAL*($AIRMASS - 1.0)
 
-  if ("$MODE" == "raw") 
+  if ("$FUNC_MODE" == "raw") 
     for ix 0 $NCELL_X
       for iy 0 $NCELL_Y
@@ -280 +346 @@
   
         set dm_adjust = dm$ix\$iy - $ZPT_NOMINAL + $ZPT_REAL_NORM + cell_off[$ix][$iy]
+        # echo set dm_adjust = dm$ix\$iy - $ZPT_NOMINAL + $ZPT_REAL_NORM + cell_off[$ix][$iy]
+
         vstat -q dm_adjust
-        tapOK {abs($MEAN) < 0.005} "addstar raw ZP $MJD_IMAGE $FILTER"
+        sprintf line "addstar raw ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : cell%d%d %6.3f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA $ix $iy cell_off[$ix][$iy] dm_adjust[]
+        tapOK {abs($MEAN) < 0.005} "$line"
       end
     end
@@ -287 +356 @@
   end
 
-  if ("$MODE" == "setphot_uc") 
-    vstat -q dm
-    tapOK {abs($MEAN) < 0.005} "setphot_uc  ZP $MJD_IMAGE $FILTER"
+  if ("$FUNC_MODE" == "setphot_uc") 
+    vstat -q dm_good
+    sprintf line "setphot_uc  ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA dm_good[]
+    tapOK {abs($MEAN) < 0.005} "$line"
+
+    subset tmp = dm if (dbflags & 0x00008000)
+    tapOK {tmp[] == dm[]} "ubercal detections have ubercal bit set"
+
+    imextract MCAL FLAGS where (abs(time - $MJD_IMAGE) < 0.0001)
+    subset tmp = FLAGS if (FLAGS & 0x0000200)
+    tapOK {tmp[] == FLAGS[]} "ubercal images have ubercal bit set"
+    
     return
   end
 
-  if ("$MODE" == "setphot_nc") 
-    vstat -q dm
-    tapOK {abs($MEAN - $ZPT_NOMINAL + $ZPT_REAL_NORM) < 0.005} "setphot_nc  ZP $MJD_IMAGE $FILTER"
+  if ("$FUNC_MODE" == "setphot_nc") 
+    vstat -q dm_good
+    sprintf line "setphot_nc  ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA dm_good[]
+    tapOK {abs($MEAN - $ZPT_NOMINAL + $ZPT_REAL_NORM) < 0.005} "$line"
+
+    subset tmp = dm if (not(dbflags & 0x00008000))
+    tapOK {tmp[] == dm[]} "non-ubercal detections do NOT have ubercal bit set"
+
+    imextract FLAGS where (abs(time - $MJD_IMAGE) < 0.0001)
+    subset tmp = FLAGS if (not(FLAGS & 0x0000200))
+    tapOK {tmp[] == FLAGS[]} "non-ubercal images do NOT have ubercal bit set"
+    
     return
   end
 
-  if ("$MODE" == "relphot") 
-    vstat -q dm
-    tapOK {abs($MEAN) < 0.005} "relphot    ZP $MJD_IMAGE $FILTER"
+  if (("$FUNC_MODE" == "relphot_uc") || ("$FUNC_MODE" == "relphot_nc"))
+    vstat -q dm_good
+    sprintf line "relphot     ZP %8.2f $FILTER %6.3f %6.3f %6.3f: %7.4f %7.4f : %3d" $MJD_IMAGE $ZPT_REAL $ZPT_REAL_NORM $ZPT_NOMINAL $MEAN $SIGMA dm_good[]
+
+    if ("$FUNC_MODE" == "relphot_uc")
+      tapOK {abs($MEAN) < 0.001} "$line"
+      tapOK {abs($SIGMA) < 0.001} "$line"
+    end
+    if ("$FUNC_MODE" == "relphot_nc")
+      # XXX NOTE relaxed condition : bad photometry points are partly clipped but still cause a bias
+      tapOK {abs($MEAN) < 0.020} "$line"
+      tapOK {abs($SIGMA) < 0.020} "$line"
+    end
+
+    mextract ra dec mag dbflags
+    subset rs = ra  if ((dbflags & 0x8000) && (mag > 13.5))
+    subset ds = dec if ((dbflags & 0x8000) && (mag > 13.5))
+    avmatch rs ds 0.1 $FILTER\:flags $FILTER\:uc_dist 
+
+    subset tmp = rs if (($FILTER\:flags & 0x8) || ($FILTER\:flags & 0x4))
+    tapOK {tmp[] == rs[]} "objects with ubercal detections have ubercal bit set or used SYNTH photometry (tmp[] vs rs[])"
+ 
+    subset tmp = rs if (($FILTER\:uc_dist < 0.1) || ($FILTER\:flags & 0x4))
+    tapOK {tmp[] == rs[]} "objects with ubercal detections have ubercal dist of 0.0 or used SYNTH photometry"
     return
   end
+
+  echo "ERROR: unknown mode for ckexposure : $FUNC_MODE"
 
 end
@@ -316 +426 @@
 
   # we have defined a set of mjd and zero point values for ubercal'ed images
-  # for i 0 mjd_uc[]
-  for i 0 mjd_uc[] 3
+  for i 0 mjd_uc[]
     mkexposure $1.uc.$i $RA_CENTER $DEC_CENTER zpt_uc[$i] exptime_uc[$i] airmass_uc[$i] mjd_uc[$i] $filt_uc:$i $2
   end
 
   # we have a second set of mjd and zero point values for non-ubercal'ed images
-  # for i 0 mjd_nc[]
-  for i 0 mjd_nc[] 3
+  for i 0 mjd_nc[]
     mkexposure $1.nc.$i $RA_CENTER $DEC_CENTER zpt_nc[$i] exptime_nc[$i] airmass_nc[$i] mjd_nc[$i] $filt_nc:$i $2
   end
@@ -334 +442 @@
  end
 
- local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR
+ local ix iy date time datetime ra dec ROOT RAo DECo ZPT MJD FILTER EXPTIME AIRMASS CATDIR cmd
 
  $ROOT    = $1
@@ -366 +474 @@
     $options = -mjd $MJD 
     $options = $options -radec $ra $dec 
-    $options = $options -type PS1_V2 
+
+    #$options = $options -type PS1_V2 
+    #$options = $options -type PS1_V3 
+    #$options = $options -type PS1_V4 
+    $options = $options -type PS1_V5 
+
     $options = $options -coords 
     $options = $options -photcode GPC1.$FILTER.XY$ix\$iy 
     $options = $options -no-noise 
+    $options = $options -flags
+    $options = $options -bad-psfqf-frac $DEFECT_FRAC
     $options = $options -size $CHIP_DX $CHIP_DY 
     $options = $options -crpix {0.5*$CHIP_DX} {0.5*$CHIP_DY} 
     $options = $options -airmass $AIRMASS
     $options = $options -exptime $EXPTIME
-    exec mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options
-    echo mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options 
-    
+
+    tapEXEC mkcmf test.in.txt $ROOT.$ix.$iy.cmf $options
+
     # the fake images have inconsistent ra,dec and airmass,sidtime values
-    echo addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass
-    exec addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass >& tmp.log
+    tapEXEC addstar $ROOT.$ix.$iy.cmf -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass
   end
  end
@@ -405 +519 @@
   subset stars_mr = stars_mag if valid
 
-  set cell_x = int(stars_x / ($CHIP_DX / $NCELL_X))
-  set cell_y = int(stars_y / ($CHIP_DX / $NCELL_X))
+  # note eddie's esoteric padding rule:
+  set cell_x = max(0 , min(1 , int((stars_x +  8) / ($CHIP_DX / $NCELL_X))))
+  set cell_y = max(0 , min(1 , int((stars_y + 10) / ($CHIP_DX / $NCELL_X))))
 
   # eddie says: M_real = M_inst + zpt + ff_offset
@@ -418 +533 @@
   end
 
-  write -f "%10.6f %10.6f %7.3f  %6.1f %6.1f %7.3f" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi
+  # create bad detections / detections to ignore or avoid, save to a file?
+
+  # create 3 types of things to ignore / avoid
+  # * badPhot (deviate the photometry)
+  # * badFlag (set a bad flag bit)
+  # * poorFlag (set a bad flag bit)
+  set badPhot  = rnd(stars_r) <  $DEFECT_FRAC
+  set badFlag  = rnd(stars_r) <  $DEFECT_FRAC
+  set poorFlag = rnd(stars_r) <  $DEFECT_FRAC
+
+  # create deviant photometry (1%)
+  # I'm setting the flag value 0x02 (EXTMODEL) if the object is deviated.  This is a simple way to pass the info
+  # note that outlier rejection only occurs for Nmeas > 5
+  set stars_mi = stars_mi + 0.2*badPhot
+  set stars_fl = zero(stars_mi) + 0x00000080*badFlag + 0x00000100*poorFlag + 0x00000002*badPhot
+
+  # bad flags that we sould respect:
+  # PM_SOURCE_MODE_FAIL             = 0x00000008, ///< Fit (non-linear) failed (non-converge, off-edge, run to zero)
+  # PM_SOURCE_MODE_SATSTAR          = 0x00000080, ///< Source model peak is above saturation
+  # PM_SOURCE_MODE_BADPSF           = 0x00000400, ///< Failed to get good estimate of object's PSF
+  # PM_SOURCE_MODE_DEFECT           = 0x00000800, ///< Source is thought to be a defect
+  # PM_SOURCE_MODE_SATURATED        = 0x00001000, ///< Source is thought to be saturated pixels (bleed trail)
+  # PM_SOURCE_MODE_CR_LIMIT         = 0x00002000, ///< Source has crNsigma above limit
+  # PM_SOURCE_MODE_MOMENTS_FAILURE  = 0x00008000, ///< could not measure the moments
+  # PM_SOURCE_MODE_SKY_FAILURE      = 0x00010000, ///< could not measure the local sky
+  # PM_SOURCE_MODE_SKYVAR_FAILURE   = 0x00020000, ///< could not measure the local sky variance
+  # PM_SOURCE_MODE_SIZE_SKIPPED     = 0x10000000, ///< size could not be determined
+
+  # poor flags that we should respect:
+  # PM_SOURCE_MODE_POOR             = 0x00000010, ///< Fit succeeds, but low-SN, high-Chisq, or large (for PSF -- drop?)
+  # PM_SOURCE_MODE_PAIR             = 0x00000020, ///< Source fitted with a double psf
+  # PM_SOURCE_MODE_BLEND            = 0x00000100, ///< Source is a blend with other sources
+  # PM_SOURCE_MODE_BELOW_MOMENTS_SN = 0x00040000, ///< moments not measured due to low S/N
+  # PM_SOURCE_MODE_BLEND_FIT        = 0x00400000, ///< source was fitted as a blend
+  # PM_SOURCE_MODE_ON_SPIKE         = 0x20000000, ///< peak lands on diffraction spike
+  # PM_SOURCE_MODE_ON_GHOST         = 0x40000000, ///< peak lands on ghost or glint
+  # PM_SOURCE_MODE_OFF_CHIP         = 0x80000000, ///< peak lands off edge of chip
+
+  write -f "%10.6f %10.6f %7.3f  %6.1f %6.1f %7.3f 0x%x" $1 stars_r stars_d stars_mr stars_x stars_y stars_mi stars_fl
 end
 
@@ -439 +592 @@
   set stars_ra  = $RA_CENTER  + $RA_RANGE  * (rnd(tmp) - 0.5) / 3600 / dcos ($DEC_CENTER)
   set stars_dec = $DEC_CENTER + $DEC_RANGE * (rnd(tmp) - 0.5) / 3600
-  set stars_mag = 15.0 + 4.0 * rnd(tmp) 
+  set stars_mag = 10.0 + 10.0 * rnd(tmp) 
+  set stars_dmag = 0.02 + 0.1*rnd(tmp)
+end
+
+macro mk2mass
+  if ($0 != 3)
+    echo "USAGE: mk2mass (fileroot) (catdir)"
+    break
+  end
+
+  if (not($DO_2MASS)) return
+
+  local ROOT CATDIR
+
+  $ROOT    = $1
+  $CATDIR  = $2
+
+  set stars_flags = 0x00c00000*(rnd(stars_ra) < 0.1) + 0x7*(rnd(stars_ra) < 0.8)
+  write $ROOT.2mass.dat -f "%10.6f %10.6f %6.3f %6.3f 0x%x" stars_ra stars_dec stars_mag stars_dmag stars_flags
+
+  tapEXEC addstar -ref $ROOT.2mass.dat -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass -photcode 2MASS_J
+
+  set r_2mass = stars_ra
+  set d_2mass = stars_dec
+  set m_2mass = stars_mag
+  set dm_2mass = stars_dmag
+  set fl_2mass = stars_flags
+end
+
+macro ck2mass
+  if ($0 != 2)
+    echo "USAGE: ck2mass (stage)"
+    break
+  end
+
+  if (not($DO_2MASS)) return
+
+  if ("$1" == "addstar") 
+    skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2} 
+    mextract ra dec mag xccd yccd dbflags photflags where (photcode == 2011)
+    tapOK {ra[] == r_2mass[]} "read in as many 2mass stars as we generated"
+    
+    match2d -closest ra dec r_2mass d_2mass 0.001 -index1 index1 -index2 index2
+    subset tmp = index1 if (index1 >= 0)
+    tapOK {tmp[] == ra[]} "2mass sources match those generated"
+  end
+
+  if ("$1" == "relphot")
+    subset rs = r_2mass if (fl_2mass & 0x00c00000)
+    subset ds = d_2mass if (fl_2mass & 0x00c00000)
+    avmatch rs ds 0.1 flags
+    subset tmp = rs if (flags & 0x02000000); # extended in 2mass
+    tapOK {tmp[] == rs[]} "2mass extended objects properly marked"
+    
+    subset rs = r_2mass if (fl_2mass & 0x07)
+    subset ds = d_2mass if (fl_2mass & 0x07)
+    avmatch rs ds 0.1 flags
+    subset tmp = rs if (flags & 0x08000000); # extended in 2mass
+    tapOK {tmp[] == rs[]} "2mass good objects properly marked"
+  end
+end
+
+macro mksynth
+  if ($0 != 3)
+    echo "USAGE: mksynth (fileroot) (catdir)"
+    break
+  end
+
+  if (not($DO_SYNTH)) return
+
+  local ROOT CATDIR
+
+  $ROOT    = $1
+  $CATDIR  = $2
+ 
+  subset r_synth  = stars_ra    if (stars_mag < 17)
+  subset d_synth  = stars_dec   if (stars_mag < 17)
+  subset m_synth  = stars_mag   if (stars_mag < 17)
+  subset dm_synth = stars_dmag  if (stars_mag < 17)
+  subset fl_synth = stars_flags if (stars_mag < 17)
+
+  write $ROOT.synth.dat -f "%10.6f %10.6f %6.3f %6.3f 0x%x" r_synth d_synth m_synth dm_synth fl_synth
+
+  tapEXEC addstar -ref $ROOT.synth.dat -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass -photcode SYNTH.g
+end
+
+macro cksynth
+  if ($0 != 2)
+    echo "USAGE: cksynth (stage)"
+    break
+  end
+
+  if (not($DO_SYNTH)) return
+
+  if ("$1" == "addstar") 
+    skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2} 
+    mextract ra dec mag xccd yccd dbflags photflags where (photcode == 3001)
+    tapOK {ra[] == r_synth[]} "read in as many synth stars as we generated"
+    
+    match2d -closest ra dec r_synth d_synth 0.001 -index1 index1 -index2 index2
+    subset tmp = index1 if (index1 >= 0)
+    tapOK {tmp[] == ra[]} "synth sources match those generated"
+  end
+
+  if ("$1" == "relphot")
+    subset rs = r_synth if (m_synth < 13.5)
+    subset ds = d_synth if (m_synth < 13.5)
+    avmatch rs ds 0.1 g:flags
+    subset tmp = rs if (g:flags & 0x04); # synth used for photometry
+    tapOK {tmp[] == rs[]} "synth objects used for photometry properly marked"
+  end
+end
+
+macro mktycho
+  if ($0 != 3)
+    echo "USAGE: mktycho (fileroot) (catdir)"
+    break
+  end
+
+  if (not($DO_TYCHO)) return
+
+  local ROOT CATDIR
+
+  $ROOT    = $1
+  $CATDIR  = $2
+
+  subset r_tycho  = stars_ra    if (stars_mag < 12)
+  subset d_tycho  = stars_dec   if (stars_mag < 12)
+  subset m_tycho  = stars_mag   if (stars_mag < 12)
+  subset dm_tycho = stars_dmag  if (stars_mag < 12)
+  subset fl_tycho = stars_flags if (stars_mag < 12)
+
+  write $ROOT.tycho.dat -f "%10.6f %10.6f %6.3f %6.3f 0x%x" r_tycho d_tycho m_tycho dm_tycho fl_tycho
+
+  tapEXEC addstar -ref $ROOT.tycho.dat -D CATDIR $CATDIR -D CAMERA gpc1 -quick-airmass -photcode TYCHO_B
+
+  $KAPA = kapa -noX
+  dev -n tmpdev
+  # resize 100 100
+  resize 50 50
+  region $RA_CENTER $DEC_CENTER 0.2
+  section default -imtool none
+  # causes trouble for this to come before region...
+  create tmp 0 1
+  cdensify tmpmap tmp tmp
+  set map = zero(tmpmap) + 0.2
+  close
+  $KAPA = kapa
+
+  local filter  
+  foreach filter g r i z y
+    if ("$filter" == "g")
+      wd map $ROOT.synthzpts.fits -bitpix -32 -bzero 0.0 -bscale 1.0
+    end
+    keyword map EXTNAME -w "map_$filter"
+    wd map $ROOT.synthzpts.fits -bitpix -32 -bzero 0.0 -bscale 1.0 -extend
+  end
+end
+
+macro cktycho
+  if ($0 != 2)
+    echo "USAGE: cktycho (stage)"
+    break
+  end
+
+  if (not($DO_TYCHO)) return
+
+  if ("$1" == "addstar") 
+    skyregion {$RA_CENTER - 0.2/dcos($DEC_CENTER)} {$RA_CENTER + 0.2/dcos($DEC_CENTER)} {$DEC_CENTER - 0.2} {$DEC_CENTER + 0.2} 
+    mextract ra dec mag xccd yccd dbflags photflags where (photcode == 2020)
+    tapOK {ra[] == r_tycho[]} "read in as many tycho stars as we generated"
+    
+    match2d -closest ra dec r_tycho d_tycho 0.001 -index1 index1 -index2 index2
+    subset tmp = index1 if (index1 >= 0)
+    tapOK {tmp[] == ra[]} "tycho sources match those generated"
+  end
+
+  if ("$1" == "relphot")
+    subset rst = r_tycho if (m_tycho < 12)
+    subset dst = d_tycho if (m_tycho < 12)
+    # XXX : should not restrict to 'g'
+    avmatch rst dst 0.1 g:flags
+    subset tmp = rst if (g:flags & 0x40); # tycho used for photometry
+    tapOK {tmp[] == rst[]} "tycho objects used for photometry properly marked"
+
+    # non-tycho stars with synth phot should be repaired
+    subset rsn = r_synth if (m_synth > 12) && (m_synth < 13.5)
+    subset dsn = d_synth if (m_synth > 12) && (m_synth < 13.5)
+    # XXX : should not restrict to 'g'
+    avmatch rsn dsn 0.1 g:flags
+    subset tmp = rsn if (g:flags & 0x80); # synth photometry repaired
+    tapOK {tmp[] == rsn[]} "tycho objects used for photometry properly marked"
+  end
 end
 
@@ -456 +801 @@
 
   # define the season boundaries
-  keyword phu TS0_0000 -wf 55000.0
-  keyword phu TS1_0000 -wf 55010.0
-  keyword phu TS0_0001 -wf 55010.0
-  keyword phu TS1_0001 -wf 55020.0
-  keyword phu TS0_0002 -wf 55020.0
-  keyword phu TS1_0002 -wf 55030.0
+  keyword phu S0_MJD0 -wf 55000.0
+  keyword phu S1_MJD0 -wf 55010.0
+  keyword phu S0_MJD1 -wf 55010.0
+  keyword phu S1_MJD1 -wf 55020.0
+  keyword phu S0_MJD2 -wf 55020.0
+  keyword phu S1_MJD2 -wf 55030.0
 
   # create a single flat-field correction map for a single chip
   mcreate cell_off 2 2
-  cell_off[0][0] =  0.01
-  cell_off[1][0] = -0.01
-  cell_off[0][1] =  0.02
-  cell_off[1][1] = -0.02
+  if ($USE_CELL_OFFSETS) 
+    cell_off[0][0] =  0.05
+    cell_off[1][0] = -0.03
+    cell_off[0][1] =  0.08
+    cell_off[1][1] = -0.10
+  end
 
   # in the correction table, we have an image of the full exposure
@@ -481 +828 @@
     for iyc 0 $NCELL_Y
       for ix 0 $NCHIP_X
+        set tcell_off = cell_off
+        if ($ix < 4)
+          rotate tcell_off 180
+        end
         for ixc 0 $NCELL_X
-          concat cell_off[$ixc][$iyc] image_offset
+          concat tcell_off[$ixc][$iyc] image_offset
         end
       end
@@ -498 +849 @@
   set mjd_obs = mjd_uc
   set zp = zpt_uc
-  set resid = 0.02*rnd(zpt_uc) - 0.01
+
+  if ($SET_UC_RESID) 
+    set resid = 0.02*rnd(zpt_uc) - 0.01
+  else 
+    set resid = zero(zpt_uc)
+  end
 
   delete -q flatcorr

trunk/Ohana/src/addstar/test/simple.dvo

@@ -10 +10 @@
   test.fields PS1_V2    PS1_V1
   test.fields PS1_V3    PS1_V1
-  test.fields PS1_V4    PS1_V1
+  test.fields PS1_V5    PS1_V1
 
   test.fields PS1_DEV_0 PS1_V2
@@ -17 +17 @@
   test.fields PS1_V2    PS1_V2
   test.fields PS1_V3    PS1_V2
-  test.fields PS1_V4    PS1_V4
+  test.fields PS1_V5    PS1_V2
 
   test.fields PS1_DEV_0 PS1_V3
@@ -24 +24 @@
   test.fields PS1_V2    PS1_V3
   test.fields PS1_V3    PS1_V3
-  test.fields PS1_V4    PS1_V4
+  test.fields PS1_V5    PS1_V3
 
   test.fields PS1_DEV_0 PS1_V4
@@ -31 +31 @@
   test.fields PS1_V2    PS1_V4
   test.fields PS1_V3    PS1_V4
-  test.fields PS1_V4    PS1_V4
+  test.fields PS1_V5    PS1_V4
 end  
 
@@ -54 +54 @@
 
   mkinput
-  echo mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC -type $1 $NO_NOISE
-  exec mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC -type $1 $NO_NOISE
-  if ($TAP_VERBOSE)
-    echo exec addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
-    exec addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
-  else
-    echo addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
-    exec addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass >& tmp.log
-  end
+  tapEXEC mkcmf test.in.txt test.cmf -date 2008/1/1 -time 01:00:00 -radec $RA $DEC -type $1 $NO_NOISE
+  tapEXEC addstar -D CATDIR catdir.test -D CAMERA simtest test.cmf -D CATFORMAT $2 -quick-airmass
 
   list testfields -copy testfields_$1
@@ -79 +72 @@
     if ("$name:0" == "SKIP") continue
     if ("$name:2" == "SKIP") continue
+ 
     read -fits Chip.psf $name:0
     mextract $name:2
@@ -92 +86 @@
   set d = $name:0 - $name:2
   vstat -q d
-  tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN)"
-  tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA)"
+  tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN = $MEAN)"
+  tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA = $SIGMA)"
 
   # detection ID must be first for this to work:
@@ -142 +136 @@
     if ("$name:0" == "PSF_THETA")
       # echo $MEAN
-      tapOK {abs($MEAN)  < 0.006} "$name:0 vs $name:2 (MEAN)"
-      tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA)"
+      tapOK {abs($MEAN)  < 0.006} "$name:0 vs $name:2 (MEAN = $MEAN)"
+      tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA = $SIGMA)"
       continue
     end
 
-    tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN)"
-    tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA)"
+    tapOK {abs($MEAN)  < 0.001} "$name:0 vs $name:2 (MEAN = $MEAN)"
+    tapOK {abs($SIGMA) < 0.001} "$name:0 vs $name:2 (SIGMA = $MEAN)"
   end    
 
@@ -231 +225 @@
   PSF_INST_MAG      : mag:inst  
   PSF_INST_MAG_SIG  : mag:err   
-  AP_MAG_STANDARD   : mag:ap # FAIL
+# AP_MAG_STANDARD   : mag:ap # FAIL
   AP_MAG_RADIUS     : SKIP # no accessor
   PEAK_FLUX_AS_MAG  : SKIP # no accessor
@@ -237 +231 @@
   CAL_PSF_MAG_SIG   : SKIP # photometry is not calibrated in the cmf
   SKY               : sky       
-  SKY_SIG           : sky_err   
+  SKY_SIGMA         : sky_err   
   PSF_CHISQ         : psf_chisq 
   CR_NSIGMA         : cr_nsigma 
@@ -267 +261 @@
   PSF_INST_MAG      : mag:inst  
   PSF_INST_MAG_SIG  : mag:err   
-  AP_MAG_STANDARD   : mag:ap # FAIL
+# AP_MAG_STANDARD   : mag:ap # FAIL
   AP_MAG_RADIUS     : SKIP # no accessor
   PEAK_FLUX_AS_MAG  : SKIP # no accessor
@@ -273 +267 @@
   CAL_PSF_MAG_SIG   : SKIP # photometry is not calibrated in the cmf
   SKY               : sky       
-  SKY_SIG           : sky_err   
+  SKY_SIGMA         : sky_err   
   PSF_CHISQ         : psf_chisq 
   CR_NSIGMA         : cr_nsigma 
@@ -303 +297 @@
   PSF_INST_FLUX     : SKIP # not ingested into DVO
   PSF_INST_FLUX_SIG : SKIP # not ingested into DVO
-  AP_MAG_STANDARD   : mag:aperinst # FAIL
+# AP_MAG_STANDARD   : mag:aperinst # FAIL
   AP_MAG_RAW        : SKIP # not ingested into DVO
   AP_MAG_RADIUS     : SKIP # not ingested into DVO
@@ -312 +306 @@
   PEAK_FLUX_AS_MAG  : SKIP # not ingested into DVO
   SKY               : sky       
-  SKY_SIG           : sky_err   
+  SKY_SIGMA         : sky_err   
   PSF_CHISQ         : psf_chisq 
   CR_NSIGMA         : cr_nsigma 
@@ -332 +326 @@
   MOMENTS_R1        : SKIP # not ingested into DVO
   MOMENTS_RH        : SKIP # not ingested into DVO
-  KRON_FLUX         : mag:kroninst
-  KRON_FLUX_ERR     : mag:kronerr
+  KRON_FLUX         : mag:kron:inst
+  KRON_FLUX_ERR     : mag:kron:err
   KRON_FLUX_INNER   : SKIP # not ingested into DVO
   KRON_FLUX_OUTER   : SKIP # not ingested into DVO

trunk/Ohana/src/addstar/test/tap.dvo

@@ -3 +3 @@
 if (not($?TAP_BREAK)) set TAP_BREAK = 0
 if (not($?TAP_VERBOSE)) set TAP_VERBOSE = 0
+
+macro tapEXEC
+  if ($0 < 3) 
+    echo "USAGE: tapEXEC (command)"
+    break
+  end
+
+  local i cmd result
+
+  $cmd = ""
+  for i 1 $0
+    $cmd = $cmd $$i
+  end
+
+  if ($TAP_VERBOSE >= 2) echo $cmd
+
+  break -auto off
+  exec $cmd >& tap.log
+  $result = $STATUS
+  break -auto on
+
+  if (not($result)) 
+    exec cat tap.log
+    echo "not ok : $cmd"
+    $TAP_NFAIL ++
+    $TAP_LAST = 0
+    if ($TAP_BREAK)
+     break
+    end
+  else
+    if ($TAP_VERBOSE >= 1) echo "ok : $cmd"
+    $TAP_LAST = 1
+  end
+  $TAP_NDONE ++
+end
 
 macro tapOK
@@ -11 +46 @@
 
   if ($1)
-    if ($TAP_VERBOSE)
+    if ($TAP_VERBOSE >= 1)
       echo "ok : $2"
     end

@@ -17 +17 @@
 relphot.Mcal.dMcal.png
 catdir.wise.test
+dvo.photcodes
+ptolemy.rc
+tap.log