Changeset 35255

Timestamp:

Mar 7, 2013, 4:33:58 AM (13 years ago)

Author:

eugene

Message:

add test for existing externID entries

Location:

branches/eam_branches/ipp-20130306/Ohana/src/addstar

Files:

• Makefile (modified) (2 diffs)
• include/addstar.h (modified) (2 diffs)
• src/ExternImageIDs.c (modified) (7 diffs)
• src/UpdateImageIDs.c (modified) (1 diff)
• src/addstar.c (modified) (1 diff)
• test/simple.dvo (modified) (3 diffs)

branches/eam_branches/ipp-20130306/Ohana/src/addstar/Makefile

@@ -90 +90 @@
 $(SRC)/GetFileMode.$(ARCH).o \
 $(SRC)/ReadImageHeader.$(ARCH).o \
+$(SRC)/ExternImageIDs.$(ARCH).o \
 $(SRC)/UpdateImageIDs.$(ARCH).o \
 $(SRC)/update_coords.$(ARCH).o \
@@ -207 +208 @@
 $(SRC)/ReadStarsTEXT.$(ARCH).o \
 $(SRC)/ReadStarsSDSS.$(ARCH).o \
+$(SRC)/ExternImageIDs.$(ARCH).o \
 $(SRC)/UpdateImageIDs.$(ARCH).o \
 $(SRC)/FilterStars.$(ARCH).o \

branches/eam_branches/ipp-20130306/Ohana/src/addstar/include/addstar.h

@@ -56 +56 @@
   char *refcat;
 } DVO_DATA;
+
+# define IDTYPE int
+
+typedef struct {
+  IDTYPE Nimage;
+  IDTYPE minID;
+  IDTYPE maxID;
+  IDTYPE range;
+  IDTYPE *imageID;
+  IDTYPE *externID;
+  char *found;
+} ExtID_Index;
 
 typedef struct sockaddr_in SockAddress;
@@ -275 +287 @@
 int UpdateImageIDs (Stars *stars, unsigned int Nstars, Image *images, off_t Nimages);
 
+int CheckDuplicateImageIDs (Image *images, off_t Nimages);
+
 int LoadDataSDSS (FILE *f, char *file, Image **images, off_t *nvalid, Stars **stars, unsigned int *Nstars, Header **headers, off_t *extsize, HeaderSet *headerSets, off_t Nimages);
 int LoadDataPMM (FILE *f, char *file, Image **images, off_t *nvalid, Stars **stars, unsigned int *Nstars);

branches/eam_branches/ipp-20130306/Ohana/src/addstar/src/ExternImageIDs.c

@@ -1 +1 @@
 # include "addstar.h"
-
-typedef struct {
-  off_t Nimage;
-  off_t minID;
-  off_t maxID;
-  off_t range;
-  off_t *value;
-  char *found;
-} ExtID_Index;
 
 ExtID_Index *ExtID_Load (char *filename) {
@@ -24 +15 @@
   FILE *f = fopen (filename, "r");
   if (!f) {
-    fprintf (stderr, "ERROR: cannot open image subset file %s\n", filename);
+    fprintf (stderr, "ERROR: cannot open imageID/externID file %s\n", filename);
     return NULL;
   }
@@ -30 +21 @@
   /* load in PHU segment (ignore) */
   if (!gfits_fread_header (f, &header)) {
-    if (VERBOSE) fprintf (stderr, "can't read image subset header\n");
+    if (VERBOSE) fprintf (stderr, "can't read imageID/externID header\n");
     fclose (f);
     return NULL;
   }
   if (!gfits_fread_matrix (f, &matrix, &header)) {
-    if (VERBOSE) fprintf (stderr, "can't read image subset matrix\n");
+    if (VERBOSE) fprintf (stderr, "can't read imageID/externID matrix\n");
     gfits_free_header (&header);
     fclose (f);
@@ -56 +47 @@
   fclose (f);
 
-  // a bit annoying : we read the entire block of data, then extract the columns, then set the image structure values.
-  // this means I need 3 copies in memory at some point.  ugh.
+  char type[16];
+  extID->externID = gfits_get_bintable_column_data (&theader, &ftable, "EXTERN_ID", type, &Nrow, &Ncol);
+  myAssert (!strcmp(type, "int"), "wrong column type");
 
-  char type[16];
-
-  extID->value = gfits_get_bintable_column_data (&theader, &ftable, "EXTERN_ID", type, &Nrow, &Ncol);
+  extID->imageID  = gfits_get_bintable_column_data (&theader, &ftable, "IMAGE_ID", type, &Nrow, &Ncol);
   myAssert (!strcmp(type, "int"), "wrong column type");
 
   extID->Nimage = Nrow;
 
+  // find the range of externID values
   extID->minID = -1;
   extID->maxID = -1;
   for (i = 0; i < extID->Nimage; i++) {
-    if (extID->minID == -1) extID->minID = extID->value[i];
-    extID->minID = MIN(extID->minID, extID->value[i]);
-    extID->maxID = MAX(extID->maxID, extID->value[i]);
+    if (extID->minID == -1) extID->minID = extID->externID[i];
+    extID->minID = MIN(extID->minID, extID->externID[i]);
+    extID->maxID = MAX(extID->maxID, extID->externID[i]);
   }
   extID->range = extID->maxID- extID->minID + 1;
   
+  // init the index
   ALLOCATE (extID->found, char, extID->range);
   for (i = 0; i < extID->range; i++) extID->found[i] = FALSE;
 
+  // generate the index (set this value to the imageID?)
   for (i = 0; i < extID->Nimage; i++) {
-    off_t N = extID->value[i] - extID->minID;
-    extID->found[N] = TRUE
+    IDTYPE N = extID->externID[i] - extID->minID;
+    extID->found[N] = TRUE;
   }
 
@@ -86 +79 @@
 }
 
+// STATUS is value expected for success
+# define CHECK_STATUS(STATUS,MSG,...)                                   \
+  if (!(STATUS)) {                                                      \
+    fprintf (stderr, MSG, __VA_ARGS__);                                 \
+    return FALSE;                                                       \
+  }
+
+int ExtID_Save (char *filename, ExtID_Index *extID) {
+
+  Header header;
+  Header theader;
+  Matrix matrix;
+  FTable ftable;
+
+  gfits_init_header (&header);
+  header.extend = TRUE;
+  gfits_create_header (&header);
+  gfits_create_matrix (&header, &matrix);
+
+  gfits_create_table_header (&theader, "BINTABLE", "EXTERN_ID");
+  gfits_define_bintable_column (&theader, "J", "IMAGE_ID", "image ID", NULL, 1.0, 0);
+  gfits_define_bintable_column (&theader, "J", "EXTERN_ID", "extern ID", NULL, 1.0, 0);
+
+  // generate the output array that carries the data
+  gfits_create_table (&theader, &ftable);
+
+  // add the columns to the output array
+  gfits_set_bintable_column (&theader, &ftable, "IMAGE_ID",       extID->imageID,  extID->Nimage);
+  gfits_set_bintable_column (&theader, &ftable, "EXTERN_ID",      extID->externID, extID->Nimage);
+
+  FILE *f = fopen (filename, "w");
+  if (!f) {
+    fprintf (stderr, "ERROR: cannot open extID file for output %s\n", filename);
+    return FALSE;
+  }
+
+  int status;
+  status = gfits_fwrite_header  (f, &header);
+  CHECK_STATUS (status, "ERROR: cannot write header for imageID/externID %s\n", filename);
+
+  status = gfits_fwrite_matrix  (f, &matrix);
+  CHECK_STATUS (status, "ERROR: cannot write matrix for imageID/externID %s\n", filename);
+
+  status = gfits_fwrite_Theader (f, &theader);
+  CHECK_STATUS (status, "ERROR: cannot write table header for imageID/externID %s\n", filename);
+
+  status = gfits_fwrite_table  (f, &ftable);
+  CHECK_STATUS (status, "ERROR: cannot write table data for imageID/externID %s\n", filename);
+
+  gfits_free_header (&header);
+  gfits_free_matrix (&matrix);
+  gfits_free_header (&theader);
+  gfits_free_table (&ftable);
+
+  int fd = fileno (f);
+
+  status = fflush (f);
+  CHECK_STATUS (!status, "ERROR: cannot flush file imageID/externID %s\n", filename);
+
+  status = fsync (fd);
+  CHECK_STATUS (!status, "ERROR: cannot flush file imageID/externID %s\n", filename);
+
+  status = fclose (f);
+  CHECK_STATUS (!status, "ERROR: problem closing imageID/externID file %s\n", filename);
+
+  return TRUE;
+}
+
 int CheckDuplicateImageIDs (Image *images, off_t Nimages) {
+
+  IDTYPE i, offset;
 
   // load the image ID table
@@ -94 +157 @@
   ExtID_Index *extID = ExtID_Load (filename);
 
-  for (i = 0; i < Nimages; i++) {
-    off_t ID = images[i].externID;
+  // if we 
+
+  for (i = 0; extID && (i < Nimages); i++) {
+    IDTYPE ID = images[i].externID;
     if (ID == 0) continue;
 
@@ -106 +171 @@
 
     if (!extID->found[offset]) continue;
-    fprintf (stderr, "duplicate external image ID "OFF_T_FMT" found, exiting\n", extID);
+    fprintf (stderr, "duplicate external image ID %lld found, exiting\n", (long long) ID);
     exit (1);
   }
   
+  if (!extID) {
+    ALLOCATE (extID, ExtID_Index, 1);
+    ALLOCATE (extID->imageID,  IDTYPE, 1);
+    ALLOCATE (extID->externID, IDTYPE, 1);
+    extID->Nimage = 0;
+  }
+
+  // extend the storage arrays
+  REALLOCATE (extID->imageID,  IDTYPE, extID->Nimage + Nimages);
+  REALLOCATE (extID->externID, IDTYPE, extID->Nimage + Nimages);
+
   // append the new ext IDs and save
   for (i = 0; i < Nimages; i++) {
-    append();
+    if (images[i].externID == 0) continue;
+    extID->imageID[extID->Nimage] = images[i].imageID;
+    extID->externID[extID->Nimage] = images[i].externID;
+    extID->Nimage ++;
   }
 

branches/eam_branches/ipp-20130306/Ohana/src/addstar/src/UpdateImageIDs.c

@@ -51 +51 @@
   }
 
+  // set and update the imageID sequence
+  CheckDuplicateImageIDs (images, Nimages);
+
   imageID += Nimages;
   status = gfits_modify (&db.header, "IMAGEID", "%u", 1, imageID);

branches/eam_branches/ipp-20130306/Ohana/src/addstar/src/addstar.c

@@ -56 +56 @@
 
       // set and update the imageID sequence
-      CheckDuplicateImageIDs (images, Nimages);
-
-      // set and update the imageID sequence
       UpdateImageIDs (stars, Nstars, images, Nimages);
 

branches/eam_branches/ipp-20130306/Ohana/src/addstar/test/simple.dvo

@@ -68 +68 @@
 
   for i 0 $testfields:n
+    if ($TAP_VERBOSE) echo $testfields:$i
     list name -split $testfields:$i
     if ("$name:0" == "SKIP") continue
@@ -337 +338 @@
   PSF_INST_FLUX     : SKIP # not ingested into DVO
   PSF_INST_FLUX_SIG : SKIP # not ingested into DVO
-  AP_MAG_STANDARD   : mag:aperinst # FAIL
+  AP_MAG            : mag:aperinst # FAIL
   AP_MAG_RAW        : SKIP # not ingested into DVO
   AP_MAG_RADIUS     : SKIP # not ingested into DVO
@@ -346 +347 @@
   PEAK_FLUX_AS_MAG  : SKIP # not ingested into DVO
   SKY               : sky       
-  SKY_SIG           : sky_err   
+  SKY_SIGMA         : sky_err   
   PSF_CHISQ         : psf_chisq 
   CR_NSIGMA         : cr_nsigma