Changeset 35721

Timestamp:

Jun 28, 2013, 2:01:13 PM (13 years ago)

Author:

eugene

Message:

add mode to reload the image stats and write summary info; add option to force -image-only but not -update

Location:

branches/eam_branches/ipp-20130509/Ohana/src/delstar

Files:

• include/delstar.h (modified) (3 diffs)
• src/ImageOpsFixLAP.c (modified) (3 diffs)
• src/args.c (modified) (2 diffs)
• src/delete_fix_LAP.c (modified) (5 diffs)
• src/delstar.c (modified) (1 diff)

branches/eam_branches/ipp-20130509/Ohana/src/delstar/include/delstar.h

@@ -77 +77 @@
 
 char *PHOTCODE_LIST;
+char *UNIQUER;
 
 int    MODE;
-enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_FIX_LAP};
+enum {MODE_NONE, MODE_IMAGENAME, MODE_IMAGEFILE, MODE_TIME, MODE_ORPHAN, MODE_MISSED, MODE_PHOTCODES, MODE_DUP_IMAGES, MODE_DUP_MEASURES, MODE_FIX_LAP, MODE_FIX_LAP_STATS};
 
 char DateKeyword[64], DateMode[64], UTKeyword[64], MJDKeyword[64], JDKeyword[64];
@@ -151 +152 @@
 int delete_fix_LAP_measures (off_t *measureDrop, Catalog *catalog, ImageSubset *image, off_t Nimage);
 int delete_fix_LAP_parallel (SkyList *sky, ImageSubset *image, off_t Nimage);
+int delete_fix_LAP_setstats (ImageSubset *image, off_t Nimage);
 
 void initImageIndex (ImageSubset *image, off_t Nimage_init);
@@ -166 +168 @@
 int ImageValidLoad(char *filename);
 void SummaryImageStats (ImageSubset *image, off_t Nimage_comp);
+void SummaryImageStats_Full (Image *image, off_t Nimage_comp);
 void BumpInvalidImage (int Seq);
 void BumpValidImage (int Seq);

branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/ImageOpsFixLAP.c

@@ -253 +253 @@
 
   int Nfixed = 0;
+  int Nphu   = 0;
+  int Nexp   = 0;
+  int Nstk   = 0;
   int Nskip  = 0;
   int Nbad_1 = 0;
@@ -258 +261 @@
 
   for (i = 0; i < Nimage_comp; i++) {
-    if (image[i].photcode == 0) continue; // skip images with 0 photcode (eg, PHU)
+    if (image[i].photcode == 0) {
+      Nphu ++;
+      continue; // skip images with 0 photcode (eg, PHU)
+    }
+
+    if ((image[i].photcode > 10000) && (image[i].photcode < 10600)) Nexp ++;
+    if ((image[i].photcode >= 11000) && (image[i].photcode <= 11600)) Nstk ++;
 
     if (Nimage_valid[i] + Nimage_invalid[i] == 0) Nskip ++;
@@ -270 +279 @@
   }
 
-  fprintf (stderr, "Nfixed: %d, Nskip %d, Ndrop (>1%%) %d, Ndrop (>5%%) %d\n", Nfixed, Nskip, Nbad_1, Nbad_2);
+  fprintf (stderr, "Nimage: "OFF_T_FMT"\n", Nimage_comp);
+  fprintf (stderr, "Nfixed: %d\n", Nfixed);
+  fprintf (stderr, "Nphu: %d\n", Nphu);
+  fprintf (stderr, "Nexp: %d\n", Nexp);
+  fprintf (stderr, "Nstk: %d\n", Nstk);
+  fprintf (stderr, "Nskip: %d\n", Nskip);
+  fprintf (stderr, "Ndrop (>1%%): %d\n", Nbad_1);
+  fprintf (stderr, "Ndrop (>5%%): %d\n", Nbad_2);
+}
+
+void SummaryImageStats_Full (Image *image, off_t Nimage_comp) {
+
+  off_t i;
+
+  // let's report the following:
+  // Nimages with corrected image IDs
+  // Nimages with Nsum / Nstar < 0.95
+  // Nimages with Nsum / Nstar < 0.99
+
+  int Nfixed = 0;
+  int Nphu   = 0;
+  int Nexp   = 0;
+  int Nstk   = 0;
+  int Nskip  = 0;
+  int Nbad_1 = 0;
+  int Nbad_2 = 0;
+
+  BuildChipMatch (image, Nimage);
+
+  for (i = 0; i < Nimage_comp; i++) {
+    if (image[i].photcode == 0) {
+      Nphu ++;
+      continue; // skip images with 0 photcode (eg, PHU)
+    }
+
+    if ((image[i].photcode > 10000) && (image[i].photcode < 10600)) Nexp ++;
+    if ((image[i].photcode >= 11000) && (image[i].photcode <= 11600)) Nstk ++;
+
+    float found_ratio = (Nimage_valid[i] + Nimage_invalid[i]) / (float) image[i].nstar;
+
+    if ((Nimage_valid[i] + Nimage_invalid[i] == 0) || (found_ratio < 0.995)) {
+      double r, d;
+      if (!FindMosaicForImage (image, Nimage, i)) { 
+        fprintf (stderr, "failed to get astrometry for %s\n", image[i].name);
+      } else {
+        if (!strcmp(&image[i].coords.ctype[4], "-DIS")) {
+          XY_to_RD (&r, &d, 0.0, 0.0, &image[i].coords);
+        } else {
+          XY_to_RD (&r, &d, 0.5*image[i].NX, 0.5*image[i].NY, &image[i].coords);
+        }
+      }
+      char *date = ohana_sec_to_date (image[i].tzero);
+      fprintf (stderr, "skip : %s %d  %10.6f %10.6f  %5.3f  %s\n", date, image[i].photcode, r, d, found_ratio, image[i].name);
+      free (date);
+      Nskip ++;
+    }
+
+    if (Nimage_invalid[i] > 0) Nfixed ++;
+   
+    
+    if (found_ratio < 0.99) Nbad_1 ++;
+    if (found_ratio < 0.95) Nbad_2 ++;
+  }
+
+  fprintf (stderr, "Nimage: "OFF_T_FMT"\n", Nimage_comp);
+  fprintf (stderr, "Nfixed: %d\n", Nfixed);
+  fprintf (stderr, "Nphu: %d\n", Nphu);
+  fprintf (stderr, "Nexp: %d\n", Nexp);
+  fprintf (stderr, "Nstk: %d\n", Nstk);
+  fprintf (stderr, "Nskip: %d\n", Nskip);
+  fprintf (stderr, "Ndrop (>1%%): %d\n", Nbad_1);
+  fprintf (stderr, "Ndrop (>5%%): %d\n", Nbad_2);
 }
 

branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/args.c

@@ -71 +71 @@
   }
 
-  // we should never delete the images before the measures (or we won't know what to delete)
-  // -image-only is for testing
+  // We should generally not delete the images before the measures (or we won't know what
+  // to delete). -image-only is for testing, -image-only-force -update WILL delete the images 
   IMAGE_ONLY = FALSE;
   if ((N = get_argument (argc, argv, "-image-only"))) {
     IMAGE_ONLY = TRUE;
     UPDATE = FALSE;
+    remove_argument (N, &argc, argv);
+  }
+  if ((N = get_argument (argc, argv, "-image-only-force"))) {
+    IMAGE_ONLY = TRUE;
     remove_argument (N, &argc, argv);
   }
@@ -179 +183 @@
     if (MODE != MODE_NONE) usage();
     MODE = MODE_FIX_LAP;
+    remove_argument (N, &argc, argv);
+  }
+  UNIQUER = NULL;
+  if ((N = get_argument (argc, argv, "-fix-LAP-imstats"))) {
+    if (MODE != MODE_NONE) usage();
+    MODE = MODE_FIX_LAP_STATS;
+    remove_argument (N, &argc, argv);
+    UNIQUER = strcreate (argv[N]);
     remove_argument (N, &argc, argv);
   }

branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delete_fix_LAP.c

@@ -23 +23 @@
   }
 
+  // this is needed in the master and client jobs (in master, just so Nimage is set in ImageOpsFixLAP.c)
+  initImageIndex (image, Nimage);
+
   // launch the remote jobs
   if (PARALLEL && !HOST_ID) {
@@ -28 +31 @@
     return status;
   }
-
-  initImageIndex (image, Nimage);
 
   // delete detections from region
@@ -170 +171 @@
 
   for (i = 0; i < table->Nhosts; i++) {
-    while (!ImageValidLoad (table->hosts[i].results)) {
+    if (!table->hosts[i].results) continue;
+    if (!ImageValidLoad (table->hosts[i].results)) {
+      fprintf (stderr, "failed to read data from %s\n", table->hosts[i].hostname);
+    }
+  }
+
+  return TRUE;
+}
+
+// CATDIR is supplied globally
+int delete_fix_LAP_setstats (ImageSubset *image, off_t Nimage) {
+
+  // load the current sky table (layout of all SkyRegions) 
+  SkyTable *sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, TRUE, -1, VERBOSE);
+  if (!sky) {
+    fprintf (stderr, "ERROR running loading sky table from %s\n", CATDIR);
+    exit (2);
+  }
+  SkyTableSetFilenames (sky, CATDIR, "cpt");
+
+  // determine the populated SkyRegions overlapping the requested area (default depth)
+  SkyList *skylist = SINGLE_CPT ? SkyRegionByCPT (sky, SINGLE_CPT) : SkyListByPatch (sky, -1, &UserPatch);
+  if (!skylist) {
+    fprintf (stderr, "ERROR setting up skylist for %s\n", CATDIR);
+    exit (2);
+  }
+  fprintf (stderr, "setup sky\n");
+
+  // this is needed in the master and client jobs (in master, just so Nimage is set in ImageOpsFixLAP.c)
+  initImageIndex (image, Nimage);
+  fprintf (stderr, "make index\n");
+
+  // write out the subset table of image information
+  char imageFile[512];
+  snprintf (imageFile, 512, "%s/delstar.fixLAP.%s.dat", CATDIR, UNIQUER);
+
+  // load the list of hosts
+  HostTable *table = HostTableLoad (CATDIR, sky->hosts);
+  if (!table) {
+    fprintf (stderr, "ERROR: failure reading Host Table %s for database %s\n", sky->hosts, CATDIR);
+    exit (1);
+  }    
+
+  int i;
+  for (i = 0; i < table->Nhosts; i++) {
+
+    // ensure that the paths are absolute path names
+    char *tmppath = abspath (table->hosts[i].pathname, DVO_MAX_PATH);
+    free (table->hosts[i].pathname);
+    table->hosts[i].pathname = tmppath;
+
+    ALLOCATE (table->hosts[i].results, char, 1024);
+    snprintf (table->hosts[i].results, 1024, "%s/delstar.fixLAP.imstats.%s.dat", table->hosts[i].pathname, UNIQUER);
+
+    fprintf (stderr, "read %s\n", table->hosts[i].results);
+    if (!ImageValidLoad (table->hosts[i].results)) {
       fprintf (stderr, "failed to read data from %s\n", table->hosts[i].hostname);
     }
@@ -517 +573 @@
         }
         measure[j].imageID = imageID;
-        BumpInvalidImage (imageN);
+        BumpInvalidImage (Seq);
         Ninvalid ++;
         continue;
@@ -568 +624 @@
         }
         measure[j].imageID = imageID;
-        BumpInvalidImage (imageN);
+        BumpInvalidImage (Seq);
         Ninvalid ++;
         continue;

branches/eam_branches/ipp-20130509/Ohana/src/delstar/src/delstar.c

@@ -46 +46 @@
         break;
       }
+    case MODE_FIX_LAP_STATS: {
+        off_t Nimage;
+        Image *image = gfits_table_get_Image (&db.ftable, &Nimage, &db.swapped);
+        if (!image) {
+          fprintf (stderr, "ERROR: failed to read images\n");
+          exit (2);
+        }
+        // dvo_image_unlock (&db); 
+        fprintf (stderr, "read images\n");
+
+        // the imageSubset here is a reduced set of fields, not a reduced set of images
+        ImageSubset *subset = ImagesToSubset (image, Nimage);
+        fprintf (stderr, "make subset\n");
+
+        if (!delete_fix_LAP_setstats (subset, Nimage)) exit (1);
+        SummaryImageStats_Full (image, Nimage);
+
+        exit (0);
+        break;
+      }
     case MODE_IMAGEFILE:
       delete_imagefile (&db);