Changeset 41559 for trunk

Timestamp:

Apr 28, 2021, 6:02:02 PM (5 years ago)

Author:

eugene

Message:

when nights go bad, propagate the last Mcal to the exposures (same when exposures go bad); do not redeem nights

Location:

trunk/Ohana/src/relphot/src

Files:

• MosaicOps.c (modified) (7 diffs)
• TGroupOps.c (modified) (11 diffs)

trunk/Ohana/src/relphot/src/MosaicOps.c

@@ -772 +772 @@
   // this mosaic has been identified as poor.
   mosaic[myMosaic].flags |= ID_IMAGE_MOSAIC_POOR;
-  mosaic[myMosaic].McalPSF  = 0.0;
-  mosaic[myMosaic].McalAPER = 0.0;
 
   image = getimages (&Nimage, NULL);
 
   // all images should be marked as coming from a bad mosaic
-  // these will be fitted independently
+  // these will be fitted independently.  Make the starting solution consistent by
+  // setting the image McalPSF for the current best fit from the mosaic
+
   for (off_t i = 0; i < MosaicN_Image[myMosaic]; i++) {
     off_t im = MosaicToImage[myMosaic][i];
     image[im].flags |= ID_IMAGE_MOSAIC_POOR;
-  }
+    image[im].McalPSF = mosaic[myMosaic].McalPSF;
+    image[im].McalAPER = mosaic[myMosaic].McalAPER;
+  }
+
+  mosaic[myMosaic].McalPSF  = 0.0;
+  mosaic[myMosaic].McalAPER = 0.0;
 }
 
@@ -1359 +1364 @@
   }
   
-  fprintf (stderr, "MOSAIC time %.6f photcode %d Mcal: %f, dMcal: %f, chisq: %f, %d of %d\n",
-           ohana_sec_to_mjd(myMosaic[0].start), myMosaic[0].photcode, myMosaic[0].McalPSF, myMosaic[0].dMcal, myMosaic[0].McalChiSq, myMosaic[0].nFitPhotom, myMosaic[0].nValPhotom);
+  fprintf (stderr, "MOSAIC time %.6f photcode %d Mcal: %f, dMcal: %f, chisq: %f, %d of %d, useMmos: %d\n",
+           ohana_sec_to_mjd(myMosaic[0].start), myMosaic[0].photcode, psfStars->bSaveArray[0][0], myMosaic[0].dMcal, myMosaic[0].McalChiSq, myMosaic[0].nFitPhotom, myMosaic[0].nValPhotom, useMmos);
 
   return TRUE;
@@ -1860 +1865 @@
 
 /* mark mosaic if: abs(Mcal - <Mcal>) too large, dMcal too large */
+
+static float MinMaxChiSq = NAN;
+static float MinMaxScatter = NAN;
+
 void clean_mosaics () {
 
@@ -1871 +1880 @@
   ALLOCATE (mlist, double, Nmosaic);
   ALLOCATE (slist, double, Nmosaic);
+
+  // Measure the good/bad statistics using mosaics which are actually calibrated.
+  // But they could be applied to all mosaics below (allowing bad mosaics to become good)
+  // However, for now, we will NOT allow mosaics to be redeemed (bad -> good)
 
   int N = 0;
@@ -1885 +1898 @@
 
   liststats (mlist, NULL, NULL, N, &stats);
-  float MaxChiSq = stats.Upper90; 
+  float ChiSqUpper90 = stats.Upper90; 
+  if (isnan (MinMaxChiSq)) MinMaxChiSq = 2.0*stats.median;
+  float MaxChiSq = MAX (MinMaxChiSq, ChiSqUpper90);
 
   liststats (slist, NULL, NULL, N, &stats);
-  float MaxScatter = stats.Upper90;
-
-  fprintf (stderr, "MOSAIC: Max ChiSq: %f, Max Scatter: %f\n", MaxChiSq, MaxScatter);
+  float ScatterUpper90 = stats.Upper90; 
+  if (isnan (MinMaxScatter)) MinMaxScatter = 2.0*stats.median;
+  float MaxScatter = MAX (MinMaxScatter, ScatterUpper90);
+
+  fprintf (stderr, "MOSAIC: Max ChiSq: %f, Max Scatter: %f | ChiSquare Upper 90: %f, Scatter Upper 90: %f\n", MaxChiSq, MaxScatter, ChiSqUpper90, ScatterUpper90);
   
-  int Ntotal = 0, Nmark = 0, Nscatter = 0, Nfew = 0, Nchisq = 0;
+  int Ntotal = 0, Npoor = 0, Nmark = 0, Nscatter = 0, Nfew = 0, Nchisq = 0;
   for (int i = 0; i < Nmosaic; i++) {
     // if we are keeping ubercal sacrosanct, then we should not be allowed to break them...
@@ -1902 +1919 @@
 
     int mark = FALSE;
+
+    // we do not allow bad nights to be redeemed
+    if (mosaic[i].flags & ID_IMAGE_MOSAIC_POOR) {
+      Npoor ++;
+      continue;
+    }
 
     if (mosaic[i].flags & ID_IMAGE_PHOTOM_FEW) {
@@ -1923 +1946 @@
   }
 
-  fprintf (stderr, "%d of %d mosaics marked poor (%d scatter, %d chisq, %d few)\n",  Nmark, Ntotal, Nscatter, Nchisq, Nfew);
+  fprintf (stderr, "%d + %d of %d mosaics marked poor (%d scatter, %d chisq, %d few)\n",  Nmark, Npoor, Ntotal, Nscatter, Nchisq, Nfew);
 
   free (mlist);

trunk/Ohana/src/relphot/src/TGroupOps.c

@@ -453 +453 @@
   // this tgroup has been identified as poor.
   tgroup->flags |= ID_IMAGE_NIGHT_POOR;
-  tgroup->McalPSF  = 0.0;
-  tgroup->McalAPER = 0.0;
-  tgroup->dKlam    = 0.0;
 
   // all associated mosaics should be marked as coming from a bad night
-  // these will be fitted independently
+  // these will be fitted independently.  Make the starting solution consistent by
+  // setting the mosaic McalPSF for the current best fit from the tgroup
+
   for (off_t j = 0; j < tgroup->Nmosaic; j++) {
-    tgroup->mosaic[j]->flags |= ID_IMAGE_NIGHT_POOR;
+    Mosaic *mos = tgroup->mosaic[j];
+    mos->flags |= ID_IMAGE_NIGHT_POOR;
+    mos->McalPSF = (TGROUP_FIT_AIRMASS) ? tgroup->McalPSF + tgroup->dKlam*(mos->secz - 1.0) : tgroup->McalPSF;
+    mos->McalAPER = (TGROUP_FIT_AIRMASS) ? tgroup->McalAPER + tgroup->dKlam*(mos->secz - 1.0) : tgroup->McalAPER;
   }
 
@@ -470 +472 @@
     image[im].flags |= ID_IMAGE_NIGHT_POOR;
   }
-}
-
+
+  // reset the tgroup to have no impact on the solution 
+  tgroup->McalPSF  = 0.0;
+  tgroup->McalAPER = 0.0;
+  tgroup->dKlam    = 0.0;
+}
+
+// for now, we do not allow a night to be redeemed, so this
+// operation should have no impact.
 void markGoodTGroup (TGroup *tgroup) {
 
@@ -793 +802 @@
   // for testing, supply the MJD of a night or nights here to dump the full list of measurements
   int testImage = FALSE;
-  // testImage |= (abs(mjdStart - 57353) < 0.1);
+  testImage |= (abs(mjdStart - 58064) < 0.1);
+  testImage |= (abs(mjdStart - 58082) < 0.1);
+  testImage |= (abs(mjdStart - 58770) < 0.1);
+  testImage |= (abs(mjdStart - 55843) < 0.1);
+  testImage |= (abs(mjdStart - 56967) < 0.1);
+  testImage |= (abs(mjdStart - 57329) < 0.1);
+  testImage |= (abs(mjdStart - 56614) < 0.1);
 
   FILE *fout = NULL;
@@ -930 +945 @@
   double altSigma = (stats.Upper80 - stats.Lower20) / 1.6;
 
-  TGTimes *mygroup = (TGTimes *) myTGroup->parent;
-  fprintf (stderr, "TGroup Stats %f,%d : %d %d %6.3f %6.3f\n", ohana_sec_to_mjd(mygroup->start), myTGroup->photcode, (int) N, (int) myTGroup->Nmeasure, stats.median, altSigma);
-
   // do anything special with identified good nights?
   // (myTGroup->flags & ID_IMAGE_PHOTOM_UBERCAL)
@@ -961 +973 @@
   // bright end scatter (systematic error)
   myTGroup->dMsys = brightStars->sigma;
+
+  TGTimes *mygroup = (TGTimes *) myTGroup->parent;
+  fprintf (stderr, "TGroup Stats %f,%d : %d %d %6.3f %6.3f %6.3f %d\n", ohana_sec_to_mjd(mygroup->start), myTGroup->photcode, (int) N, (int) myTGroup->Nmeasure, psfStars->bSaveArray[0][0], stats.median, altSigma, useMgrp);
 
   if (testImage) {
@@ -1242 +1257 @@
  */
 
+static float MinMaxChiSq = NAN;
+static float MinMaxScatter = NAN;
+
 void clean_tgroups () {
 
@@ -1254 +1272 @@
 
   // measure the good/bad statistics using nights which are actually calibrated
-  // but they will be applied to all nights below (allowing bad nights to become good)
+  // But they could be applied to all nights below (allowing bad nights to become good)
+  // However, for now, we will NOT allow nights to be redeemed (bad -> good)
   int N = 0;
   for (int i = 0; i < NtgroupTimes; i++) {
@@ -1270 +1289 @@
 
   liststats (mlist, NULL, NULL, N, &stats);
-  float MaxChiSq = stats.Upper90; 
+  float ChiSqUpper90 = stats.Upper90; 
+  if (isnan (MinMaxChiSq)) MinMaxChiSq = 2.0*stats.median;
+  float MaxChiSq = MAX (MinMaxChiSq, ChiSqUpper90);
 
   liststats (slist, NULL, NULL, N, &stats);
-  float MaxScatter = stats.Upper90;
-
-  fprintf (stderr, "TGROUPS: Max ChiSq: %f, Max Scatter: %f\n", MaxChiSq, MaxScatter);
+  float ScatterUpper90 = stats.Upper90; 
+  if (isnan (MinMaxScatter)) MinMaxScatter = 2.0*stats.median;
+  float MaxScatter = MAX (MinMaxScatter, ScatterUpper90);
+
+  fprintf (stderr, "TGROUPS: Max ChiSq: %f, Max Scatter: %f | ChiSquare Upper 90: %f, Scatter Upper 90: %f\n", MaxChiSq, MaxScatter, ChiSqUpper90, ScatterUpper90);
   
-  int Ntotal = 0, Nmark = 0, Nscatter = 0, Nchisq = 0, NfewNights = 0, NfewExp = 0;
+  int Ntotal = 0, Npoor = 0, Nmark = 0, Nscatter = 0, Nchisq = 0, NfewNights = 0, NfewExp = 0;
   for (int i = 0; i < NtgroupTimes; i++) {
     TGroup *tgroup = tgroupTimes[i][0].byCode;
@@ -1285 +1308 @@
 
       int mark = FALSE;
+
+      // we do not allow bad nights to be redeemed
+      if (tgroup[j].flags & ID_IMAGE_NIGHT_POOR) {
+        fprintf (stderr, "TGroup Already Poor, not redeemed %f,%d : %6.3f %6.3f %6.3f\n", ohana_sec_to_mjd(tgroupTimes[i][0].start), tgroup[j].photcode, tgroup[j].McalPSF, tgroup[j].stdev, tgroup[j].McalChiSq);
+        Npoor ++;
+        continue;
+      }
 
       // too few measurements (set in setMgrp_tgroup)
@@ -1309 +1339 @@
         Nmark ++;
         markBadTGroup(&tgroup[j]); // mark the tgroup & associated images & mosaics with a bad night
+        fprintf (stderr, "TGroup Poor %f,%d : %6.3f %6.3f %6.3f\n", ohana_sec_to_mjd(tgroupTimes[i][0].start), tgroup[j].photcode, tgroup[j].McalPSF, tgroup[j].stdev, tgroup[j].McalChiSq);
       } else {
         markGoodTGroup(&tgroup[j]); // mark the tgroup & associated images & mosaics with a good night
@@ -1315 +1346 @@
   }
 
-  fprintf (stderr, "%d of %d tgroups marked poor (%d scatter, %d few nights, %d few exposures, %d chisq)\n",  Nmark, Ntotal, Nscatter, NfewNights, NfewExp, Nchisq);
+  fprintf (stderr, "%d + %d of %d tgroups marked poor (%d scatter, %d few nights, %d few exposures, %d chisq)\n",  Nmark, Npoor, Ntotal, Nscatter, NfewNights, NfewExp, Nchisq);
 
   free (mlist);