Index: /branches/eam_branch_20070830/psphot/src/psphotApResid.c
===================================================================
--- /branches/eam_branch_20070830/psphot/src/psphotApResid.c	(revision 14918)
+++ /branches/eam_branch_20070830/psphot/src/psphotApResid.c	(revision 14919)
@@ -51,4 +51,5 @@
 
     psVector *mask    = psVectorAllocEmpty (300, PS_TYPE_U8);
+    psVector *mag     = psVectorAllocEmpty (300, PS_TYPE_F32);
     psVector *xPos    = psVectorAllocEmpty (300, PS_TYPE_F32);
     psVector *yPos    = psVectorAllocEmpty (300, PS_TYPE_F32);
@@ -56,9 +57,4 @@
     psVector *dMag    = psVectorAllocEmpty (300, PS_TYPE_F32);
     Npsf = 0;
-
-    FILE *dumpFile = NULL;
-    if (psTraceGetLevel("psphot") >= 5) {
-	dumpFile = fopen ("apresid.dat", "w");
-    }
 
     // select all good PM_SOURCE_TYPE_STAR entries
@@ -94,4 +90,5 @@
         }
 
+	mag->data.F32[Npsf]     = source->psfMag;
         apResid->data.F32[Npsf] = dap;
         xPos->data.F32[Npsf]    = model->params->data.F32[PM_PAR_XPOS];
@@ -101,11 +98,4 @@
 
         dMag->data.F32[Npsf] = model->dparams->data.F32[PM_PAR_I0] / model->params->data.F32[PM_PAR_I0];
-
-	if (psTraceGetLevel("psphot") >= 5) {
-	    fprintf (dumpFile, "%f %f  %f %f %f  %x\n", 
-		     xPos->data.F32[Npsf], yPos->data.F32[Npsf], 
-		     source->psfMag, dMag->data.F32[Npsf], apResid->data.F32[Npsf], 
-		     mask->data.U8[Npsf]);
-	}
 
         psVectorExtend (mask,    100, 1);
@@ -116,8 +106,4 @@
         Npsf ++;
     }
-
-    if (psTraceGetLevel("psphot") >= 5) {
-	fclose (dumpFile);
-    }
     
     psLogMsg ("psphot.apresid", PS_LOG_DETAIL, "measure aperture residuals for %d objects (%d skipped, %d failed, %ld invalid)\n",
@@ -134,34 +120,56 @@
     // XXX is this asymmetric clipping still needed?  this analysis should come after neighbors are subtracted...
     // 3hi/1lo sigma clipping on the rflux vs metric fit
-    psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN);
-    stats->min = 2.0;
-    stats->max = 3.0;
-
-    // XXX set the pmTrend2D mode, nXtrend, nYtrend based on the user inputs
-    // XXX psf->ApTrend = pmTrend2DAlloc (mode, readout->image->numCols, readout->image->numRows, nXtrend, nYtrend, stats);
-
-    psf->ApTrend = pmTrend2DAlloc (PM_TREND_MAP, readout->image, 5, 5, stats);
-    pmTrend2DFit (psf->ApTrend, xPos, yPos, apResid, dMag);
-    
+    // systematic error information
+    float errorScale = 0.0;
+    float errorFloor = 0.0;
+
+    float errorFloorMin = FLT_MAX;
+    int entryMin = -1;
+
+    // *** iterate over spatial scale until error Floor increases
+    // *** fit out the dap vs mag trend?
+    // *** stop if Npsf / (Nx * Ny) < 3
+    for (int i = 1; i < 10; i++) {
+
+	if (!psphotApResidTrend (readout, psf, Npsf, i, &errorScale, &errorFloor, mask, xPos, yPos, apResid, dMag)) {
+	    break;
+	}
+
+	// store the resulting errorFloor values and the scales, redo the best
+	if (errorFloor < errorFloorMin) {
+	    errorFloorMin = errorFloor;
+	    entryMin = i;
+	}
+    }
+    if (entryMin == -1) {
+	psAbort ("failed on ApResid Trend");
+    }
+
+    psphotApResidTrend (readout, psf, Npsf, entryMin, &errorScale, &errorFloor, mask, xPos, yPos, apResid, dMag);
+
+    // construct the fitted values and the residuals
+    psVector *apResidFit = pmTrend2DEvalVector (psf->ApTrend, xPos, yPos);
+    psVector *apResidRes = (psVector *) psBinaryOp (NULL, (void *) apResid, "-", (void *) apResidFit);
+    psVector *dMagSys = (psVector *) psBinaryOp (NULL, (void *) dMag, "*", (void *) psScalarAlloc(errorScale, PS_TYPE_F32));
+
     psphotSaveImage (NULL, psf->ApTrend->map->map, "apresid.map.fits");
 
-    // construct the fitted values and the residuals
-    psVector *fit   = pmTrend2DEvalVector (psf->ApTrend, xPos, yPos);
-    psVector *resid = (psVector *) psBinaryOp (NULL, (void *) apResid, "-", (void *) fit);
-
-    // measure scatter for sources with dMag < 0.01 (S/N = 100)
-    int Nkeep = 0;
-    psStats *residStats = psStatsAlloc (PS_STAT_SAMPLE_STDEV);
-    for (int i = 0; i < dMag->n; i++) {
-        if (dMag->data.F32[i] > 0.01) {
-            mask->data.U8[i] |= 0x02;
-        }
-        if (! mask->data.U8[i]) Nkeep ++;
-    }
-    psVectorStats (residStats, resid, NULL, mask, 0x03);
+    if (psTraceGetLevel("psphot") >= 5) {
+	FILE *dumpFile = fopen ("apresid.dat", "w");
+	for (int i = 0; i < xPos->n; i++) {
+	    fprintf (dumpFile, "%f %f  %f %f %f  %f %f  %x\n", 
+		     xPos->data.F32[i], yPos->data.F32[i], 
+		     mag->data.F32[i], dMag->data.F32[i], dMagSys->data.F32[i],
+		     apResid->data.F32[i], apResidRes->data.F32[i],
+		     mask->data.U8[i]);
+	}
+	fclose (dumpFile);
+    }
 
     // apply ApTrend results
-    psf->ApResid  = pmTrend2DEval (psf->ApTrend, 0.0, 0.0); // XXX use chip center?
-    psf->dApResid = residStats->sampleStdev;
+    float xc = 0.5*readout->image->numCols + readout->image->col0 + 0.5;
+    float yc = 0.5*readout->image->numRows + readout->image->row0 + 0.5;
+    psf->ApResid  = pmTrend2DEval (psf->ApTrend, xc, yc); // ap-fit at chip center
+    psf->dApResid = errorFloor;
     psf->nApResid = Npsf;
 
@@ -174,7 +182,8 @@
     psMetadataAdd (recipe, PS_LIST_TAIL, "NAPMIFIT", PS_DATA_S32 | PS_META_REPLACE, "ap residual scatter", psf->nApResid);
 
-    psLogMsg ("psphot.apresid", PS_LOG_INFO, "measure full-frame aperture residuals for %d of %d objects: %f sec\n", Nkeep, Npsf, psTimerMark ("psphot"));
+    // psLogMsg ("psphot.apresid", PS_LOG_INFO, "measure full-frame aperture residuals for %d of %d objects: %f sec\n", Nkeep, Npsf, psTimerMark ("psphot"));
     psLogMsg ("psphot.apresid", PS_LOG_DETAIL, "aperture residual: %f +/- %f\n", psf->ApResid, psf->dApResid);
 
+    psFree (mag);
     psFree (mask);
     psFree (xPos);
@@ -183,8 +192,4 @@
     psFree (dMag);
 
-    psFree (fit);
-    psFree (resid);
-    psFree (stats);
-    psFree (residStats);
     return true;
 }
@@ -196,2 +201,111 @@
 */
 
+bool psphotMagErrorScale (float *errorScale, float *errorFloor, psVector *dMag, psVector *dap, int nGroup) {
+
+    psStats *statsS = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
+    psStats *statsM = psStatsAlloc (PS_STAT_SAMPLE_MEAN);
+
+    // measure the trend in bins with 10 values each; if < 10 total, use them all
+    int nBin = PS_MAX (dMag->n / nGroup, 1);
+
+    // output vectors for ApResid trend 
+    psVector *dSo = psVectorAlloc (nBin, PS_TYPE_F32);
+    psVector *dMo = psVectorAlloc (nBin, PS_TYPE_F32);
+    psVector *dRo = psVectorAlloc (nBin, PS_TYPE_F32);
+
+    // use dMag to group the dMag and dap vectors
+    psVector *index = psVectorSortIndex (NULL, dMag);
+
+    // subset vectors for dMag and dap values within the given range
+    psVector *dMSubset = psVectorAllocEmpty (nGroup, PS_TYPE_F32);
+    psVector *dASubset = psVectorAllocEmpty (nGroup, PS_TYPE_F32);
+
+    int n = 0;
+    for (int i = 0; i < dMo->n; i++) {
+	int j;
+	for (j = 0; (j < nGroup) && (n < dMag->n); j++, n++) {
+	    int N = index->data.U32[n];
+	    dMSubset->data.F32[j] = dMag->data.F32[N];
+	    dASubset->data.F32[j] = dap->data.F32[N];
+	}
+	dMSubset->n = j;
+	dASubset->n = j;
+
+	psStatsInit (statsS);
+	psStatsInit (statsM);
+
+	psVectorStats (statsS, dASubset, NULL, NULL, 0xff);
+	psVectorStats (statsM, dMSubset, NULL, NULL, 0xff);
+
+	dSo->data.F32[i] = statsS->robustStdev;
+	dMo->data.F32[i] = statsM->sampleMean;
+
+	dRo->data.F32[i] = statsS->robustStdev / statsM->sampleMean;
+    }
+    psFree (dMSubset);
+    psFree (dASubset);
+    
+    psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN);
+    psVectorStats (stats, dRo, NULL, NULL, 0);
+
+    *errorScale = stats->sampleMedian;
+    *errorFloor = dSo->data.F32[0];
+
+    psFree (stats);
+    psFree (index);
+
+    psFree (dRo);
+    psFree (dMo);
+    psFree (dSo);
+    
+    psFree (statsS);
+    psFree (statsM);
+
+    return true;
+}
+
+bool psphotApResidTrend (pmReadout *readout, pmPSF *psf, int Npsf, int scale, float *errorScale, float *errorFloor, psVector *mask, psVector *xPos, psVector *yPos, psVector *apResid, psVector *dMag) {
+
+    int Nx, Ny;
+	
+    psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
+    stats->min = 2.0;
+    stats->max = 3.0;
+
+    if (readout->image->numCols > readout->image->numRows) {
+	Nx = scale;
+	Ny = PS_MAX (1, Nx * (readout->image->numRows / readout->image->numCols));
+    } else {
+	Ny = scale;
+	Nx = PS_MAX (1, Ny * (readout->image->numCols / readout->image->numRows));
+    }
+
+    if (Npsf < 3*Nx*Ny) {
+	return false;
+    }
+
+    // measure Trend2D for the current spatial scale
+    psFree (psf->ApTrend);
+    psf->ApTrend = pmTrend2DAlloc (PM_TREND_MAP, readout->image, Nx, Ny, stats);
+    pmTrend2DFit (psf->ApTrend, mask, 0xff, xPos, yPos, apResid, dMag);
+    
+    // construct the fitted values and the residuals
+    psVector *apResidFit = pmTrend2DEvalVector (psf->ApTrend, xPos, yPos);
+    psVector *apResidRes = (psVector *) psBinaryOp (NULL, (void *) apResid, "-", (void *) apResidFit);
+
+    // measure systematic errorFloor & systematic / photon scale factor
+    // XXX this is a bit arbitrary, but it forces ~3 stars from the bright bin per spatial bin
+    int nGroup = PS_MAX (3*Nx*Ny, 10);
+    psphotMagErrorScale (errorScale, errorFloor, dMag, apResidRes, nGroup);
+
+    psLogMsg ("psphot.apresid", PS_LOG_INFO, "result of %d x %d grid (%d stars per bin)\n", Nx, Ny, nGroup);
+    psLogMsg ("psphot.apresid", PS_LOG_INFO, "systematic error / photon error: %f\n", *errorScale);
+    psLogMsg ("psphot.apresid", PS_LOG_INFO, "systematic scatter floor: %f\n", *errorFloor);
+
+    psFree (stats);
+    psFree (apResidFit);
+    psFree (apResidRes);
+
+    return true;
+}
+